Comment typo fix

We forgot to lock it during replace, so data rate. This is simpler.

.gitignore

@@ -1,4 +1,6 @@
-./syncthing
+syncthing
+!gui/syncthing
+!Godeps/_workspace/src/github.com/syncthing
 syncthing.exe
 *.tar.gz
 *.zip
@@ -9,8 +11,6 @@ files/pidx
 bin
 perfstats*.csv
 coverage.xml
-!gui/scripts/syncthing
-syncthing.md5
-syncthing.exe.md5
+syncthing.sig
 RELEASE
 deb

AUTHORS

@@ -56,7 +56,7 @@ Phill Luby <phill.luby@newredo.com>
 Piotr Bejda <piotrb10@gmail.com>
 Ryan Sullivan <kayoticsully@gmail.com>
 Sergey Mishin <ralder@yandex.ru>
-Stefan Tatschner <stefan@sevenbyte.org>
+Stefan Tatschner <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
 Tim Abell <tim@timwise.co.uk>
 Tobias Nygren <tnn@nygren.pp.se>
 Tomas Cerveny <kozec@kozec.com>

Godeps/Godeps.json

@@ -1,13 +1,13 @@
 {
 	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.5",
+	"GoVersion": "go1.5.1",
 	"Packages": [
 		"./cmd/..."
 	],
 	"Deps": [
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "d47913b1412890a261b9fefae99d72d2bf5aebd8"
+			"Rev": "74ddf82598bc4745b965729e9c6a463bedd33049"
 		},
 		{
 			"ImportPath": "github.com/calmh/du",
@@ -39,15 +39,23 @@
 		},
 		{
 			"ImportPath": "github.com/syncthing/protocol",
-			"Rev": "388a29bbe21d8772ee4c29f4520aa8040309607d"
+			"Rev": "f91191218b192ace841c878f161832d19c09145a"
+		},
+		{
+			"ImportPath": "github.com/syncthing/relaysrv/client",
+			"Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147"
+		},
+		{
+			"ImportPath": "github.com/syncthing/relaysrv/protocol",
+			"Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147"
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "b743d92d3215f11c9b5ce8830fafe1f16786adf4"
+			"Rev": "1a9d62f03ea92815b46fcaab357cfd4df264b1a0"
 		},
 		{
 			"ImportPath": "github.com/thejerf/suture",
-			"Rev": "fc7aaeabdc43fe41c5328efa1479ffea0b820978"
+			"Rev": "860b44045335c64a6d54ac7eed22a3aedfc687c9"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
@@ -63,11 +71,11 @@
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
+			"Rev": "81bf7719a6b7ce9b665598222362b50122dfc13b"
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
+			"Rev": "81bf7719a6b7ce9b665598222362b50122dfc13b"
 		},
 		{
 			"ImportPath": "golang.org/x/net/internal/iana",

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -4,5 +4,6 @@ go:
   - 1.1
   - 1.2
   - 1.3
-  - 1.4
+  - 1.4
+  - 1.5
   - tip

Godeps/_workspace/src/github.com/syncthing/protocol/message.go

@@ -20,6 +20,7 @@ type FileInfo struct {
 	Modified     int64
 	Version      Vector
 	LocalVersion int64
+	CachedSize   int64       // noencode (cache only)
 	Blocks       []BlockInfo // max:1000000
 }
 

Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go

@@ -28,7 +28,6 @@ const (
 	messageTypeRequest       = 2
 	messageTypeResponse      = 3
 	messageTypePing          = 4
-	messageTypePong          = 5
 	messageTypeIndexUpdate   = 6
 	messageTypeClose         = 7
 )
@@ -71,13 +70,12 @@ const (
 )
 
 var (
-	ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
-	ErrClosed      = errors.New("connection closed")
+	ErrClosed  = errors.New("connection closed")
+	ErrTimeout = errors.New("read timeout")
 )
 
 // Specific variants of empty messages...
 type pingMessage struct{ EmptyMessage }
-type pongMessage struct{ EmptyMessage }
 
 type Model interface {
 	// An index was received from the peer device
@@ -146,9 +144,11 @@ type isEofer interface {
 	IsEOF() bool
 }
 
-var (
-	PingTimeout  = 30 * time.Second
-	PingIdleTime = 60 * time.Second
+const (
+	// We make sure to send a message at least this often, by triggering pings.
+	PingSendInterval = 90 * time.Second
+	// If we haven't received a message from the other side for this long, close the connection.
+	ReceiveTimeout = 300 * time.Second
 )
 
 func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection {
@@ -180,7 +180,8 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv
 func (c *rawConnection) Start() {
 	go c.readerLoop()
 	go c.writerLoop()
-	go c.pingerLoop()
+	go c.pingSender()
+	go c.pingReceiver()
 	go c.idGenerator()
 }
 
@@ -278,18 +279,7 @@ func (c *rawConnection) ping() bool {
 		return false
 	}
 
-	rc := make(chan asyncResult, 1)
-	c.awaitingMut.Lock()
-	c.awaiting[id] = rc
-	c.awaitingMut.Unlock()
-
-	ok := c.send(id, messageTypePing, nil, nil)
-	if !ok {
-		return false
-	}
-
-	res, ok := <-rc
-	return ok && res.err == nil
+	return c.send(id, messageTypePing, nil, nil)
 }
 
 func (c *rawConnection) readerLoop() (err error) {
@@ -352,13 +342,7 @@ func (c *rawConnection) readerLoop() (err error) {
 			if state != stateReady {
 				return fmt.Errorf("protocol error: ping message in state %d", state)
 			}
-			c.send(hdr.msgID, messageTypePong, pongMessage{}, nil)
-
-		case pongMessage:
-			if state != stateReady {
-				return fmt.Errorf("protocol error: pong message in state %d", state)
-			}
-			c.handlePong(hdr.msgID)
+			// Nothing
 
 		case CloseMessage:
 			return errors.New(msg.Reason)
@@ -467,9 +451,6 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
 	case messageTypePing:
 		msg = pingMessage{}
 
-	case messageTypePong:
-		msg = pongMessage{}
-
 	case messageTypeClusterConfig:
 		var cc ClusterConfigMessage
 		err = cc.UnmarshalXDR(msgBuf)
@@ -729,42 +710,55 @@ func (c *rawConnection) idGenerator() {
 	}
 }
 
-func (c *rawConnection) pingerLoop() {
-	var rc = make(chan bool, 1)
-	ticker := time.Tick(PingIdleTime / 2)
+// The pingSender makes sure that we've sent a message within the last
+// PingSendInterval. If we already have something sent in the last
+// PingSendInterval/2, we do nothing. Otherwise we send a ping message. This
+// results in an effecting ping interval of somewhere between
+// PingSendInterval/2 and PingSendInterval.
+func (c *rawConnection) pingSender() {
+	ticker := time.Tick(PingSendInterval / 2)
+
 	for {
 		select {
 		case <-ticker:
-			if d := time.Since(c.cr.Last()); d < PingIdleTime {
-				if debug {
-					l.Debugln(c.id, "ping skipped after rd", d)
-				}
-				continue
-			}
-			if d := time.Since(c.cw.Last()); d < PingIdleTime {
+			d := time.Since(c.cw.Last())
+			if d < PingSendInterval/2 {
 				if debug {
 					l.Debugln(c.id, "ping skipped after wr", d)
 				}
 				continue
 			}
-			go func() {
+
+			if debug {
+				l.Debugln(c.id, "ping -> after", d)
+			}
+			c.ping()
+
+		case <-c.closed:
+			return
+		}
+	}
+}
+
+// The pingReciever checks that we've received a message (any message will do,
+// but we expect pings in the absence of other messages) within the last
+// ReceiveTimeout. If not, we close the connection with an ErrTimeout.
+func (c *rawConnection) pingReceiver() {
+	ticker := time.Tick(ReceiveTimeout / 2)
+
+	for {
+		select {
+		case <-ticker:
+			d := time.Since(c.cr.Last())
+			if d > ReceiveTimeout {
 				if debug {
-					l.Debugln(c.id, "ping ->")
+					l.Debugln(c.id, "ping timeout", d)
 				}
-				rc <- c.ping()
-			}()
-			select {
-			case ok := <-rc:
-				if debug {
-					l.Debugln(c.id, "<- pong")
-				}
-				if !ok {
-					c.close(fmt.Errorf("ping failure"))
-				}
-			case <-time.After(PingTimeout):
-				c.close(fmt.Errorf("ping timeout"))
-			case <-c.closed:
-				return
+				c.close(ErrTimeout)
+			}
+
+			if debug {
+				l.Debugln(c.id, "last read within", d)
 			}
 
 		case <-c.closed:

Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go

@@ -6,7 +6,6 @@ import (
 	"bytes"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -82,94 +81,6 @@ func TestPing(t *testing.T) {
 	}
 }
 
-func TestPingErr(t *testing.T) {
-	e := errors.New("something broke")
-
-	for i := 0; i < 32; i++ {
-		for j := 0; j < 32; j++ {
-			m0 := newTestModel()
-			m1 := newTestModel()
-
-			ar, aw := io.Pipe()
-			br, bw := io.Pipe()
-			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
-			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
-
-			c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-			c0.Start()
-			c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways)
-			c1.Start()
-			c0.ClusterConfig(ClusterConfigMessage{})
-			c1.ClusterConfig(ClusterConfigMessage{})
-
-			res := c0.ping()
-			if (i < 8 || j < 8) && res {
-				// This should have resulted in failure, as there is no way an empty ClusterConfig plus a Ping message fits in eight bytes.
-				t.Errorf("Unexpected ping success; i=%d, j=%d", i, j)
-			} else if (i >= 28 && j >= 28) && !res {
-				// This should have worked though, as 28 bytes is plenty for both.
-				t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j)
-			}
-		}
-	}
-}
-
-// func TestRequestResponseErr(t *testing.T) {
-// 	e := errors.New("something broke")
-
-// 	var pass bool
-// 	for i := 0; i < 48; i++ {
-// 		for j := 0; j < 38; j++ {
-// 			m0 := newTestModel()
-// 			m0.data = []byte("response data")
-// 			m1 := newTestModel()
-
-// 			ar, aw := io.Pipe()
-// 			br, bw := io.Pipe()
-// 			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
-// 			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
-
-// 			NewConnection(c0ID, ar, ebw, m0, nil)
-// 			c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection)
-
-// 			d, err := c1.Request("default", "tn", 1234, 5678)
-// 			if err == e || err == ErrClosed {
-// 				t.Logf("Error at %d+%d bytes", i, j)
-// 				if !m1.isClosed() {
-// 					t.Fatal("c1 not closed")
-// 				}
-// 				if !m0.isClosed() {
-// 					t.Fatal("c0 not closed")
-// 				}
-// 				continue
-// 			}
-// 			if err != nil {
-// 				t.Fatal(err)
-// 			}
-// 			if string(d) != "response data" {
-// 				t.Fatalf("Incorrect response data %q", string(d))
-// 			}
-// 			if m0.folder != "default" {
-// 				t.Fatalf("Incorrect folder %q", m0.folder)
-// 			}
-// 			if m0.name != "tn" {
-// 				t.Fatalf("Incorrect name %q", m0.name)
-// 			}
-// 			if m0.offset != 1234 {
-// 				t.Fatalf("Incorrect offset %d", m0.offset)
-// 			}
-// 			if m0.size != 5678 {
-// 				t.Fatalf("Incorrect size %d", m0.size)
-// 			}
-// 			t.Logf("Pass at %d+%d bytes", i, j)
-// 			pass = true
-// 		}
-// 	}
-// 	if !pass {
-// 		t.Fatal("Never passed")
-// 	}
-// }
-
 func TestVersionErr(t *testing.T) {
 	m0 := newTestModel()
 	m1 := newTestModel()
@@ -278,11 +189,12 @@ func TestMarshalIndexMessage(t *testing.T) {
 	}
 
 	f := func(m1 IndexMessage) bool {
-		for _, f := range m1.Files {
-			for i := range f.Blocks {
-				f.Blocks[i].Offset = 0
-				if len(f.Blocks[i].Hash) == 0 {
-					f.Blocks[i].Hash = nil
+		for i, f := range m1.Files {
+			m1.Files[i].CachedSize = 0
+			for j := range f.Blocks {
+				f.Blocks[j].Offset = 0
+				if len(f.Blocks[j].Hash) == 0 {
+					f.Blocks[j].Hash = nil
 				}
 			}
 		}

Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go

@@ -0,0 +1,298 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package client
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log"
+	"net"
+	"net/url"
+	"time"
+
+	syncthingprotocol "github.com/syncthing/protocol"
+	"github.com/syncthing/relaysrv/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type ProtocolClient struct {
+	URI         *url.URL
+	Invitations chan protocol.SessionInvitation
+
+	closeInvitationsOnFinish bool
+
+	config *tls.Config
+
+	timeout time.Duration
+
+	stop    chan struct{}
+	stopped chan struct{}
+
+	conn *tls.Conn
+
+	mut       sync.RWMutex
+	connected bool
+	latency   time.Duration
+}
+
+func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient {
+	closeInvitationsOnFinish := false
+	if invitations == nil {
+		closeInvitationsOnFinish = true
+		invitations = make(chan protocol.SessionInvitation)
+	}
+
+	return &ProtocolClient{
+		URI:         uri,
+		Invitations: invitations,
+
+		closeInvitationsOnFinish: closeInvitationsOnFinish,
+
+		config: configForCerts(certs),
+
+		timeout: time.Minute * 2,
+
+		stop:    make(chan struct{}),
+		stopped: make(chan struct{}),
+
+		mut:       sync.NewRWMutex(),
+		connected: false,
+	}
+}
+
+func (c *ProtocolClient) Serve() {
+	c.stop = make(chan struct{})
+	c.stopped = make(chan struct{})
+	defer close(c.stopped)
+
+	if err := c.connect(); err != nil {
+		if debug {
+			l.Debugln("Relay connect:", err)
+		}
+		return
+	}
+
+	if debug {
+		l.Debugln(c, "connected", c.conn.RemoteAddr())
+	}
+
+	if err := c.join(); err != nil {
+		c.conn.Close()
+		l.Infoln("Relay join:", err)
+		return
+	}
+
+	if err := c.conn.SetDeadline(time.Time{}); err != nil {
+		l.Infoln("Relay set deadline:", err)
+		return
+	}
+
+	if debug {
+		l.Debugln(c, "joined", c.conn.RemoteAddr(), "via", c.conn.LocalAddr())
+	}
+
+	defer c.cleanup()
+	c.mut.Lock()
+	c.connected = true
+	c.mut.Unlock()
+
+	messages := make(chan interface{})
+	errors := make(chan error, 1)
+
+	go messageReader(c.conn, messages, errors)
+
+	timeout := time.NewTimer(c.timeout)
+
+	for {
+		select {
+		case message := <-messages:
+			timeout.Reset(c.timeout)
+			if debug {
+				log.Printf("%s received message %T", c, message)
+			}
+
+			switch msg := message.(type) {
+			case protocol.Ping:
+				if err := protocol.WriteMessage(c.conn, protocol.Pong{}); err != nil {
+					l.Infoln("Relay write:", err)
+					return
+
+				}
+				if debug {
+					l.Debugln(c, "sent pong")
+				}
+
+			case protocol.SessionInvitation:
+				ip := net.IP(msg.Address)
+				if len(ip) == 0 || ip.IsUnspecified() {
+					msg.Address = c.conn.RemoteAddr().(*net.TCPAddr).IP[:]
+				}
+				c.Invitations <- msg
+
+			default:
+				l.Infoln("Relay: protocol error: unexpected message %v", msg)
+				return
+			}
+
+		case <-c.stop:
+			if debug {
+				l.Debugln(c, "stopping")
+			}
+			return
+
+		case err := <-errors:
+			l.Infoln("Relay received:", err)
+			return
+
+		case <-timeout.C:
+			if debug {
+				l.Debugln(c, "timed out")
+			}
+			return
+		}
+	}
+}
+
+func (c *ProtocolClient) Stop() {
+	if c.stop == nil {
+		return
+	}
+
+	close(c.stop)
+	<-c.stopped
+}
+
+func (c *ProtocolClient) StatusOK() bool {
+	c.mut.RLock()
+	con := c.connected
+	c.mut.RUnlock()
+	return con
+}
+
+func (c *ProtocolClient) Latency() time.Duration {
+	c.mut.RLock()
+	lat := c.latency
+	c.mut.RUnlock()
+	return lat
+}
+
+func (c *ProtocolClient) String() string {
+	return fmt.Sprintf("ProtocolClient@%p", c)
+}
+
+func (c *ProtocolClient) connect() error {
+	if c.URI.Scheme != "relay" {
+		return fmt.Errorf("Unsupported relay schema:", c.URI.Scheme)
+	}
+
+	t0 := time.Now()
+	tcpConn, err := net.Dial("tcp", c.URI.Host)
+	if err != nil {
+		return err
+	}
+
+	c.mut.Lock()
+	c.latency = time.Since(t0)
+	c.mut.Unlock()
+
+	conn := tls.Client(tcpConn, c.config)
+	if err = conn.Handshake(); err != nil {
+		return err
+	}
+
+	if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
+		conn.Close()
+		return err
+	}
+
+	if err := performHandshakeAndValidation(conn, c.URI); err != nil {
+		conn.Close()
+		return err
+	}
+
+	c.conn = conn
+	return nil
+}
+
+func (c *ProtocolClient) cleanup() {
+	if c.closeInvitationsOnFinish {
+		close(c.Invitations)
+		c.Invitations = make(chan protocol.SessionInvitation)
+	}
+
+	if debug {
+		l.Debugln(c, "cleaning up")
+	}
+
+	c.mut.Lock()
+	c.connected = false
+	c.mut.Unlock()
+
+	c.conn.Close()
+}
+
+func (c *ProtocolClient) join() error {
+	if err := protocol.WriteMessage(c.conn, protocol.JoinRelayRequest{}); err != nil {
+		return err
+	}
+
+	message, err := protocol.ReadMessage(c.conn)
+	if err != nil {
+		return err
+	}
+
+	switch msg := message.(type) {
+	case protocol.Response:
+		if msg.Code != 0 {
+			return fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message)
+		}
+
+	default:
+		return fmt.Errorf("protocol error: expecting response got %v", msg)
+	}
+
+	return nil
+}
+
+func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error {
+	if err := conn.Handshake(); err != nil {
+		return err
+	}
+
+	cs := conn.ConnectionState()
+	if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != protocol.ProtocolName {
+		return fmt.Errorf("protocol negotiation error")
+	}
+
+	q := uri.Query()
+	relayIDs := q.Get("id")
+	if relayIDs != "" {
+		relayID, err := syncthingprotocol.DeviceIDFromString(relayIDs)
+		if err != nil {
+			return fmt.Errorf("relay address contains invalid verification id: %s", err)
+		}
+
+		certs := cs.PeerCertificates
+		if cl := len(certs); cl != 1 {
+			return fmt.Errorf("unexpected certificate count: %d", cl)
+		}
+
+		remoteID := syncthingprotocol.NewDeviceID(certs[0].Raw)
+		if remoteID != relayID {
+			return fmt.Errorf("relay id does not match. Expected %v got %v", relayID, remoteID)
+		}
+	}
+
+	return nil
+}
+
+func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
+	for {
+		msg, err := protocol.ReadMessage(conn)
+		if err != nil {
+			errors <- err
+			return
+		}
+		messages <- msg
+	}
+}

Godeps/_workspace/src/github.com/syncthing/relaysrv/client/debug.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package client
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "relay") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go

@@ -0,0 +1,141 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package client
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	syncthingprotocol "github.com/syncthing/protocol"
+	"github.com/syncthing/relaysrv/protocol"
+)
+
+func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs []tls.Certificate) (protocol.SessionInvitation, error) {
+	if uri.Scheme != "relay" {
+		return protocol.SessionInvitation{}, fmt.Errorf("Unsupported relay scheme:", uri.Scheme)
+	}
+
+	conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs))
+	if err != nil {
+		return protocol.SessionInvitation{}, err
+	}
+	conn.SetDeadline(time.Now().Add(10 * time.Second))
+
+	if err := performHandshakeAndValidation(conn, uri); err != nil {
+		return protocol.SessionInvitation{}, err
+	}
+
+	defer conn.Close()
+
+	request := protocol.ConnectRequest{
+		ID: id[:],
+	}
+
+	if err := protocol.WriteMessage(conn, request); err != nil {
+		return protocol.SessionInvitation{}, err
+	}
+
+	message, err := protocol.ReadMessage(conn)
+	if err != nil {
+		return protocol.SessionInvitation{}, err
+	}
+
+	switch msg := message.(type) {
+	case protocol.Response:
+		return protocol.SessionInvitation{}, fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message)
+	case protocol.SessionInvitation:
+		if debug {
+			l.Debugln("Received invitation", msg, "via", conn.LocalAddr())
+		}
+		ip := net.IP(msg.Address)
+		if len(ip) == 0 || ip.IsUnspecified() {
+			msg.Address = conn.RemoteAddr().(*net.TCPAddr).IP[:]
+		}
+		return msg, nil
+	default:
+		return protocol.SessionInvitation{}, fmt.Errorf("protocol error: unexpected message %v", msg)
+	}
+}
+
+func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) {
+	addr := net.JoinHostPort(net.IP(invitation.Address).String(), strconv.Itoa(int(invitation.Port)))
+
+	conn, err := net.Dial("tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+
+	request := protocol.JoinSessionRequest{
+		Key: invitation.Key,
+	}
+
+	conn.SetDeadline(time.Now().Add(10 * time.Second))
+	err = protocol.WriteMessage(conn, request)
+	if err != nil {
+		return nil, err
+	}
+
+	message, err := protocol.ReadMessage(conn)
+	if err != nil {
+		return nil, err
+	}
+
+	conn.SetDeadline(time.Time{})
+
+	switch msg := message.(type) {
+	case protocol.Response:
+		if msg.Code != 0 {
+			return nil, fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message)
+		}
+		return conn, nil
+	default:
+		return nil, fmt.Errorf("protocol error: expecting response got %v", msg)
+	}
+}
+
+func TestRelay(uri *url.URL, certs []tls.Certificate, sleep time.Duration, times int) bool {
+	id := syncthingprotocol.NewDeviceID(certs[0].Certificate[0])
+	invs := make(chan protocol.SessionInvitation, 1)
+	c := NewProtocolClient(uri, certs, invs)
+	go c.Serve()
+	defer func() {
+		close(invs)
+		c.Stop()
+	}()
+
+	for i := 0; i < times; i++ {
+		_, err := GetInvitationFromRelay(uri, id, certs)
+		if err == nil {
+			return true
+		}
+		if !strings.Contains(err.Error(), "Incorrect response code") {
+			return false
+		}
+		time.Sleep(sleep)
+	}
+	return false
+}
+
+func configForCerts(certs []tls.Certificate) *tls.Config {
+	return &tls.Config{
+		Certificates:           certs,
+		NextProtos:             []string{protocol.ProtocolName},
+		ClientAuth:             tls.RequestClientCert,
+		SessionTicketsDisabled: true,
+		InsecureSkipVerify:     true,
+		MinVersion:             tls.VersionTLS12,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+		},
+	}
+}

Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go

@@ -0,0 +1,66 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+//go:generate -command genxdr go run ../../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
+//go:generate genxdr -o packets_xdr.go packets.go
+
+package protocol
+
+import (
+	"fmt"
+	"net"
+
+	syncthingprotocol "github.com/syncthing/protocol"
+)
+
+const (
+	messageTypePing int32 = iota
+	messageTypePong
+	messageTypeJoinRelayRequest
+	messageTypeJoinSessionRequest
+	messageTypeResponse
+	messageTypeConnectRequest
+	messageTypeSessionInvitation
+)
+
+type header struct {
+	magic         uint32
+	messageType   int32
+	messageLength int32
+}
+
+type Ping struct{}
+type Pong struct{}
+type JoinRelayRequest struct{}
+
+type JoinSessionRequest struct {
+	Key []byte // max:32
+}
+
+type Response struct {
+	Code    int32
+	Message string
+}
+
+type ConnectRequest struct {
+	ID []byte // max:32
+}
+
+type SessionInvitation struct {
+	From         []byte // max:32
+	Key          []byte // max:32
+	Address      []byte // max:32
+	Port         uint16
+	ServerSocket bool
+}
+
+func (i SessionInvitation) String() string {
+	return fmt.Sprintf("%s@%s", syncthingprotocol.DeviceIDFromBytes(i.From), i.AddressString())
+}
+
+func (i SessionInvitation) GoString() string {
+	return i.String()
+}
+
+func (i SessionInvitation) AddressString() string {
+	return fmt.Sprintf("%s:%d", net.IP(i.Address), i.Port)
+}

Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets_xdr.go

@@ -0,0 +1,567 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package protocol
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+header Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             magic                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         message Type                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        message Length                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct header {
+	unsigned int magic;
+	int messageType;
+	int messageLength;
+}
+
+*/
+
+func (o header) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o header) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o header) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o header) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o header) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(o.magic)
+	xw.WriteUint32(uint32(o.messageType))
+	xw.WriteUint32(uint32(o.messageLength))
+	return xw.Tot(), xw.Error()
+}
+
+func (o *header) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *header) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *header) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.magic = xr.ReadUint32()
+	o.messageType = int32(xr.ReadUint32())
+	o.messageLength = int32(xr.ReadUint32())
+	return xr.Error()
+}
+
+/*
+
+Ping Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Ping {
+}
+
+*/
+
+func (o Ping) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Ping) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Ping) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Ping) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Ping) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Ping) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Ping) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Ping) DecodeXDRFrom(xr *xdr.Reader) error {
+	return xr.Error()
+}
+
+/*
+
+Pong Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Pong {
+}
+
+*/
+
+func (o Pong) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Pong) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Pong) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Pong) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Pong) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Pong) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Pong) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Pong) DecodeXDRFrom(xr *xdr.Reader) error {
+	return xr.Error()
+}
+
+/*
+
+JoinRelayRequest Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct JoinRelayRequest {
+}
+
+*/
+
+func (o JoinRelayRequest) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o JoinRelayRequest) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o JoinRelayRequest) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o JoinRelayRequest) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o JoinRelayRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	return xw.Tot(), xw.Error()
+}
+
+func (o *JoinRelayRequest) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *JoinRelayRequest) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *JoinRelayRequest) DecodeXDRFrom(xr *xdr.Reader) error {
+	return xr.Error()
+}
+
+/*
+
+JoinSessionRequest Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of Key                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     Key (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct JoinSessionRequest {
+	opaque Key<32>;
+}
+
+*/
+
+func (o JoinSessionRequest) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o JoinSessionRequest) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o JoinSessionRequest) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o JoinSessionRequest) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o JoinSessionRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	if l := len(o.Key); l > 32 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 32)
+	}
+	xw.WriteBytes(o.Key)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *JoinSessionRequest) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *JoinSessionRequest) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *JoinSessionRequest) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.Key = xr.ReadBytesMax(32)
+	return xr.Error()
+}
+
+/*
+
+Response Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Code                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Length of Message                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                   Message (variable length)                   \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Response {
+	int Code;
+	string Message<>;
+}
+
+*/
+
+func (o Response) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Response) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Response) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Response) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Response) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(uint32(o.Code))
+	xw.WriteString(o.Message)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Response) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Response) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Response) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.Code = int32(xr.ReadUint32())
+	o.Message = xr.ReadString()
+	return xr.Error()
+}
+
+/*
+
+ConnectRequest Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of ID                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     ID (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct ConnectRequest {
+	opaque ID<32>;
+}
+
+*/
+
+func (o ConnectRequest) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o ConnectRequest) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o ConnectRequest) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o ConnectRequest) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o ConnectRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	if l := len(o.ID); l > 32 {
+		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32)
+	}
+	xw.WriteBytes(o.ID)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *ConnectRequest) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *ConnectRequest) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *ConnectRequest) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.ID = xr.ReadBytesMax(32)
+	return xr.Error()
+}
+
+/*
+
+SessionInvitation Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of From                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    From (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of Key                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     Key (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Length of Address                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                   Address (variable length)                   \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |             Port              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                  Server Socket (V=0 or 1)                   |V|
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct SessionInvitation {
+	opaque From<32>;
+	opaque Key<32>;
+	opaque Address<32>;
+	unsigned int Port;
+	bool ServerSocket;
+}
+
+*/
+
+func (o SessionInvitation) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o SessionInvitation) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o SessionInvitation) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o SessionInvitation) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o SessionInvitation) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	if l := len(o.From); l > 32 {
+		return xw.Tot(), xdr.ElementSizeExceeded("From", l, 32)
+	}
+	xw.WriteBytes(o.From)
+	if l := len(o.Key); l > 32 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 32)
+	}
+	xw.WriteBytes(o.Key)
+	if l := len(o.Address); l > 32 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Address", l, 32)
+	}
+	xw.WriteBytes(o.Address)
+	xw.WriteUint16(o.Port)
+	xw.WriteBool(o.ServerSocket)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *SessionInvitation) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *SessionInvitation) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *SessionInvitation) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.From = xr.ReadBytesMax(32)
+	o.Key = xr.ReadBytesMax(32)
+	o.Address = xr.ReadBytesMax(32)
+	o.Port = xr.ReadUint16()
+	o.ServerSocket = xr.ReadBool()
+	return xr.Error()
+}

Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/protocol.go

@@ -0,0 +1,114 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package protocol
+
+import (
+	"fmt"
+	"io"
+)
+
+const (
+	magic        = 0x9E79BC40
+	ProtocolName = "bep-relay"
+)
+
+var (
+	ResponseSuccess           = Response{0, "success"}
+	ResponseNotFound          = Response{1, "not found"}
+	ResponseAlreadyConnected  = Response{2, "already connected"}
+	ResponseInternalError     = Response{99, "internal error"}
+	ResponseUnexpectedMessage = Response{100, "unexpected message"}
+)
+
+func WriteMessage(w io.Writer, message interface{}) error {
+	header := header{
+		magic: magic,
+	}
+
+	var payload []byte
+	var err error
+
+	switch msg := message.(type) {
+	case Ping:
+		payload, err = msg.MarshalXDR()
+		header.messageType = messageTypePing
+	case Pong:
+		payload, err = msg.MarshalXDR()
+		header.messageType = messageTypePong
+	case JoinRelayRequest:
+		payload, err = msg.MarshalXDR()
+		header.messageType = messageTypeJoinRelayRequest
+	case JoinSessionRequest:
+		payload, err = msg.MarshalXDR()
+		header.messageType = messageTypeJoinSessionRequest
+	case Response:
+		payload, err = msg.MarshalXDR()
+		header.messageType = messageTypeResponse
+	case ConnectRequest:
+		payload, err = msg.MarshalXDR()
+		header.messageType = messageTypeConnectRequest
+	case SessionInvitation:
+		payload, err = msg.MarshalXDR()
+		header.messageType = messageTypeSessionInvitation
+	default:
+		err = fmt.Errorf("Unknown message type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	header.messageLength = int32(len(payload))
+
+	headerpayload, err := header.MarshalXDR()
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(append(headerpayload, payload...))
+	return err
+}
+
+func ReadMessage(r io.Reader) (interface{}, error) {
+	var header header
+	if err := header.DecodeXDR(r); err != nil {
+		return nil, err
+	}
+
+	if header.magic != magic {
+		return nil, fmt.Errorf("magic mismatch")
+	}
+
+	switch header.messageType {
+	case messageTypePing:
+		var msg Ping
+		err := msg.DecodeXDR(r)
+		return msg, err
+	case messageTypePong:
+		var msg Pong
+		err := msg.DecodeXDR(r)
+		return msg, err
+	case messageTypeJoinRelayRequest:
+		var msg JoinRelayRequest
+		err := msg.DecodeXDR(r)
+		return msg, err
+	case messageTypeJoinSessionRequest:
+		var msg JoinSessionRequest
+		err := msg.DecodeXDR(r)
+		return msg, err
+	case messageTypeResponse:
+		var msg Response
+		err := msg.DecodeXDR(r)
+		return msg, err
+	case messageTypeConnectRequest:
+		var msg ConnectRequest
+		err := msg.DecodeXDR(r)
+		return msg, err
+	case messageTypeSessionInvitation:
+		var msg SessionInvitation
+		err := msg.DecodeXDR(r)
+		return msg, err
+	}
+
+	return nil, fmt.Errorf("Unknown message type")
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go

@@ -201,6 +201,7 @@ func (p *BufferPool) String() string {
 
 func (p *BufferPool) drain() {
 	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
 	for {
 		select {
 		case <-ticker.C:

Godeps/_workspace/src/github.com/thejerf/suture/README.md

@@ -6,10 +6,8 @@ Suture
 Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" ->
 "sutree" -> "suture" -> holds your code together when it's trying to die.
 
-This is intended to be a production-quality library going into code that I
-will be very early on the phone tree to support when it goes down. However,
-it has not been deployed into something quite that serious yet. (I will
-update this statement when that changes.)
+This library has hit maturity, and isn't expected to be changed
+radically. This can also be imported via gopkg.in/thejerf/suture.v1 .
 
 It is intended to deal gracefully with the real failure cases that can
 occur with supervision trees (such as burning all your CPU time endlessly
@@ -24,10 +22,6 @@ This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/su
 including an example, usage, and everything else you might expect from a
 README.md on GitHub. (DRY.)
 
-This is not currently tagged with particular git tags for Go as this is
-currently considered to be alpha code. As I move this into production and
-feel more confident about it, I'll give it relevant tags.
-
 Code Signing
 ------------
 

NICKS

@@ -52,7 +52,7 @@ pluby		<phill.luby@newredo.com>
 pyfisch		<pyfisch@gmail.com>
 qbit		<qbit@deftly.net>
 ralder		<ralder@yandex.ru>
-rumpelsepp	<stefan@sevenbyte.org>
+rumpelsepp	<stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
 sciurius	<jvromans@squirrel.nl>
 seehuhn		<voss@seehuhn.de>
 snnd		<dw@risu.io>

README.md

@@ -1,9 +1,10 @@
 Syncthing
 =========
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
-[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
-[![MPLv2 License](http://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
+[![Latest Build (Official)](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
+[![AppVeyor Build](https://img.shields.io/appveyor/ci/calmh/syncthing/master.svg?style=flat-square)](https://ci.appveyor.com/project/calmh/syncthing)
+[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
+[![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 
 This is the Syncthing project which pursues the following goals:
 

appveyor.yaml

@@ -0,0 +1,12 @@
+version: '{branch}-{build}'
+clone_folder: C:\src\github.com\syncthing\syncthing
+init:
+- go version
+environment:
+  GOPATH: C:\
+build_script:
+- go run build.go zip
+test_script:
+- go run build.go test
+artifacts:
+- path: '*.zip'

build.go

@@ -13,7 +13,6 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
-	"crypto/md5"
 	"flag"
 	"fmt"
 	"io"
@@ -215,7 +214,7 @@ func build(pkg string, tags []string) {
 		binary += ".exe"
 	}
 
-	rmr(binary, binary+".md5")
+	rmr(binary)
 	args := []string{"build", "-ldflags", ldflags()}
 	if len(tags) > 0 {
 		args = append(args, "-tags", strings.Join(tags, ","))
@@ -226,13 +225,6 @@ func build(pkg string, tags []string) {
 	args = append(args, pkg)
 	setBuildEnv()
 	runPrint("go", args...)
-
-	// Create an md5 checksum of the binary, to be included in the archive for
-	// automatic upgrades.
-	err := md5File(binary)
-	if err != nil {
-		log.Fatal(err)
-	}
 }
 
 func buildTar() {
@@ -249,7 +241,6 @@ func buildTar() {
 		{src: "LICENSE", dst: name + "/LICENSE.txt"},
 		{src: "AUTHORS", dst: name + "/AUTHORS.txt"},
 		{src: "syncthing", dst: name + "/syncthing"},
-		{src: "syncthing.md5", dst: name + "/syncthing.md5"},
 	}
 
 	for _, file := range listFiles("etc") {
@@ -277,7 +268,6 @@ func buildZip() {
 		{src: "LICENSE", dst: name + "/LICENSE.txt"},
 		{src: "AUTHORS", dst: name + "/AUTHORS.txt"},
 		{src: "syncthing.exe", dst: name + "/syncthing.exe"},
-		{src: "syncthing.exe.md5", dst: name + "/syncthing.exe.md5"},
 	}
 
 	for _, file := range listFiles("extra") {
@@ -712,32 +702,6 @@ func zipFile(out string, files []archiveFile) {
 	}
 }
 
-func md5File(file string) error {
-	fd, err := os.Open(file)
-	if err != nil {
-		return err
-	}
-	defer fd.Close()
-
-	h := md5.New()
-	_, err = io.Copy(h, fd)
-	if err != nil {
-		return err
-	}
-
-	out, err := os.Create(file + ".md5")
-	if err != nil {
-		return err
-	}
-
-	_, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
-	if err != nil {
-		return err
-	}
-
-	return out.Close()
-}
-
 func vet(pkg string) {
 	bs, err := runError("go", "vet", pkg)
 	if err != nil && err.Error() == "exit status 3" || bytes.Contains(bs, []byte("no such tool \"vet\"")) {

cmd/stfileinfo/main.go

@@ -68,7 +68,7 @@ func main() {
 		if *standardBlocks || blockSize < protocol.BlockSize {
 			blockSize = protocol.BlockSize
 		}
-		bs, err := scanner.Blocks(fd, blockSize, fi.Size())
+		bs, err := scanner.Blocks(fd, blockSize, fi.Size(), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -7,37 +7,105 @@
 package main
 
 import (
+	"crypto/tls"
+	"errors"
 	"flag"
-	"log"
+	"fmt"
+	"net/url"
 	"os"
+	"time"
 
 	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
 )
 
-func main() {
-	log.SetFlags(0)
-	log.SetOutput(os.Stdout)
+var timeout = 5 * time.Second
 
+func main() {
 	var server string
 
-	flag.StringVar(&server, "server", "udp4://announce.syncthing.net:22026", "Announce server")
+	flag.StringVar(&server, "server", "", "Announce server (blank for default set)")
+	flag.DurationVar(&timeout, "timeout", timeout, "Query timeout")
+	flag.Usage = usage
 	flag.Parse()
 
-	if len(flag.Args()) != 1 || server == "" {
-		log.Printf("Usage: %s [-server=\"udp4://announce.syncthing.net:22026\"] <device>", os.Args[0])
+	if flag.NArg() != 1 {
+		flag.Usage()
 		os.Exit(64)
 	}
 
 	id, err := protocol.DeviceIDFromString(flag.Args()[0])
 	if err != nil {
-		log.Println(err)
+		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	discoverer := discover.NewDiscoverer(protocol.LocalDeviceID, nil)
-	discoverer.StartGlobal([]string{server}, 1)
-	for _, addr := range discoverer.Lookup(id) {
-		log.Println(addr)
+	if server != "" {
+		checkServers(id, server)
+	} else {
+		checkServers(id, config.DefaultDiscoveryServers...)
 	}
 }
+
+type checkResult struct {
+	server string
+	direct []string
+	relays []discover.Relay
+	error
+}
+
+func checkServers(deviceID protocol.DeviceID, servers ...string) {
+	t0 := time.Now()
+	resc := make(chan checkResult)
+	for _, srv := range servers {
+		srv := srv
+		go func() {
+			res := checkServer(deviceID, srv)
+			res.server = srv
+			resc <- res
+		}()
+	}
+
+	for _ = range servers {
+		res := <-resc
+
+		u, _ := url.Parse(res.server)
+		fmt.Printf("%s (%v):\n", u.Host, time.Since(t0))
+
+		if res.error != nil {
+			fmt.Println("  " + res.error.Error())
+		}
+		for _, addr := range res.direct {
+			fmt.Println("  address:", addr)
+		}
+		for _, rel := range res.relays {
+			fmt.Printf("  relay: %s (%d ms)\n", rel.URL, rel.Latency)
+		}
+	}
+}
+
+func checkServer(deviceID protocol.DeviceID, server string) checkResult {
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, nil)
+	if err != nil {
+		return checkResult{error: err}
+	}
+
+	res := make(chan checkResult, 1)
+
+	time.AfterFunc(timeout, func() {
+		res <- checkResult{error: errors.New("timeout")}
+	})
+
+	go func() {
+		direct, relays, err := disco.Lookup(deviceID)
+		res <- checkResult{direct: direct, relays: relays, error: err}
+	}()
+
+	return <-res
+}
+
+func usage() {
+	fmt.Printf("Usage:\n\t%s [options] <device ID>\n\nOptions:\n", os.Args[0])
+	flag.PrintDefaults()
+}

cmd/stsigtool/main.go

@@ -0,0 +1,115 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/signature"
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
+
+	flag.Parse()
+
+	if flag.NArg() < 1 {
+		log.Println(`Usage:
+	stsigtool <command>
+
+Where command is one of:
+
+	gen
+		- generate a new key pair
+
+	sign <privkeyfile> <datafile>
+		- sign a file
+
+	verify <signaturefile> <datafile>
+		- verify a signature, using the built in public key
+
+	verify <signaturefile> <datafile> <pubkeyfile>
+		- verify a signature, using the specified public key file
+`)
+	}
+
+	switch flag.Arg(0) {
+	case "gen":
+		gen()
+	case "sign":
+		sign(flag.Arg(1), flag.Arg(2))
+	case "verify":
+		if flag.NArg() == 4 {
+			verifyWithFile(flag.Arg(1), flag.Arg(2), flag.Arg(3))
+		} else {
+			verifyWithKey(flag.Arg(1), flag.Arg(2), upgrade.SigningKey)
+		}
+	}
+}
+
+func gen() {
+	priv, pub, err := signature.GenerateKeys()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Stdout.Write(priv)
+	os.Stdout.Write(pub)
+}
+
+func sign(keyname, dataname string) {
+	privkey, err := ioutil.ReadFile(keyname)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fd, err := os.Open(dataname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fd.Close()
+
+	sig, err := signature.Sign(privkey, fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Stdout.Write(sig)
+}
+
+func verifyWithFile(signame, dataname, keyname string) {
+	pubkey, err := ioutil.ReadFile(keyname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	verifyWithKey(signame, dataname, pubkey)
+}
+
+func verifyWithKey(signame, dataname string, pubkey []byte) {
+	sig, err := ioutil.ReadFile(signame)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fd, err := os.Open(dataname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fd.Close()
+
+	err = signature.Verify(pubkey, sig, fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("correct signature")
+}

cmd/syncthing/addresslister.go

@@ -0,0 +1,119 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+type addressLister struct {
+	upnpSvc *upnpSvc
+	cfg     *config.Wrapper
+}
+
+func newAddressLister(upnpSvc *upnpSvc, cfg *config.Wrapper) *addressLister {
+	return &addressLister{
+		upnpSvc: upnpSvc,
+		cfg:     cfg,
+	}
+}
+
+// ExternalAddresses returns a list of addresses that are our best guess for
+// where we are reachable from the outside. As a special case, we may return
+// one or more addresses with an empty IP address (0.0.0.0 or ::) and just
+// port number - this means that the outside address of a NAT gateway should
+// be substituted.
+func (e *addressLister) ExternalAddresses() []string {
+	return e.addresses(false)
+}
+
+// AllAddresses returns a list of addresses that are our best guess for where
+// we are reachable from the local network. Same conditions as
+// ExternalAddresses, but private IPv4 addresses are included.
+func (e *addressLister) AllAddresses() []string {
+	return e.addresses(true)
+}
+
+func (e *addressLister) addresses(includePrivateIPV4 bool) []string {
+	var addrs []string
+
+	// Grab our listen addresses from the config. Unspecified ones are passed
+	// on verbatim (to be interpreted by a global discovery server or local
+	// discovery peer). Public addresses are passed on verbatim. Private
+	// addresses are filtered.
+	for _, addrStr := range e.cfg.Options().ListenAddress {
+		addrURL, err := url.Parse(addrStr)
+		if err != nil {
+			l.Infoln("Listen address", addrStr, "is invalid:", err)
+			continue
+		}
+		addr, err := net.ResolveTCPAddr("tcp", addrURL.Host)
+		if err != nil {
+			l.Infoln("Listen address", addrStr, "is invalid:", err)
+			continue
+		}
+
+		if addr.IP == nil || addr.IP.IsUnspecified() {
+			// Address like 0.0.0.0:22000 or [::]:22000 or :22000; include as is.
+			addrs = append(addrs, "tcp://"+addr.String())
+		} else if isPublicIPv4(addr.IP) || isPublicIPv6(addr.IP) {
+			// A public address; include as is.
+			addrs = append(addrs, "tcp://"+addr.String())
+		} else if includePrivateIPV4 && addr.IP.To4().IsGlobalUnicast() {
+			// A private IPv4 address.
+			addrs = append(addrs, "tcp://"+addr.String())
+		}
+	}
+
+	// Get an external port mapping from the upnpSvc, if it has one. If so,
+	// add it as another unspecified address.
+	if e.upnpSvc != nil {
+		if port := e.upnpSvc.ExternalPort(); port != 0 {
+			addrs = append(addrs, fmt.Sprintf("tcp://:%d", port))
+		}
+	}
+
+	return addrs
+}
+
+func isPublicIPv4(ip net.IP) bool {
+	ip = ip.To4()
+	if ip == nil {
+		// Not an IPv4 address (IPv6)
+		return false
+	}
+
+	// IsGlobalUnicast below only checks that it's not link local or
+	// multicast, and we want to exclude private (NAT:ed) addresses as well.
+	rfc1918 := []net.IPNet{
+		{IP: net.IP{10, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{172, 16, 0, 0}, Mask: net.IPMask{255, 240, 0, 0}},
+		{IP: net.IP{192, 168, 0, 0}, Mask: net.IPMask{255, 255, 0, 0}},
+	}
+	for _, n := range rfc1918 {
+		if n.Contains(ip) {
+			return false
+		}
+	}
+
+	return ip.IsGlobalUnicast()
+}
+
+func isPublicIPv6(ip net.IP) bool {
+	if ip.To4() != nil {
+		// Not an IPv6 address (IPv4)
+		// (To16() returns a v6 mapped v4 address so can't be used to check
+		// that it's an actual v6 address)
+		return false
+	}
+
+	return ip.IsGlobalUnicast()
+}

cmd/syncthing/connections.go

@@ -11,66 +11,72 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"strings"
+	"net/url"
+	"sync"
 	"time"
 
 	"github.com/syncthing/protocol"
+	"github.com/syncthing/relaysrv/client"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/relay"
+
 	"github.com/thejerf/suture"
 )
 
+type DialerFactory func(*url.URL, *tls.Config) (*tls.Conn, error)
+type ListenerFactory func(*url.URL, *tls.Config, chan<- model.IntermediateConnection)
+
+var (
+	dialers   = make(map[string]DialerFactory, 0)
+	listeners = make(map[string]ListenerFactory, 0)
+)
+
 // The connection service listens on TLS and dials configured unconnected
 // devices. Successful connections are handed to the model.
 type connectionSvc struct {
 	*suture.Supervisor
-	cfg    *config.Wrapper
-	myID   protocol.DeviceID
-	model  *model.Model
-	tlsCfg *tls.Config
-	conns  chan *tls.Conn
+	cfg        *config.Wrapper
+	myID       protocol.DeviceID
+	model      *model.Model
+	tlsCfg     *tls.Config
+	discoverer discover.Finder
+	conns      chan model.IntermediateConnection
+	relaySvc   *relay.Svc
+
+	lastRelayCheck map[protocol.DeviceID]time.Time
+
+	mut           sync.RWMutex
+	connType      map[protocol.DeviceID]model.ConnectionType
+	relaysEnabled bool
 }
 
-func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, model *model.Model, tlsCfg *tls.Config) *connectionSvc {
+func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, mdl *model.Model, tlsCfg *tls.Config, discoverer discover.Finder, relaySvc *relay.Svc) *connectionSvc {
 	svc := &connectionSvc{
 		Supervisor: suture.NewSimple("connectionSvc"),
 		cfg:        cfg,
 		myID:       myID,
-		model:      model,
+		model:      mdl,
 		tlsCfg:     tlsCfg,
-		conns:      make(chan *tls.Conn),
+		discoverer: discoverer,
+		relaySvc:   relaySvc,
+		conns:      make(chan model.IntermediateConnection),
+
+		connType:       make(map[protocol.DeviceID]model.ConnectionType),
+		relaysEnabled:  cfg.Options().RelaysEnabled,
+		lastRelayCheck: make(map[protocol.DeviceID]time.Time),
 	}
+	cfg.Subscribe(svc)
 
 	// There are several moving parts here; one routine per listening address
 	// to handle incoming connections, one routine to periodically attempt
-	// outgoing connections, and lastly one routine to the the common handling
-	// regardless of whether the connection was incoming or outgoing. It ends
-	// up as in the diagram below. We embed a Supervisor to manage the
-	// routines (i.e. log and restart if they crash or exit, etc).
-	//
-	//                +-----------------+
-	//    Incoming    | +---------------+-+      +-----------------+
-	//   Connections  | |                 |      |                 |   Outgoing
-	// -------------->| |   svc.listen    |      |                 |  Connections
-	//                | |  (1 per listen  |      |   svc.connect   |-------------->
-	//                | |    address)     |      |                 |
-	//                +-+                 |      |                 |
-	//                  +-----------------+      +-----------------+
-	//                           v                        v
-	//                           |                        |
-	//                           |                        |
-	//                           +------------+-----------+
-	//                                        |
-	//                                        | svc.conns
-	//                                        v
-	//                               +-----------------+
-	//                               |                 |
-	//                               |                 |
-	//                               |   svc.handle    |------> model.AddConnection()
-	//                               |                 |
-	//                               |                 |
-	//                               +-----------------+
+	// outgoing connections, one routine to the the common handling
+	// regardless of whether the connection was incoming or outgoing.
+	// Furthermore, a relay service which handles incoming requests to connect
+	// via the relays.
 	//
 	// TODO: Clean shutdown, and/or handling config changes on the fly. We
 	// partly do this now - new devices and addresses will be picked up, but
@@ -79,28 +85,46 @@ func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, model *model.
 
 	svc.Add(serviceFunc(svc.connect))
 	for _, addr := range svc.cfg.Options().ListenAddress {
-		addr := addr
-		listener := serviceFunc(func() {
-			svc.listen(addr)
-		})
-		svc.Add(listener)
+		uri, err := url.Parse(addr)
+		if err != nil {
+			l.Infoln("Failed to parse listen address:", addr, err)
+			continue
+		}
+
+		listener, ok := listeners[uri.Scheme]
+		if !ok {
+			l.Infoln("Unknown listen address scheme:", uri.String())
+			continue
+		}
+
+		if debugNet {
+			l.Debugln("listening on", uri.String())
+		}
+
+		svc.Add(serviceFunc(func() {
+			listener(uri, svc.tlsCfg, svc.conns)
+		}))
 	}
 	svc.Add(serviceFunc(svc.handle))
 
+	if svc.relaySvc != nil {
+		svc.Add(serviceFunc(svc.acceptRelayConns))
+	}
+
 	return svc
 }
 
 func (s *connectionSvc) handle() {
 next:
-	for conn := range s.conns {
-		cs := conn.ConnectionState()
+	for c := range s.conns {
+		cs := c.Conn.ConnectionState()
 
 		// We should have negotiated the next level protocol "bep/1.0" as part
 		// of the TLS handshake. Unfortunately this can't be a hard error,
 		// because there are implementations out there that don't support
 		// protocol negotiation (iOS for one...).
 		if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != bepProtocolName {
-			l.Infof("Peer %s did not negotiate bep/1.0", conn.RemoteAddr())
+			l.Infof("Peer %s did not negotiate bep/1.0", c.Conn.RemoteAddr())
 		}
 
 		// We should have received exactly one certificate from the other
@@ -108,8 +132,8 @@ next:
 		// connection.
 		certs := cs.PeerCertificates
 		if cl := len(certs); cl != 1 {
-			l.Infof("Got peer certificate list of length %d != 1 from %s; protocol error", cl, conn.RemoteAddr())
-			conn.Close()
+			l.Infof("Got peer certificate list of length %d != 1 from %s; protocol error", cl, c.Conn.RemoteAddr())
+			c.Conn.Close()
 			continue
 		}
 		remoteCert := certs[0]
@@ -120,19 +144,33 @@ next:
 		// clients between the same NAT gateway, and global discovery.
 		if remoteID == myID {
 			l.Infof("Connected to myself (%s) - should not happen", remoteID)
-			conn.Close()
+			c.Conn.Close()
 			continue
 		}
 
-		// We should not already be connected to the other party. TODO: This
-		// could use some better handling. If the old connection is dead but
-		// hasn't timed out yet we may want to drop *that* connection and keep
-		// this one. But in case we are two devices connecting to each other
-		// in parallel we don't want to do that or we end up with no
-		// connections still established...
-		if s.model.ConnectedTo(remoteID) {
+		// If we have a relay connection, and the new incoming connection is
+		// not a relay connection, we should drop that, and prefer the this one.
+		s.mut.RLock()
+		ct, ok := s.connType[remoteID]
+		s.mut.RUnlock()
+		if ok && !ct.IsDirect() && c.Type.IsDirect() {
+			if debugNet {
+				l.Debugln("Switching connections", remoteID)
+			}
+			s.model.Close(remoteID, fmt.Errorf("switching connections"))
+		} else if s.model.ConnectedTo(remoteID) {
+			// We should not already be connected to the other party. TODO: This
+			// could use some better handling. If the old connection is dead but
+			// hasn't timed out yet we may want to drop *that* connection and keep
+			// this one. But in case we are two devices connecting to each other
+			// in parallel we don't want to do that or we end up with no
+			// connections still established...
 			l.Infof("Connected to already connected device (%s)", remoteID)
-			conn.Close()
+			c.Conn.Close()
+			continue
+		} else if s.model.IsPaused(remoteID) {
+			l.Infof("Connection from paused device (%s)", remoteID)
+			c.Conn.Close()
 			continue
 		}
 
@@ -150,35 +188,42 @@ next:
 					// Incorrect certificate name is something the user most
 					// likely wants to know about, since it's an advanced
 					// config. Warn instead of Info.
-					l.Warnf("Bad certificate from %s (%v): %v", remoteID, conn.RemoteAddr(), err)
-					conn.Close()
+					l.Warnf("Bad certificate from %s (%v): %v", remoteID, c.Conn.RemoteAddr(), err)
+					c.Conn.Close()
 					continue next
 				}
 
 				// If rate limiting is set, and based on the address we should
 				// limit the connection, then we wrap it in a limiter.
 
-				limit := s.shouldLimit(conn.RemoteAddr())
+				limit := s.shouldLimit(c.Conn.RemoteAddr())
 
-				wr := io.Writer(conn)
+				wr := io.Writer(c.Conn)
 				if limit && writeRateLimit != nil {
-					wr = &limitedWriter{conn, writeRateLimit}
+					wr = &limitedWriter{c.Conn, writeRateLimit}
 				}
 
-				rd := io.Reader(conn)
+				rd := io.Reader(c.Conn)
 				if limit && readRateLimit != nil {
-					rd = &limitedReader{conn, readRateLimit}
+					rd = &limitedReader{c.Conn, readRateLimit}
 				}
 
-				name := fmt.Sprintf("%s-%s", conn.LocalAddr(), conn.RemoteAddr())
+				name := fmt.Sprintf("%s-%s (%s)", c.Conn.LocalAddr(), c.Conn.RemoteAddr(), c.Type)
 				protoConn := protocol.NewConnection(remoteID, rd, wr, s.model, name, deviceCfg.Compression)
 
 				l.Infof("Established secure connection to %s at %s", remoteID, name)
 				if debugNet {
-					l.Debugf("cipher suite: %04X in lan: %t", conn.ConnectionState().CipherSuite, !limit)
+					l.Debugf("cipher suite: %04X in lan: %t", c.Conn.ConnectionState().CipherSuite, !limit)
 				}
 
-				s.model.AddConnection(conn, protoConn)
+				s.model.AddConnection(model.Connection{
+					c.Conn,
+					protoConn,
+					c.Type,
+				})
+				s.mut.Lock()
+				s.connType[remoteID] = c.Type
+				s.mut.Unlock()
 				continue next
 			}
 		}
@@ -186,54 +231,12 @@ next:
 		if !s.cfg.IgnoredDevice(remoteID) {
 			events.Default.Log(events.DeviceRejected, map[string]string{
 				"device":  remoteID.String(),
-				"address": conn.RemoteAddr().String(),
+				"address": c.Conn.RemoteAddr().String(),
 			})
-			l.Infof("Connection from %s with unknown device ID %s", conn.RemoteAddr(), remoteID)
-		} else {
-			l.Infof("Connection from %s with ignored device ID %s", conn.RemoteAddr(), remoteID)
 		}
 
-		conn.Close()
-	}
-}
-
-func (s *connectionSvc) listen(addr string) {
-	if debugNet {
-		l.Debugln("listening on", addr)
-	}
-
-	tcaddr, err := net.ResolveTCPAddr("tcp", addr)
-	if err != nil {
-		l.Fatalln("listen (BEP):", err)
-	}
-	listener, err := net.ListenTCP("tcp", tcaddr)
-	if err != nil {
-		l.Fatalln("listen (BEP):", err)
-	}
-
-	for {
-		conn, err := listener.Accept()
-		if err != nil {
-			l.Warnln("Accepting connection:", err)
-			continue
-		}
-
-		if debugNet {
-			l.Debugln("connect from", conn.RemoteAddr())
-		}
-
-		tcpConn := conn.(*net.TCPConn)
-		s.setTCPOptions(tcpConn)
-
-		tc := tls.Server(conn, s.tlsCfg)
-		err = tc.Handshake()
-		if err != nil {
-			l.Infoln("TLS handshake:", err)
-			tc.Close()
-			continue
-		}
-
-		s.conns <- tc
+		l.Infof("Connection from %s (%s) with ignored device ID %s", c.Conn.RemoteAddr(), c.Type, remoteID)
+		c.Conn.Close()
 	}
 }
 
@@ -246,19 +249,29 @@ func (s *connectionSvc) connect() {
 				continue
 			}
 
-			if s.model.ConnectedTo(deviceID) {
+			if s.model.IsPaused(deviceID) {
+				continue
+			}
+
+			connected := s.model.ConnectedTo(deviceID)
+
+			s.mut.RLock()
+			ct, ok := s.connType[deviceID]
+			relaysEnabled := s.relaysEnabled
+			s.mut.RUnlock()
+			if connected && ok && ct.IsDirect() {
 				continue
 			}
 
 			var addrs []string
+			var relays []discover.Relay
 			for _, addr := range deviceCfg.Addresses {
 				if addr == "dynamic" {
-					if discoverer != nil {
-						t := discoverer.Lookup(deviceID)
-						if len(t) == 0 {
-							continue
+					if s.discoverer != nil {
+						if t, r, err := s.discoverer.Lookup(deviceID); err == nil {
+							addrs = append(addrs, t...)
+							relays = append(relays, r...)
 						}
-						addrs = append(addrs, t...)
 					}
 				} else {
 					addrs = append(addrs, addr)
@@ -266,45 +279,108 @@ func (s *connectionSvc) connect() {
 			}
 
 			for _, addr := range addrs {
-				host, port, err := net.SplitHostPort(addr)
-				if err != nil && strings.HasPrefix(err.Error(), "missing port") {
-					// addr is on the form "1.2.3.4"
-					addr = net.JoinHostPort(addr, "22000")
-				} else if err == nil && port == "" {
-					// addr is on the form "1.2.3.4:"
-					addr = net.JoinHostPort(host, "22000")
+				uri, err := url.Parse(addr)
+				if err != nil {
+					l.Infoln("Failed to parse connection url:", addr, err)
+					continue
 				}
+
+				dialer, ok := dialers[uri.Scheme]
+				if !ok {
+					l.Infoln("Unknown address schema", uri.String())
+					continue
+				}
+
 				if debugNet {
-					l.Debugln("dial", deviceCfg.DeviceID, addr)
+					l.Debugln("dial", deviceCfg.DeviceID, uri.String())
 				}
-
-				raddr, err := net.ResolveTCPAddr("tcp", addr)
+				conn, err := dialer(uri, s.tlsCfg)
 				if err != nil {
 					if debugNet {
-						l.Debugln(err)
+						l.Debugln("dial failed", deviceCfg.DeviceID, uri.String(), err)
 					}
 					continue
 				}
 
-				conn, err := net.DialTCP("tcp", nil, raddr)
+				if connected {
+					s.model.Close(deviceID, fmt.Errorf("switching connections"))
+				}
+
+				s.conns <- model.IntermediateConnection{
+					conn, model.ConnectionTypeDirectDial,
+				}
+				continue nextDevice
+			}
+
+			// Only connect via relays if not already connected
+			// Also, do not set lastRelayCheck time if we have no relays,
+			// as otherwise when we do discover relays, we might have to
+			// wait up to RelayReconnectIntervalM to connect again.
+			// Also, do not try relays if we are explicitly told not to.
+			if connected || len(relays) == 0 || !relaysEnabled {
+				continue nextDevice
+			}
+
+			reconIntv := time.Duration(s.cfg.Options().RelayReconnectIntervalM) * time.Minute
+			if last, ok := s.lastRelayCheck[deviceID]; ok && time.Since(last) < reconIntv {
+				if debugNet {
+					l.Debugln("Skipping connecting via relay to", deviceID, "last checked at", last)
+				}
+				continue nextDevice
+			} else if debugNet {
+				l.Debugln("Trying relay connections to", deviceID, relays)
+			}
+
+			s.lastRelayCheck[deviceID] = time.Now()
+
+			for _, addr := range relays {
+				uri, err := url.Parse(addr.URL)
 				if err != nil {
-					if debugNet {
-						l.Debugln(err)
-					}
+					l.Infoln("Failed to parse relay connection url:", addr, err)
 					continue
 				}
 
-				s.setTCPOptions(conn)
+				inv, err := client.GetInvitationFromRelay(uri, deviceID, s.tlsCfg.Certificates)
+				if err != nil {
+					if debugNet {
+						l.Debugf("Failed to get invitation for %s from %s: %v", deviceID, uri, err)
+					}
+					continue
+				} else if debugNet {
+					l.Debugln("Succesfully retrieved relay invitation", inv, "from", uri)
+				}
 
-				tc := tls.Client(conn, s.tlsCfg)
+				conn, err := client.JoinSession(inv)
+				if err != nil {
+					if debugNet {
+						l.Debugf("Failed to join relay session %s: %v", inv, err)
+					}
+					continue
+				} else if debugNet {
+					l.Debugln("Sucessfully joined relay session", inv)
+				}
+
+				err = osutil.SetTCPOptions(conn.(*net.TCPConn))
+				if err != nil {
+					l.Infoln(err)
+				}
+
+				var tc *tls.Conn
+
+				if inv.ServerSocket {
+					tc = tls.Server(conn, s.tlsCfg)
+				} else {
+					tc = tls.Client(conn, s.tlsCfg)
+				}
 				err = tc.Handshake()
 				if err != nil {
-					l.Infoln("TLS handshake:", err)
+					l.Infof("TLS handshake (BEP/relay %s): %v", inv, err)
 					tc.Close()
 					continue
 				}
-
-				s.conns <- tc
+				s.conns <- model.IntermediateConnection{
+					tc, model.ConnectionTypeRelayDial,
+				}
 				continue nextDevice
 			}
 		}
@@ -317,19 +393,13 @@ func (s *connectionSvc) connect() {
 	}
 }
 
-func (*connectionSvc) setTCPOptions(conn *net.TCPConn) {
-	var err error
-	if err = conn.SetLinger(0); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetNoDelay(false); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetKeepAlive(true); err != nil {
-		l.Infoln(err)
+func (s *connectionSvc) acceptRelayConns() {
+	for {
+		conn := s.relaySvc.Accept()
+		s.conns <- model.IntermediateConnection{
+			Conn: conn,
+			Type: model.ConnectionTypeRelayAccept,
+		}
 	}
 }
 
@@ -355,6 +425,10 @@ func (s *connectionSvc) VerifyConfiguration(from, to config.Configuration) error
 }
 
 func (s *connectionSvc) CommitConfiguration(from, to config.Configuration) bool {
+	s.mut.Lock()
+	s.relaysEnabled = to.Options.RelaysEnabled
+	s.mut.Unlock()
+
 	// We require a restart if a device as been removed.
 
 	newDevices := make(map[protocol.DeviceID]bool, len(to.Devices))

cmd/syncthing/connections_tcp.go

@@ -0,0 +1,105 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"crypto/tls"
+	"net"
+	"net/url"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/osutil"
+)
+
+func init() {
+	dialers["tcp"] = tcpDialer
+	listeners["tcp"] = tcpListener
+}
+
+func tcpDialer(uri *url.URL, tlsCfg *tls.Config) (*tls.Conn, error) {
+	host, port, err := net.SplitHostPort(uri.Host)
+	if err != nil && strings.HasPrefix(err.Error(), "missing port") {
+		// addr is on the form "1.2.3.4"
+		uri.Host = net.JoinHostPort(uri.Host, "22000")
+	} else if err == nil && port == "" {
+		// addr is on the form "1.2.3.4:"
+		uri.Host = net.JoinHostPort(host, "22000")
+	}
+
+	raddr, err := net.ResolveTCPAddr("tcp", uri.Host)
+	if err != nil {
+		if debugNet {
+			l.Debugln(err)
+		}
+		return nil, err
+	}
+
+	conn, err := net.DialTCP("tcp", nil, raddr)
+	if err != nil {
+		if debugNet {
+			l.Debugln(err)
+		}
+		return nil, err
+	}
+
+	err = osutil.SetTCPOptions(conn)
+	if err != nil {
+		l.Infoln(err)
+	}
+
+	tc := tls.Client(conn, tlsCfg)
+	err = tc.Handshake()
+	if err != nil {
+		tc.Close()
+		return nil, err
+	}
+
+	return tc, nil
+}
+
+func tcpListener(uri *url.URL, tlsCfg *tls.Config, conns chan<- model.IntermediateConnection) {
+	tcaddr, err := net.ResolveTCPAddr("tcp", uri.Host)
+	if err != nil {
+		l.Fatalln("listen (BEP/tcp):", err)
+		return
+	}
+	listener, err := net.ListenTCP("tcp", tcaddr)
+	if err != nil {
+		l.Fatalln("listen (BEP/tcp):", err)
+		return
+	}
+
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			l.Warnln("Accepting connection (BEP/tcp):", err)
+			continue
+		}
+
+		if debugNet {
+			l.Debugln("connect from", conn.RemoteAddr())
+		}
+
+		err = osutil.SetTCPOptions(conn.(*net.TCPConn))
+		if err != nil {
+			l.Infoln(err)
+		}
+
+		tc := tls.Server(conn, tlsCfg)
+		err = tc.Handshake()
+		if err != nil {
+			l.Infoln("TLS handshake (BEP/tcp):", err)
+			tc.Close()
+			continue
+		}
+
+		conns <- model.IntermediateConnection{
+			tc, model.ConnectionTypeDirectAccept,
+		}
+	}
+}

cmd/syncthing/gui.go

@@ -33,7 +33,9 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/relay"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/vitrun/qart/qr"
 	"golang.org/x/crypto/bcrypt"
@@ -56,21 +58,25 @@ type apiSvc struct {
 	cfg             config.GUIConfiguration
 	assetDir        string
 	model           *model.Model
+	eventSub        *events.BufferedSubscription
+	discoverer      *discover.CachingMux
+	relaySvc        *relay.Svc
 	listener        net.Listener
 	fss             *folderSummarySvc
 	stop            chan struct{}
 	systemConfigMut sync.Mutex
-	eventSub        *events.BufferedSubscription
 }
 
-func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription) (*apiSvc, error) {
+func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc) (*apiSvc, error) {
 	svc := &apiSvc{
 		id:              id,
 		cfg:             cfg,
 		assetDir:        assetDir,
 		model:           m,
-		systemConfigMut: sync.NewMutex(),
 		eventSub:        eventSub,
+		discoverer:      discoverer,
+		relaySvc:        relaySvc,
+		systemConfigMut: sync.NewMutex(),
 	}
 
 	var err error
@@ -92,7 +98,7 @@ func (s *apiSvc) getListener(cfg config.GUIConfiguration) (net.Listener, error)
 			name = tlsDefaultCommonName
 		}
 
-		cert, err = newCertificate(locations[locHTTPSCertFile], locations[locHTTPSKeyFile], name)
+		cert, err = tlsutil.NewCertificate(locations[locHTTPSCertFile], locations[locHTTPSKeyFile], name, tlsRSABits)
 	}
 	if err != nil {
 		return nil, err
@@ -120,7 +126,7 @@ func (s *apiSvc) getListener(cfg config.GUIConfiguration) (net.Listener, error)
 		return nil, err
 	}
 
-	listener := &DowngradingListener{rawListener, tlsCfg}
+	listener := &tlsutil.DowngradingListener{rawListener, tlsCfg}
 	return listener, nil
 }
 
@@ -161,7 +167,6 @@ func (s *apiSvc) Serve() {
 	postRestMux.HandleFunc("/rest/db/override", s.postDBOverride)              // folder
 	postRestMux.HandleFunc("/rest/db/scan", s.postDBScan)                      // folder [sub...] [delay]
 	postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig)          // <body>
-	postRestMux.HandleFunc("/rest/system/discovery", s.postSystemDiscovery)    // device addr
 	postRestMux.HandleFunc("/rest/system/error", s.postSystemError)            // <body>
 	postRestMux.HandleFunc("/rest/system/error/clear", s.postSystemErrorClear) // -
 	postRestMux.HandleFunc("/rest/system/ping", s.restPing)                    // -
@@ -169,6 +174,8 @@ func (s *apiSvc) Serve() {
 	postRestMux.HandleFunc("/rest/system/restart", s.postSystemRestart)        // -
 	postRestMux.HandleFunc("/rest/system/shutdown", s.postSystemShutdown)      // -
 	postRestMux.HandleFunc("/rest/system/upgrade", s.postSystemUpgrade)        // -
+	postRestMux.HandleFunc("/rest/system/pause", s.postSystemPause)            // device
+	postRestMux.HandleFunc("/rest/system/resume", s.postSystemResume)          // device
 
 	// Debug endpoints, not for general use
 	getRestMux.HandleFunc("/rest/debug/peerCompletion", s.getPeerCompletion)
@@ -625,8 +632,30 @@ func (s *apiSvc) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["alloc"] = m.Alloc
 	res["sys"] = m.Sys - m.HeapReleased
 	res["tilde"] = tilde
-	if cfg.Options().GlobalAnnEnabled && discoverer != nil {
-		res["extAnnounceOK"] = discoverer.ExtAnnounceOK()
+	if cfg.Options().LocalAnnEnabled || cfg.Options().GlobalAnnEnabled {
+		res["discoveryEnabled"] = true
+		discoErrors := make(map[string]string)
+		discoMethods := 0
+		for disco, err := range s.discoverer.ChildErrors() {
+			discoMethods++
+			if err != nil {
+				discoErrors[disco] = err.Error()
+			}
+		}
+		res["discoveryMethods"] = discoMethods
+		res["discoveryErrors"] = discoErrors
+	}
+	if s.relaySvc != nil {
+		res["relaysEnabled"] = true
+		relayClientStatus := make(map[string]bool)
+		relayClientLatency := make(map[string]int)
+		for _, relay := range s.relaySvc.Relays() {
+			latency, ok := s.relaySvc.RelayStatus(relay)
+			relayClientStatus[relay] = ok
+			relayClientLatency[relay] = int(latency / time.Millisecond)
+		}
+		res["relayClientStatus"] = relayClientStatus
+		res["relayClientLatency"] = relayClientLatency
 	}
 	cpuUsageLock.RLock()
 	var cpusum float64
@@ -637,6 +666,7 @@ func (s *apiSvc) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["cpuPercent"] = cpusum / float64(len(cpuUsagePercent)) / float64(runtime.NumCPU())
 	res["pathSeparator"] = string(filepath.Separator)
 	res["uptime"] = int(time.Since(startTime).Seconds())
+	res["startTime"] = startTime
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(res)
@@ -670,25 +700,16 @@ func (s *apiSvc) showGuiError(l logger.LogLevel, err string) {
 	guiErrorsMut.Unlock()
 }
 
-func (s *apiSvc) postSystemDiscovery(w http.ResponseWriter, r *http.Request) {
-	var qs = r.URL.Query()
-	var device = qs.Get("device")
-	var addr = qs.Get("addr")
-	if len(device) != 0 && len(addr) != 0 && discoverer != nil {
-		discoverer.Hint(device, []string{addr})
-	}
-}
-
 func (s *apiSvc) getSystemDiscovery(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	devices := map[string][]discover.CacheEntry{}
+	devices := make(map[string]discover.CacheEntry)
 
-	if discoverer != nil {
+	if s.discoverer != nil {
 		// Device ids can't be marshalled as keys so we need to manually
 		// rebuild this map using strings. Discoverer may be nil if discovery
 		// has not started yet.
-		for device, entries := range discoverer.All() {
-			devices[device.String()] = entries
+		for device, entry := range s.discoverer.Cache() {
+			devices[device.String()] = entry
 		}
 	}
 
@@ -766,7 +787,7 @@ func (s *apiSvc) getSystemUpgrade(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, upgrade.ErrUpgradeUnsupported.Error(), 500)
 		return
 	}
-	rel, err := upgrade.LatestRelease(Version)
+	rel, err := upgrade.LatestRelease(cfg.Options().ReleasesURL, Version)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return
@@ -809,7 +830,7 @@ func (s *apiSvc) getLang(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *apiSvc) postSystemUpgrade(w http.ResponseWriter, r *http.Request) {
-	rel, err := upgrade.LatestRelease(Version)
+	rel, err := upgrade.LatestRelease(cfg.Options().ReleasesURL, Version)
 	if err != nil {
 		l.Warnln("getting latest release:", err)
 		http.Error(w, err.Error(), 500)
@@ -830,6 +851,32 @@ func (s *apiSvc) postSystemUpgrade(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *apiSvc) postSystemPause(w http.ResponseWriter, r *http.Request) {
+	var qs = r.URL.Query()
+	var deviceStr = qs.Get("device")
+
+	device, err := protocol.DeviceIDFromString(deviceStr)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	s.model.PauseDevice(device)
+}
+
+func (s *apiSvc) postSystemResume(w http.ResponseWriter, r *http.Request) {
+	var qs = r.URL.Query()
+	var deviceStr = qs.Get("device")
+
+	device, err := protocol.DeviceIDFromString(deviceStr)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	s.model.ResumeDevice(device)
+}
+
 func (s *apiSvc) postDBScan(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")

cmd/syncthing/main.go

@@ -34,8 +34,11 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/relay"
 	"github.com/syncthing/syncthing/lib/symlinks"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
+
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -45,7 +48,7 @@ import (
 
 var (
 	Version     = "unknown-dev"
-	Codename    = "Aluminium Ant"
+	Codename    = "Beryllium Bedbug"
 	BuildEnv    = "default"
 	BuildStamp  = "0"
 	BuildDate   time.Time
@@ -65,8 +68,10 @@ const (
 )
 
 const (
-	bepProtocolName   = "bep/1.0"
-	pingEventInterval = time.Minute
+	bepProtocolName      = "bep/1.0"
+	tlsDefaultCommonName = "syncthing"
+	tlsRSABits           = 3072
+	pingEventInterval    = time.Minute
 )
 
 var l = logger.DefaultLogger
@@ -109,7 +114,6 @@ var (
 	writeRateLimit *ratelimit.Bucket
 	readRateLimit  *ratelimit.Bucket
 	stop           = make(chan int)
-	discoverer     *discover.Discoverer
 	cert           tls.Certificate
 	lans           []*net.IPNet
 )
@@ -198,6 +202,7 @@ var (
 	logFile           string
 	auditEnabled      bool
 	verbose           bool
+	paused            bool
 	noRestart         = os.Getenv("STNORESTART") != ""
 	noUpgrade         = os.Getenv("STNOUPGRADE") != ""
 	guiAddress        = os.Getenv("STGUIADDRESS") // legacy
@@ -237,6 +242,7 @@ func main() {
 	flag.StringVar(&upgradeTo, "upgrade-to", upgradeTo, "Force upgrade directly from specified URL")
 	flag.BoolVar(&auditEnabled, "audit", false, "Write events to audit file")
 	flag.BoolVar(&verbose, "verbose", false, "Print verbose log output")
+	flag.BoolVar(&paused, "paused", false, "Start with all devices paused")
 
 	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, baseDirs["config"]))
 	flag.Parse()
@@ -293,10 +299,13 @@ func main() {
 			l.Warnln("Key exists; will not overwrite.")
 			l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
 		} else {
-			cert, err = newCertificate(certFile, keyFile, tlsDefaultCommonName)
+			cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, tlsRSABits)
+			if err != nil {
+				l.Fatalln("Create certificate:", err)
+			}
 			myID = protocol.NewDeviceID(cert.Certificate[0])
 			if err != nil {
-				l.Fatalln("load cert:", err)
+				l.Fatalln("Load certificate:", err)
 			}
 			if err == nil {
 				l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
@@ -336,7 +345,7 @@ func main() {
 	}
 
 	if doUpgrade || doUpgradeCheck {
-		rel, err := upgrade.LatestRelease(Version)
+		rel, err := upgrade.LatestRelease(cfg.Options().ReleasesURL, Version)
 		if err != nil {
 			l.Fatalln("Upgrade:", err) // exits 1
 		}
@@ -459,9 +468,10 @@ func syncthingMain() {
 	// Ensure that that we have a certificate and key.
 	cert, err := tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
 	if err != nil {
-		cert, err = newCertificate(locations[locCertFile], locations[locKeyFile], tlsDefaultCommonName)
+		l.Infof("Generating RSA key and certificate for %s...", tlsDefaultCommonName)
+		cert, err = tlsutil.NewCertificate(locations[locCertFile], locations[locKeyFile], tlsDefaultCommonName, tlsRSABits)
 		if err != nil {
-			l.Fatalln("load cert:", err)
+			l.Fatalln(err)
 		}
 	}
 
@@ -567,9 +577,6 @@ func syncthingMain() {
 		symlinks.Supported = false
 	}
 
-	protocol.PingTimeout = time.Duration(opts.PingTimeoutS) * time.Second
-	protocol.PingIdleTime = time.Duration(opts.PingIdleTimeS) * time.Second
-
 	if opts.MaxSendKbps > 0 {
 		writeRateLimit = ratelimit.NewBucketWithRate(float64(1000*opts.MaxSendKbps), int64(5*1000*opts.MaxSendKbps))
 	}
@@ -583,6 +590,14 @@ func syncthingMain() {
 		for _, lan := range lans {
 			networks = append(networks, lan.String())
 		}
+		for _, lan := range opts.AlwaysLocalNets {
+			_, ipnet, err := net.ParseCIDR(lan)
+			if err != nil {
+				l.Infoln("Network", lan, "is malformed:", err)
+				continue
+			}
+			networks = append(networks, ipnet.String())
+		}
 		l.Infoln("Local networks:", strings.Join(networks, ", "))
 	}
 
@@ -623,7 +638,13 @@ func syncthingMain() {
 			m.StartDeadlockDetector(time.Duration(it) * time.Second)
 		}
 	} else if !IsRelease || IsBeta {
-		m.StartDeadlockDetector(20 * 60 * time.Second)
+		m.StartDeadlockDetector(20 * time.Minute)
+	}
+
+	if paused {
+		for device := range cfg.Devices() {
+			m.PauseDevice(device)
+		}
 	}
 
 	// Clear out old indexes for other devices. Otherwise we'll start up and
@@ -648,32 +669,88 @@ func syncthingMain() {
 
 	mainSvc.Add(m)
 
-	// GUI
-
-	setupGUI(mainSvc, cfg, m, apiSub)
-
 	// The default port we announce, possibly modified by setupUPnP next.
 
-	addr, err := net.ResolveTCPAddr("tcp", opts.ListenAddress[0])
+	uri, err := url.Parse(opts.ListenAddress[0])
+	if err != nil {
+		l.Fatalf("Failed to parse listen address %s: %v", opts.ListenAddress[0], err)
+	}
+
+	addr, err := net.ResolveTCPAddr("tcp", uri.Host)
 	if err != nil {
 		l.Fatalln("Bad listen address:", err)
 	}
 
-	// Start discovery
+	// The externalAddr tracks our external addresses for discovery purposes.
 
-	localPort := addr.Port
-	discoverer = discovery(localPort)
+	var addrList *addressLister
 
-	// Start UPnP. The UPnP service will restart global discovery if the
-	// external port changes.
+	// Start UPnP
 
 	if opts.UPnPEnabled {
-		upnpSvc := newUPnPSvc(cfg, localPort)
+		upnpSvc := newUPnPSvc(cfg, addr.Port)
 		mainSvc.Add(upnpSvc)
+
+		// The external address tracker needs to know about the UPnP service
+		// so it can check for an external mapped port.
+		addrList = newAddressLister(upnpSvc, cfg)
+	} else {
+		addrList = newAddressLister(nil, cfg)
 	}
 
-	connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg)
-	cfg.Subscribe(connectionSvc)
+	// Start relay management
+
+	var relaySvc *relay.Svc
+	if opts.RelaysEnabled && (opts.GlobalAnnEnabled || opts.RelayWithoutGlobalAnn) {
+		relaySvc = relay.NewSvc(cfg, tlsCfg)
+		mainSvc.Add(relaySvc)
+	}
+
+	// Start discovery
+
+	cachedDiscovery := discover.NewCachingMux()
+	mainSvc.Add(cachedDiscovery)
+
+	if cfg.Options().GlobalAnnEnabled {
+		for _, srv := range cfg.GlobalDiscoveryServers() {
+			l.Infoln("Using discovery server", srv)
+			gd, err := discover.NewGlobal(srv, cert, addrList, relaySvc)
+			if err != nil {
+				l.Warnln("Global discovery:", err)
+				continue
+			}
+
+			// Each global discovery server gets its results cached for five
+			// minutes, and is not asked again for a minute when it's returned
+			// unsuccessfully.
+			cachedDiscovery.Add(gd, 5*time.Minute, time.Minute)
+		}
+	}
+
+	if cfg.Options().LocalAnnEnabled {
+		// v4 broadcasts
+		bcd, err := discover.NewLocal(myID, fmt.Sprintf(":%d", cfg.Options().LocalAnnPort), addrList, relaySvc)
+		if err != nil {
+			l.Warnln("IPv4 local discovery:", err)
+		} else {
+			cachedDiscovery.Add(bcd, 0, 0)
+		}
+		// v6 multicasts
+		mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, addrList, relaySvc)
+		if err != nil {
+			l.Warnln("IPv6 local discovery:", err)
+		} else {
+			cachedDiscovery.Add(mcd, 0, 0)
+		}
+	}
+
+	// GUI
+
+	setupGUI(mainSvc, cfg, m, apiSub, cachedDiscovery, relaySvc)
+
+	// Start connection management
+
+	connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg, cachedDiscovery, relaySvc)
 	mainSvc.Add(connectionSvc)
 
 	if cpuProfile {
@@ -796,7 +873,7 @@ func startAuditing(mainSvc *suture.Supervisor) {
 	l.Infoln("Audit log in", auditFile)
 }
 
-func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription) {
+func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc) {
 	opts := cfg.Options()
 	guiCfg := overrideGUIConfig(cfg.GUI(), guiAddress, guiAuthentication, guiAPIKey)
 
@@ -825,7 +902,7 @@ func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, a
 
 			urlShow := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostShow, strconv.Itoa(addr.Port)))
 			l.Infoln("Starting web GUI on", urlShow)
-			api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub)
+			api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub, discoverer, relaySvc)
 			if err != nil {
 				l.Fatalln("Cannot start GUI:", err)
 			}
@@ -871,7 +948,7 @@ func defaultConfig(myName string) config.Configuration {
 	if err != nil {
 		l.Fatalln("get free port (BEP):", err)
 	}
-	newCfg.Options.ListenAddress = []string{fmt.Sprintf("0.0.0.0:%d", port)}
+	newCfg.Options.ListenAddress = []string{fmt.Sprintf("tcp://0.0.0.0:%d", port)}
 	return newCfg
 }
 
@@ -896,23 +973,6 @@ func shutdown() {
 	stop <- exitSuccess
 }
 
-func discovery(extPort int) *discover.Discoverer {
-	opts := cfg.Options()
-	disc := discover.NewDiscoverer(myID, opts.ListenAddress)
-
-	if opts.LocalAnnEnabled {
-		l.Infoln("Starting local discovery announcements")
-		disc.StartLocal(opts.LocalAnnPort, opts.LocalAnnMCAddr)
-	}
-
-	if opts.GlobalAnnEnabled {
-		l.Infoln("Starting global discovery announcements")
-		disc.StartGlobal(opts.GlobalAnnServers, uint16(extPort))
-	}
-
-	return disc
-}
-
 func ensureDir(dir string, mode int) {
 	fi, err := os.Stat(dir)
 	if os.IsNotExist(err) {
@@ -1026,7 +1086,7 @@ func autoUpgrade() {
 		case <-timer.C:
 		}
 
-		rel, err := upgrade.LatestRelease(Version)
+		rel, err := upgrade.LatestRelease(cfg.Options().ReleasesURL, Version)
 		if err == upgrade.ErrUpgradeUnsupported {
 			events.Default.Unsubscribe(sub)
 			return

cmd/syncthing/upnpsvc.go

@@ -11,26 +11,30 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/upnp"
 )
 
 // The UPnP service runs a loop for discovery of IGDs (Internet Gateway
 // Devices) and setup/renewal of a port mapping.
 type upnpSvc struct {
-	cfg       *config.Wrapper
-	localPort int
-	stop      chan struct{}
+	cfg        *config.Wrapper
+	localPort  int
+	extPort    int
+	extPortMut sync.Mutex
+	stop       chan struct{}
 }
 
 func newUPnPSvc(cfg *config.Wrapper, localPort int) *upnpSvc {
 	return &upnpSvc{
-		cfg:       cfg,
-		localPort: localPort,
+		cfg:        cfg,
+		localPort:  localPort,
+		extPortMut: sync.NewMutex(),
 	}
 }
 
 func (s *upnpSvc) Serve() {
-	extPort := 0
 	foundIGD := true
 	s.stop = make(chan struct{})
 
@@ -38,7 +42,15 @@ func (s *upnpSvc) Serve() {
 		igds := upnp.Discover(time.Duration(s.cfg.Options().UPnPTimeoutS) * time.Second)
 		if len(igds) > 0 {
 			foundIGD = true
-			extPort = s.tryIGDs(igds, extPort)
+			s.extPortMut.Lock()
+			oldExtPort := s.extPort
+			s.extPortMut.Unlock()
+
+			newExtPort := s.tryIGDs(igds, oldExtPort)
+
+			s.extPortMut.Lock()
+			s.extPort = newExtPort
+			s.extPortMut.Unlock()
 		} else if foundIGD {
 			// Only print a notice if we've previously found an IGD or this is
 			// the first time around.
@@ -64,6 +76,13 @@ func (s *upnpSvc) Stop() {
 	close(s.stop)
 }
 
+func (s *upnpSvc) ExternalPort() int {
+	s.extPortMut.Lock()
+	port := s.extPort
+	s.extPortMut.Unlock()
+	return port
+}
+
 func (s *upnpSvc) tryIGDs(igds []upnp.IGD, prevExtPort int) int {
 	// Lets try all the IGDs we found and use the first one that works.
 	// TODO: Use all of them, and sort out the resulting mess to the
@@ -76,13 +95,8 @@ func (s *upnpSvc) tryIGDs(igds []upnp.IGD, prevExtPort int) int {
 		}
 
 		if extPort != prevExtPort {
-			// External port changed; refresh the discovery announcement.
-			// TODO: Don't reach out to some magic global here?
 			l.Infof("New UPnP port mapping: external port %d to local port %d.", extPort, s.localPort)
-			if s.cfg.Options().GlobalAnnEnabled {
-				discoverer.StopGlobal()
-				discoverer.StartGlobal(s.cfg.Options().GlobalAnnServers, uint16(extPort))
-			}
+			events.Default.Log(events.ExternalPortMappingChanged, map[string]int{"port": extPort})
 		}
 		if debugNet {
 			l.Debugf("Created/updated UPnP port mapping for external port %d on device %s.", extPort, igd.FriendlyIdentifier())

cmd/syncthing/usage_report.go

@@ -10,22 +10,26 @@ import (
 	"bytes"
 	"crypto/rand"
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
 	"runtime"
+	"sort"
 	"time"
 
+	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/thejerf/suture"
 )
 
 // Current version number of the usage report, for acceptance purposes. If
 // fields are added or changed this integer must be incremented so that users
 // are prompted for acceptance of the new report.
-const usageReportVersion = 1
+const usageReportVersion = 2
 
 type usageReportingManager struct {
 	model *model.Model
@@ -77,6 +81,7 @@ func (m *usageReportingManager) String() string {
 // various places, so not part of the usageReportingSvc object.
 func reportData(m *model.Model) map[string]interface{} {
 	res := make(map[string]interface{})
+	res["urVersion"] = usageReportVersion
 	res["uniqueID"] = cfg.Options().URUniqueID
 	res["version"] = Version
 	res["longVersion"] = LongVersion
@@ -120,10 +125,118 @@ func reportData(m *model.Model) map[string]interface{} {
 	if err == nil {
 		res["memorySize"] = bytes / 1024 / 1024
 	}
+	res["numCPU"] = runtime.NumCPU()
+
+	var rescanIntvs []int
+	folderUses := map[string]int{
+		"readonly":      0,
+		"ignorePerms":   0,
+		"ignoreDelete":  0,
+		"autoNormalize": 0,
+	}
+	for _, cfg := range cfg.Folders() {
+		rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS)
+
+		if cfg.ReadOnly {
+			folderUses["readonly"]++
+		}
+		if cfg.IgnorePerms {
+			folderUses["ignorePerms"]++
+		}
+		if cfg.IgnoreDelete {
+			folderUses["ignoreDelete"]++
+		}
+		if cfg.AutoNormalize {
+			folderUses["autoNormalize"]++
+		}
+	}
+	sort.Ints(rescanIntvs)
+	res["rescanIntvs"] = rescanIntvs
+	res["folderUses"] = folderUses
+
+	deviceUses := map[string]int{
+		"introducer":       0,
+		"customCertName":   0,
+		"compressAlways":   0,
+		"compressMetadata": 0,
+		"compressNever":    0,
+		"dynamicAddr":      0,
+		"staticAddr":       0,
+	}
+	for _, cfg := range cfg.Devices() {
+		if cfg.Introducer {
+			deviceUses["introducer"]++
+		}
+		if cfg.CertName != "" && cfg.CertName != "syncthing" {
+			deviceUses["customCertName"]++
+		}
+		if cfg.Compression == protocol.CompressAlways {
+			deviceUses["compressAlways"]++
+		} else if cfg.Compression == protocol.CompressMetadata {
+			deviceUses["compressMetadata"]++
+		} else if cfg.Compression == protocol.CompressNever {
+			deviceUses["compressNever"]++
+		}
+		for _, addr := range cfg.Addresses {
+			if addr == "dynamic" {
+				deviceUses["dynamicAddr"]++
+			} else {
+				deviceUses["staticAddr"]++
+			}
+		}
+	}
+	res["deviceUses"] = deviceUses
+
+	defaultAnnounceServersDNS, defaultAnnounceServersIP, otherAnnounceServers := 0, 0, 0
+	for _, addr := range cfg.Options().GlobalAnnServers {
+		if addr == "default" {
+			defaultAnnounceServersDNS++
+		} else if stringIn(addr, config.DefaultDiscoveryServersIP) {
+			defaultAnnounceServersIP++
+		} else {
+			otherAnnounceServers++
+		}
+	}
+	res["announce"] = map[string]interface{}{
+		"globalEnabled":     cfg.Options().GlobalAnnEnabled,
+		"localEnabled":      cfg.Options().LocalAnnEnabled,
+		"defaultServersDNS": defaultAnnounceServersDNS,
+		"defaultServersIP":  defaultAnnounceServersIP,
+		"otherServers":      otherAnnounceServers,
+	}
+
+	defaultRelayServers, otherRelayServers := 0, 0
+	for _, addr := range cfg.Options().RelayServers {
+		switch addr {
+		case "dynamic+https://relays.syncthing.net":
+			defaultRelayServers++
+		default:
+			otherRelayServers++
+		}
+	}
+	res["relays"] = map[string]interface{}{
+		"enabled":        cfg.Options().RelaysEnabled,
+		"defaultServers": defaultRelayServers,
+		"otherServers":   otherRelayServers,
+	}
+
+	res["usesRateLimit"] = cfg.Options().MaxRecvKbps > 0 || cfg.Options().MaxSendKbps > 0
+
+	res["upgradeAllowedManual"] = !(upgrade.DisabledByCompilation || noUpgrade)
+	res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgrade) && cfg.Options().AutoUpgradeIntervalH > 0
 
 	return res
 }
 
+func stringIn(needle string, haystack []string) bool {
+	for _, s := range haystack {
+		if needle == s {
+			return true
+		}
+	}
+	return false
+}
+
 type usageReportingService struct {
 	model *model.Model
 	stop  chan struct{}
@@ -134,17 +247,21 @@ func (s *usageReportingService) sendUsageReport() error {
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(d)
 
-	var client = http.DefaultClient
+	transp := &http.Transport{}
+	client := &http.Client{Transport: transp}
 	if BuildEnv == "android" {
 		// This works around the lack of DNS resolution on Android... :(
-		tr := &http.Transport{
-			Dial: func(network, addr string) (net.Conn, error) {
-				return net.Dial(network, "194.126.249.13:443")
-			},
+		transp.Dial = func(network, addr string) (net.Conn, error) {
+			return net.Dial(network, "194.126.249.13:443")
 		}
-		client = &http.Client{Transport: tr}
 	}
-	_, err := client.Post("https://data.syncthing.net/newdata", "application/json", &b)
+
+	if cfg.Options().URPostInsecurely {
+		transp.TLSClientConfig = &tls.Config{
+			InsecureSkipVerify: true,
+		}
+	}
+	_, err := client.Post(cfg.Options().URURL, "application/json", &b)
 	return err
 }
 
@@ -154,7 +271,7 @@ func (s *usageReportingService) Serve() {
 	l.Infoln("Starting usage reporting")
 	defer l.Infoln("Stopping usage reporting")
 
-	t := time.NewTimer(10 * time.Minute) // time to initial report at start
+	t := time.NewTimer(time.Duration(cfg.Options().URInitialDelayS) * time.Second) // time to initial report at start
 	for {
 		select {
 		case <-s.stop:

cmd/syncthing/verbose.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/syncthing/syncthing/lib/events"
 )
@@ -60,7 +61,7 @@ func (s *verboseSvc) WaitForStart() {
 
 func (s *verboseSvc) formatEvent(ev events.Event) string {
 	switch ev.Type {
-	case events.Ping, events.DownloadProgress:
+	case events.Ping, events.DownloadProgress, events.LocalIndexUpdated:
 		// Skip
 		return ""
 
@@ -74,7 +75,7 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
 		return fmt.Sprintf("Discovered device %v at %v", data["device"], data["addrs"])
 	case events.DeviceConnected:
 		data := ev.Data.(map[string]string)
-		return fmt.Sprintf("Connected to device %v at %v", data["id"], data["addr"])
+		return fmt.Sprintf("Connected to device %v at %v (type %s)", data["id"], data["addr"], data["type"])
 	case events.DeviceDisconnected:
 		data := ev.Data.(map[string]string)
 		return fmt.Sprintf("Disconnected from device %v", data["id"])
@@ -86,9 +87,6 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
 	case events.RemoteIndexUpdated:
 		data := ev.Data.(map[string]interface{})
 		return fmt.Sprintf("Device %v sent an index update for %q with %d items", data["device"], data["folder"], data["items"])
-	case events.LocalIndexUpdated:
-		data := ev.Data.(map[string]interface{})
-		return fmt.Sprintf("Updated index for folder %q with %v items", data["folder"], data["items"])
 
 	case events.DeviceRejected:
 		data := ev.Data.(map[string]interface{})
@@ -123,6 +121,35 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
 		delete(sum, "ignorePatterns")
 		delete(sum, "stateChanged")
 		return fmt.Sprintf("Summary for folder %q is %v", data["folder"], data["summary"])
+	case events.FolderScanProgress:
+		data := ev.Data.(map[string]interface{})
+		folder := data["folder"].(string)
+		current := data["current"].(int64)
+		total := data["total"].(int64)
+		var pct int64
+		if total > 0 {
+			pct = 100 * current / total
+		}
+		return fmt.Sprintf("Scanning folder %q, %d%% done", folder, pct)
+
+	case events.DevicePaused:
+		data := ev.Data.(map[string]string)
+		device := data["device"]
+		return fmt.Sprintf("Device %v was paused", device)
+	case events.DeviceResumed:
+		data := ev.Data.(map[string]string)
+		device := data["device"]
+		return fmt.Sprintf("Device %v was resumed", device)
+
+	case events.ExternalPortMappingChanged:
+		data := ev.Data.(map[string]int)
+		port := data["port"]
+		return fmt.Sprintf("External port mapping changed; new port is %d.", port)
+	case events.RelayStateChanged:
+		data := ev.Data.(map[string][]string)
+		newRelays := data["new"]
+		return fmt.Sprintf("Relay state changed; connected relay(s) are %s.", strings.Join(newRelays, ", "))
+
 	}
 
 	return fmt.Sprintf("%s %#v", ev.Type, ev)

etc/linux-systemd/README.md

@@ -1,8 +1,8 @@
 # Systemd Configuration
 
-This directory contains configuration files for running syncthing under the
+This directory contains configuration files for running Syncthing under the
 "systemd" service manager on Linux both under either a systemd system service or
 systemd user service. For further documentation take a look at the [systemd
-section][1] on the Github Wiki.
+section][1] on http://docs.syncthing.net.
 
-[1]: http://docs.syncthing.net/users/autostart.html#systemd 
+[1]: http://docs.syncthing.net/users/autostart.html#systemd

gui/assets/css/overrides.css

@@ -41,8 +41,17 @@ identicon {
     position: relative;
     width: 1em;
     height: 1em;
-    line-height: 1em;
-    overflow: visible;
+    line-height: 1;
+    margin-right: 5px;
+}
+
+.identicon {
+    width: 1em;
+    height: 1em;
+}
+
+.identicon rect {
+    fill: #333;
 }
 
 .checkbox {
@@ -57,17 +66,6 @@ identicon {
     max-width: none;
 }
 
-.identicon {
-    width: 20px;
-    height: 20px;
-}
-
-.panel-heading .identicon {
-    display: block;
-    position: absolute;
-    top: 1px;
-}
-
 .panel-heading .fa, .modal-header .fa {
     margin-right: 10px;
 }
@@ -77,10 +75,6 @@ identicon {
     overflow: hidden;
 }
 
-.identicon rect {
-    fill: #666;
-}
-
 .text-monospace {
     font-family: Menlo, Monaco, Consolas, "Courier New", monospace;
 }
@@ -255,3 +249,7 @@ ul.three-columns li, ul.two-columns li {
         position: static;
     }
 }
+
+.popover {
+    min-width: 250px;
+}

gui/assets/lang/lang-en.json

@@ -49,7 +49,7 @@
    "Edit Folder": "Edit Folder",
    "Editing": "Editing",
    "Enable UPnP": "Enable UPnP",
-   "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.",
+   "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.",
    "Enter ignore patterns, one per line.": "Enter ignore patterns, one per line.",
    "Error": "Error",
    "External File Versioning": "External File Versioning",
@@ -113,6 +113,8 @@
    "Override Changes": "Override Changes",
    "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for",
    "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Path where versions should be stored (leave empty for the default .stversions folder in the folder).",
+   "Pause": "Pause",
+   "Paused": "Paused",
    "Please consult the release notes before performing a major upgrade.": "Please consult the release notes before performing a major upgrade.",
    "Please wait": "Please wait",
    "Preview": "Preview",
@@ -120,6 +122,8 @@
    "Quick guide to supported patterns": "Quick guide to supported patterns",
    "RAM Utilization": "RAM Utilization",
    "Random": "Random",
+   "Relayed via": "Relayed via",
+   "Relays": "Relays",
    "Release Notes": "Release Notes",
    "Remove": "Remove",
    "Rescan": "Rescan",
@@ -128,6 +132,7 @@
    "Restart": "Restart",
    "Restart Needed": "Restart Needed",
    "Restarting": "Restarting",
+   "Resume": "Resume",
    "Reused": "Reused",
    "Save": "Save",
    "Scanning": "Scanning",
@@ -207,6 +212,7 @@
    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.",
    "Yes": "Yes",
    "You must keep at least one version.": "You must keep at least one version.",
+   "days": "days",
    "full documentation": "full documentation",
    "items": "items",
    "{%device%} wants to share folder \"{%folder%}\".": "{{device}} wants to share folder \"{{folder}}\"."

gui/index.html

@@ -190,18 +190,25 @@
       <!-- Folder list (top left) -->
 
       <div class="col-md-6">
+        <h3 translate>Folders</h3>
         <div class="panel-group" id="folders">
-        <div class="visible-xs"><h3><span translate>Folders</span></h3><hr></div>
           <div class="panel panel-default" ng-repeat="folder in folderList()">
             <div class="panel-heading" data-toggle="collapse" data-parent="#folders" href="#folder-{{$index}}" style="cursor: pointer">
               <div class="panel-progress" ng-show="folderStatus(folder) == 'syncing'" ng-attr-style="width: {{syncPercentage(folder.id)}}%"></div>
+              <div class="panel-progress" ng-show="folderStatus(folder) == 'scanning' && scanProgress[folder.id] != undefined" ng-attr-style="width: {{scanPercentage(folder.id)}}%"></div>
               <h3 class="panel-title">
                 <span class="fa fa-folder hidden-xs"></span>{{folder.id}}
                 <span class="pull-right text-{{folderClass(folder)}}" ng-switch="folderStatus(folder)">
                   <span ng-switch-when="unknown"><span class="hidden-xs" translate>Unknown</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="unshared"><span class="hidden-xs" translate>Unshared</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="stopped"><span class="hidden-xs" translate>Stopped</span><span class="visible-xs">&#9724;</span></span>
-                  <span ng-switch-when="scanning"><span class="hidden-xs" translate>Scanning</span><span class="visible-xs">&#9724;</span></span>
+                  <span ng-switch-when="scanning">
+                    <span class="hidden-xs" translate>Scanning</span>
+                    <span class="hidden-xs" ng-if="scanPercentage(folder.id) != undefined">
+                        ({{scanPercentage(folder.id)}}%)
+                    </span>
+                    <span class="visible-xs">&#9724;</span>
+                  </span>
                   <span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="syncing">
                     <span class="hidden-xs" translate>Syncing</span>
@@ -341,7 +348,7 @@
       <!-- This device -->
 
       <div class="col-md-6">
-      <div class="visible-xs"><h3><span translate>Devices</span></h3><hr></div>
+        <h3 translate>Devices</h3>
         <div class="panel panel-default" ng-repeat="deviceCfg in [thisDevice()]">
           <div class="panel-heading" data-toggle="collapse" href="#device-this" style="cursor: pointer">
             <h3 class="panel-title">
@@ -372,15 +379,28 @@
                     <th><span class="fa fa-fw fa-tachometer"></span>&nbsp;<span translate>CPU Utilization</span></th>
                     <td class="text-right">{{system.cpuPercent | alwaysNumber | natural:1}}%</td>
                   </tr>
-                  <tr ng-if="system.extAnnounceOK != undefined && announceServersTotal > 0">
-                    <th><span class="fa fa-fw fa-bullhorn"></span>&nbsp;<span translate>Global Discovery</span></th>
+                  <tr ng-if="system.discoveryEnabled">
+                    <th><span class="fa fa-fw fa-map-signs"></span>&nbsp;<span translate>Discovery</span></th>
                     <td class="text-right">
-                      <span ng-if="announceServersFailed.length == 0" class="data text-success">
-                        <span>OK</span>
+                      <span ng-if="discoveryFailed.length == 0" class="data text-success">
+                        <span>{{discoveryTotal}}/{{discoveryTotal}}</span>
                       </span>
-                      <span ng-if="announceServersFailed.length != 0" class="data" ng-class="{'text-danger': announceServersFailed.length == announceServersTotal}">
-                        <span popover data-trigger="hover" data-placement="bottom" data-content="{{announceServersFailed.join('\n')}}">
-                          {{announceServersTotal-announceServersFailed.length}}/{{announceServersTotal}}
+                      <span ng-if="discoveryFailed.length != 0" class="data" ng-class="{'text-danger': discoveryFailed.length == discoveryTotal}">
+                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{discoveryFailed.join('<br>\n')}}">
+                          {{discoveryTotal-discoveryFailed.length}}/{{discoveryTotal}}
+                        </span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr ng-if="system.relaysEnabled">
+                    <th><span class="fa fa-fw fa-sitemap"></span>&nbsp;<span translate>Relays</span></th>
+                    <td class="text-right">
+                      <span ng-if="relaysFailed.length == 0" class="data text-success">
+                        <span>{{relaysTotal}}/{{relaysTotal}}</span>
+                      </span>
+                      <span ng-if="relaysFailed.length != 0" class="data" ng-class="{'text-danger': relaysFailed.length == relaysTotal}">
+                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{relaysFailed.join('<br>\n')}}">
+                          {{relaysTotal-relaysFailed.length}}/{{relaysTotal}}
                         </span>
                       </span>
                     </td>
@@ -412,6 +432,7 @@
                   <span ng-switch-when="syncing">
                     <span class="hidden-xs" translate>Syncing</span> ({{completion[deviceCfg.deviceID]._total | number:0}}%)
                   </span>
+                  <span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="disconnected"><span class="hidden-xs" translate>Disconnected</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="unused"><span class="hidden-xs" translate>Unused</span><span class="visible-xs">&#9724;</span></span>
                 </span>
@@ -421,16 +442,20 @@
               <div class="panel-body">
                 <table class="table table-condensed table-striped">
                   <tbody>
-                    <tr ng-if="connections[deviceCfg.deviceID]">
+                    <tr ng-if="connections[deviceCfg.deviceID].connected">
                       <th><span class="fa fa-fw fa-cloud-download"></span>&nbsp;<span translate>Download Rate</span></th>
                       <td class="text-right">{{connections[deviceCfg.deviceID].inbps | binary}}B/s ({{connections[deviceCfg.deviceID].inBytesTotal | binary}}B)</td>
                     </tr>
-                    <tr ng-if="connections[deviceCfg.deviceID]">
+                    <tr ng-if="connections[deviceCfg.deviceID].connected">
                       <th><span class="fa fa-fw fa-cloud-upload"></span>&nbsp;<span translate>Upload Rate</span></th>
                       <td class="text-right">{{connections[deviceCfg.deviceID].outbps | binary}}B/s ({{connections[deviceCfg.deviceID].outBytesTotal | binary}}B)</td>
                     </tr>
                     <tr>
-                      <th><span class="fa fa-fw fa-link"></span>&nbsp;<span translate>Address</span></th>
+                      <th>
+                        <span class="fa fa-fw fa-link"></span>
+                        <span translate ng-if="connections[deviceCfg.deviceID].type.indexOf('direct') == 0" >Address</span>
+                        <span translate ng-if="connections[deviceCfg.deviceID].type.indexOf('relay') == 0" >Relayed via</span>
+                      </th>
                       <td class="text-right">{{deviceAddr(deviceCfg)}}</td>
                     </tr>
                     <tr ng-if="deviceCfg.compression != 'metadata'">
@@ -444,11 +469,11 @@
                       <th><span class="fa fa-fw fa-thumbs-o-up"></span>&nbsp;<span translate>Introducer</span></th>
                       <td translate class="text-right">Yes</td>
                     </tr>
-                    <tr ng-if="connections[deviceCfg.deviceID]">
+                    <tr ng-if="connections[deviceCfg.deviceID].version">
                       <th><span class="fa fa-fw fa-tag"></span>&nbsp;<span translate>Version</span></th>
                       <td class="text-right">{{connections[deviceCfg.deviceID].clientVersion}}</td>
                     </tr>
-                    <tr ng-if="!connections[deviceCfg.deviceID]">
+                    <tr ng-if="!connections[deviceCfg.deviceID].connected">
                       <th><span class="fa fa-fw fa-eye"></span>&nbsp;<span translate>Last seen</span></th>
                       <td translate ng-if="!deviceStats[deviceCfg.deviceID].lastSeenDays || deviceStats[deviceCfg.deviceID].lastSeenDays >= 365" class="text-right">Never</td>
                       <td ng-if="deviceStats[deviceCfg.deviceID].lastSeenDays < 365" class="text-right">{{deviceStats[deviceCfg.deviceID].lastSeen | date:"yyyy-MM-dd HH:mm:ss"}}</td>
@@ -465,6 +490,12 @@
                   <button type="button" class="btn btn-sm btn-default" ng-click="editDevice(deviceCfg)">
                     <span class="fa fa-pencil"></span>&nbsp;<span translate>Edit</span>
                   </button>
+                  <button ng-if="!connections[deviceCfg.deviceID].paused" type="button" class="btn btn-sm btn-default" ng-click="pauseDevice(deviceCfg.deviceID)">
+                    <span class="fa fa-pause"></span>&nbsp;<span translate>Pause</span>
+                  </button>
+                  <button ng-if="connections[deviceCfg.deviceID].paused" type="button" class="btn btn-sm btn-default" ng-click="resumeDevice(deviceCfg.deviceID)">
+                    <span class="fa fa-play"></span>&nbsp;<span translate>Resume</span>
+                  </button>
                 </span>
                 <div class="clearfix"></div>
               </div>

gui/syncthing/core/eventService.js

@@ -63,6 +63,8 @@ angular.module('syncthing.core')
             DEVICE_DISCONNECTED:  'DeviceDisconnected',   // Generated each time a connection to a device has been terminated
             DEVICE_DISCOVERED:    'DeviceDiscovered',   // Emitted when a new device is discovered using local discovery
             DEVICE_REJECTED:      'DeviceRejected',   // Emitted when there is a connection from a device we are not configured to talk to
+            DEVICE_PAUSED:        'DevicePaused',   // Emitted when a device has been paused
+            DEVICE_RESUMED:       'DeviceResumed',   // Emitted when a device has been resumed
             DOWNLOAD_PROGRESS:    'DownloadProgress',   // Emitted during file downloads for each folder for each file
             FOLDER_COMPLETION:    'FolderCompletion',   //Emitted when the local or remote contents for a folder changes
             FOLDER_REJECTED:      'FolderRejected',   // Emitted when a device sends index information for a folder we do not have, or have but do not share with the device in question
@@ -76,6 +78,7 @@ angular.module('syncthing.core')
             STARTUP_COMPLETED:    'StartupCompleted',   // Emitted exactly once, when initialization is complete and Syncthing is ready to start exchanging data with other devices
             STATE_CHANGED:        'StateChanged',   // Emitted when a folder changes state
             FOLDER_ERRORS:        'FolderErrors',   // Emitted when a folder has errors preventing a full sync
+            FOLDER_SCAN_PROGRESS: 'FolderScanProgress',   // Emitted every ScanProgressIntervalS seconds, indicating how far into the scan it is at.
 
             start: function() {
                 $http.get(urlbase + '/events?limit=1')

gui/syncthing/core/syncthingController.js

@@ -48,6 +48,7 @@ angular.module('syncthing.core')
         $scope.failedCurrentPage = 1;
         $scope.failedCurrentFolder = undefined;
         $scope.failedPageSize = 10;
+        $scope.scanProgress = {};
 
         $scope.localStateTotal = {
             bytes: 0,
@@ -163,6 +164,12 @@ angular.module('syncthing.core')
                 if (data.to === 'syncing') {
                     $scope.failed[data.folder] = [];
                 }
+
+                // If a folder has started scanning, then any scan progress is
+                // also obsolete.
+                if (data.to === 'scanning') {
+                    delete $scope.scanProgress[data.folder];
+                }
             }
         });
 
@@ -171,7 +178,7 @@ angular.module('syncthing.core')
         });
 
         $scope.$on(Events.DEVICE_DISCONNECTED, function (event, arg) {
-            delete $scope.connections[arg.data.id];
+            $scope.connections[arg.data.id].connected = false;
             refreshDeviceStats();
         });
 
@@ -182,6 +189,7 @@ angular.module('syncthing.core')
                     outbps: 0,
                     inBytesTotal: 0,
                     outBytesTotal: 0,
+                    type: arg.data.type,
                     address: arg.data.addr
                 };
                 $scope.completion[arg.data.id] = {
@@ -214,6 +222,14 @@ angular.module('syncthing.core')
             $scope.deviceRejections[arg.data.device] = arg;
         });
 
+        $scope.$on(Events.DEVICE_PAUSED, function (event, arg) {
+            $scope.connections[arg.data.device].paused = true;
+        });
+
+        $scope.$on(Events.DEVICE_RESUMED, function (event, arg) {
+            $scope.connections[arg.data.device].paused = false;
+        });
+
         $scope.$on(Events.FOLDER_REJECTED, function (event, arg) {
             $scope.folderRejections[arg.data.folder + "-" + arg.data.device] = arg;
         });
@@ -301,6 +317,15 @@ angular.module('syncthing.core')
             $scope.failed[data.folder] = data.errors;
         });
 
+        $scope.$on(Events.FOLDER_SCAN_PROGRESS, function (event, arg) {
+            var data = arg.data;
+            $scope.scanProgress[data.folder] = {
+                current: data.current,
+                total: data.total
+            };
+            console.log("FolderScanProgress", data);
+        });
+
         $scope.emitHTTPError = function (data, status, headers, config) {
             $scope.$emit('HTTPError', {data: data, status: status, headers: headers, config: config});
         };
@@ -352,15 +377,26 @@ angular.module('syncthing.core')
             $http.get(urlbase + '/system/status').success(function (data) {
                 $scope.myID = data.myID;
                 $scope.system = data;
-                $scope.announceServersTotal = data.extAnnounceOK ? Object.keys(data.extAnnounceOK).length : 0;
-                var failed = [];
-                for (var server in data.extAnnounceOK) {
-                    if (!data.extAnnounceOK[server]) {
-                        failed.push(server);
+
+                $scope.discoveryTotal = data.discoveryMethods;
+                var discoveryFailed = [];
+                for (var disco in data.discoveryErrors) {
+                    if (data.discoveryErrors[disco]) {
+                        discoveryFailed.push(disco + ": " + data.discoveryErrors[disco]);
                     }
                 }
-                $scope.announceServersFailed = failed;
+                $scope.discoveryFailed = discoveryFailed;
 
+                var relaysFailed = [];
+                var relaysTotal = 0;
+                for (var relay in data.relayClientStatus) {
+                    if (!data.relayClientStatus[relay]) {
+                        relaysFailed.push(relay);
+                    }
+                    relaysTotal++;
+                }
+                $scope.relaysFailed = relaysFailed;
+                $scope.relaysTotal = relaysTotal;
 
                 console.log("refreshSystem", data);
             }).error($scope.emitHTTPError);
@@ -615,12 +651,28 @@ angular.module('syncthing.core')
             return Math.floor(pct);
         };
 
+        $scope.scanPercentage = function (folder) {
+            if (!$scope.scanProgress[folder]) {
+                return undefined;
+            }
+            var pct = 100 * $scope.scanProgress[folder].current / $scope.scanProgress[folder].total;
+            return Math.floor(pct);
+        }
+
         $scope.deviceStatus = function (deviceCfg) {
             if ($scope.deviceFolders(deviceCfg).length === 0) {
                 return 'unused';
             }
 
-            if ($scope.connections[deviceCfg.deviceID]) {
+            if (typeof $scope.connections[deviceCfg.deviceID] === 'undefined') {
+                return 'unknown';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].paused) {
+                return 'paused';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].connected) {
                 if ($scope.completion[deviceCfg.deviceID] && $scope.completion[deviceCfg.deviceID]._total === 100) {
                     return 'insync';
                 } else {
@@ -638,7 +690,15 @@ angular.module('syncthing.core')
                 return 'warning';
             }
 
-            if ($scope.connections[deviceCfg.deviceID]) {
+            if (typeof $scope.connections[deviceCfg.deviceID] === 'undefined') {
+                return 'info';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].paused) {
+                return 'default';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].connected) {
                 if ($scope.completion[deviceCfg.deviceID] && $scope.completion[deviceCfg.deviceID]._total === 100) {
                     return 'success';
                 } else {
@@ -652,7 +712,7 @@ angular.module('syncthing.core')
 
         $scope.deviceAddr = function (deviceCfg) {
             var conn = $scope.connections[deviceCfg.deviceID];
-            if (conn) {
+            if (conn && conn.connected) {
                 return conn.address;
             }
             return '?';
@@ -697,6 +757,14 @@ angular.module('syncthing.core')
             return device.deviceID.substr(0, 6);
         };
 
+        $scope.pauseDevice = function (device) {
+            $http.post(urlbase + "/system/pause?device=" + device);
+        };
+
+        $scope.resumeDevice = function (device) {
+            $http.post(urlbase + "/system/resume?device=" + device);
+        };
+
         $scope.editSettings = function () {
             // Make a working copy
             $scope.tmpOptions = angular.copy($scope.config.options);

gui/syncthing/device/editDeviceModalView.html

@@ -13,7 +13,7 @@
             <label translate for="deviceID">Device ID</label>
             <input ng-if="!editingExisting" name="deviceID" id="deviceID" class="form-control text-monospace" type="text" ng-model="currentDevice.deviceID" required valid-deviceid list="discovery-list" />
             <datalist id="discovery-list" ng-if="!editingExisting">
-              <option ng-repeat="(id,address) in discovery" value="{{ id }}" />
+              <option ng-repeat="(id, data) in discovery" value="{{id}}" />
             </datalist>
             <div ng-if="editingExisting" class="well well-sm text-monospace">{{currentDevice.deviceID}}</div>
             <p class="help-block">
@@ -32,7 +32,7 @@
           <div class="form-group">
             <label translate for="addresses">Addresses</label>
             <input ng-disabled="currentDevice.deviceID == myID" id="addresses" class="form-control" type="text" ng-model="currentDevice._addressesStr"></input>
-            <p translate class="help-block">Enter comma separated "ip:port" addresses or "dynamic" to perform automatic discovery of the address.</p>
+            <p translate class="help-block">Enter comma separated  ("tcp://ip:port", "tcp://host:port") addresses or "dynamic" to perform automatic discovery of the address.</p>
           </div>
           <div class="form-group">
             <label translate>Compression</label>

gui/syncthing/folder/editFolderModalView.html

@@ -98,7 +98,7 @@
                 <label translate for="trashcanClean">Clean out after</label>
                 <div class="input-group">
                   <input name="trashcanClean" id="trashcanClean" class="form-control text-right" type="number" ng-model="currentFolder.trashcanClean" required min="0"></input>
-                  <div class="input-group-addon">days</div>
+                  <div class="input-group-addon" translate>days</div>
                 </div>
                 <p class="help-block">
                   <span translate ng-if="folderEditor.trashcanClean.$valid || folderEditor.trashcanClean.$pristine">The number of days to keep files in the trash can. Zero means forever.</span>

gui/syncthing/usagereport/usageReportPreviewModalView.html

@@ -1,5 +1,7 @@
 <modal id="urPreview" status="success" icon="bar-chart" title="{{'Anonymous Usage Reporting' | translate}}" large="yes" close="yes" tabindex="-1">
   <p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
   <p translate translate-value-url="<a href=&quot;https://data.syncthing.net&quot; target=&quot;_blank&quot;>https://data.syncthing.net</a>">The aggregated statistics are publicly available at {%url%}.</p>
-  <pre><small>{{reportData | json}}</small></pre>
+  <form>
+  <textarea class="form-control" rows="20">{{reportData | json}}</textarea>
+  </form>
 </modal>

lib/beacon/beacon.go

@@ -6,7 +6,12 @@
 
 package beacon
 
-import "net"
+import (
+	"net"
+	stdsync "sync"
+
+	"github.com/thejerf/suture"
+)
 
 type recv struct {
 	data []byte
@@ -14,34 +19,30 @@ type recv struct {
 }
 
 type Interface interface {
+	suture.Service
 	Send(data []byte)
 	Recv() ([]byte, net.Addr)
+	Error() error
 }
 
 type readerFrom interface {
 	ReadFrom([]byte) (int, net.Addr, error)
 }
 
-func genericReader(conn readerFrom, outbox chan<- recv) {
-	bs := make([]byte, 65536)
-	for {
-		n, addr, err := conn.ReadFrom(bs)
-		if err != nil {
-			l.Warnln("multicast read:", err)
-			return
-		}
-		if debug {
-			l.Debugf("recv %d bytes from %s", n, addr)
-		}
+type errorHolder struct {
+	err error
+	mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
+}
 
-		c := make([]byte, n)
-		copy(c, bs)
-		select {
-		case outbox <- recv{c, addr}:
-		default:
-			if debug {
-				l.Debugln("dropping message")
-			}
-		}
-	}
+func (e *errorHolder) setError(err error) {
+	e.mut.Lock()
+	e.err = err
+	e.mut.Unlock()
+}
+
+func (e *errorHolder) Error() error {
+	e.mut.Lock()
+	err := e.err
+	e.mut.Unlock()
+	return err
 }

lib/beacon/broadcast.go

@@ -19,6 +19,8 @@ type Broadcast struct {
 	port   int
 	inbox  chan []byte
 	outbox chan recv
+	br     *broadcastReader
+	bw     *broadcastWriter
 }
 
 func NewBroadcast(port int) *Broadcast {
@@ -41,14 +43,16 @@ func NewBroadcast(port int) *Broadcast {
 		outbox: make(chan recv, 16),
 	}
 
-	b.Add(&broadcastReader{
+	b.br = &broadcastReader{
 		port:   port,
 		outbox: b.outbox,
-	})
-	b.Add(&broadcastWriter{
+	}
+	b.Add(b.br)
+	b.bw = &broadcastWriter{
 		port:  port,
 		inbox: b.inbox,
-	})
+	}
+	b.Add(b.bw)
 
 	return b
 }
@@ -62,11 +66,18 @@ func (b *Broadcast) Recv() ([]byte, net.Addr) {
 	return recv.data, recv.src
 }
 
+func (b *Broadcast) Error() error {
+	if err := b.br.Error(); err != nil {
+		return err
+	}
+	return b.bw.Error()
+}
+
 type broadcastWriter struct {
-	port   int
-	inbox  chan []byte
-	conn   *net.UDPConn
-	failed bool // Have we already logged a failure reason?
+	port  int
+	inbox chan []byte
+	conn  *net.UDPConn
+	errorHolder
 }
 
 func (w *broadcastWriter) Serve() {
@@ -78,22 +89,21 @@ func (w *broadcastWriter) Serve() {
 	var err error
 	w.conn, err = net.ListenUDP("udp4", nil)
 	if err != nil {
-		if !w.failed {
-			l.Warnln("Local discovery over IPv4 unavailable:", err)
-			w.failed = true
+		if debug {
+			l.Debugln(err)
 		}
+		w.setError(err)
 		return
 	}
 	defer w.conn.Close()
 
-	w.failed = false
-
 	for bs := range w.inbox {
 		addrs, err := net.InterfaceAddrs()
 		if err != nil {
 			if debug {
-				l.Debugln("Local discovery (broadcast writer):", err)
+				l.Debugln(err)
 			}
+			w.setError(err)
 			continue
 		}
 
@@ -117,13 +127,16 @@ func (w *broadcastWriter) Serve() {
 		for _, ip := range dsts {
 			dst := &net.UDPAddr{IP: ip, Port: w.port}
 
-			w.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
+			w.conn.SetWriteDeadline(time.Now().Add(time.Second))
 			_, err := w.conn.WriteTo(bs, dst)
+			w.conn.SetWriteDeadline(time.Time{})
 			if err, ok := err.(net.Error); ok && err.Timeout() {
 				// Write timeouts should not happen. We treat it as a fatal
 				// error on the socket.
-				l.Infoln("Local discovery (broadcast writer):", err)
-				w.failed = true
+				if debug {
+					l.Debugln(err)
+				}
+				w.setError(err)
 				return
 			} else if err, ok := err.(net.Error); ok && err.Temporary() {
 				// A transient error. Lets hope for better luck in the future.
@@ -133,11 +146,14 @@ func (w *broadcastWriter) Serve() {
 				continue
 			} else if err != nil {
 				// Some other error that we don't expect. Bail and retry.
-				l.Infoln("Local discovery (broadcast writer):", err)
-				w.failed = true
+				if debug {
+					l.Debugln(err)
+				}
+				w.setError(err)
 				return
 			} else if debug {
 				l.Debugf("sent %d bytes to %s", len(bs), dst)
+				w.setError(nil)
 			}
 		}
 	}
@@ -155,7 +171,7 @@ type broadcastReader struct {
 	port   int
 	outbox chan recv
 	conn   *net.UDPConn
-	failed bool
+	errorHolder
 }
 
 func (r *broadcastReader) Serve() {
@@ -167,10 +183,10 @@ func (r *broadcastReader) Serve() {
 	var err error
 	r.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
 	if err != nil {
-		if !r.failed {
-			l.Warnln("Local discovery over IPv4 unavailable:", err)
-			r.failed = true
+		if debug {
+			l.Debugln(err)
 		}
+		r.setError(err)
 		return
 	}
 	defer r.conn.Close()
@@ -179,14 +195,14 @@ func (r *broadcastReader) Serve() {
 	for {
 		n, addr, err := r.conn.ReadFrom(bs)
 		if err != nil {
-			if !r.failed {
-				l.Infoln("Local discovery (broadcast reader):", err)
-				r.failed = true
+			if debug {
+				l.Debugln(err)
 			}
+			r.setError(err)
 			return
 		}
 
-		r.failed = false
+		r.setError(nil)
 
 		if debug {
 			l.Debugf("recv %d bytes from %s", n, addr)

lib/beacon/multicast.go

@@ -8,39 +8,200 @@ package beacon
 
 import (
 	"errors"
+	"fmt"
 	"net"
+	"time"
 
+	"github.com/thejerf/suture"
 	"golang.org/x/net/ipv6"
 )
 
 type Multicast struct {
-	conn   *ipv6.PacketConn
+	*suture.Supervisor
 	addr   *net.UDPAddr
 	inbox  chan []byte
 	outbox chan recv
-	intfs  []net.Interface
+	mr     *multicastReader
+	mw     *multicastWriter
 }
 
-func NewMulticast(addr string) (*Multicast, error) {
-	gaddr, err := net.ResolveUDPAddr("udp6", addr)
-	if err != nil {
-		return nil, err
+func NewMulticast(addr string) *Multicast {
+	m := &Multicast{
+		Supervisor: suture.New("multicastBeacon", suture.Spec{
+			// Don't retry too frenetically: an error to open a socket or
+			// whatever is usually something that is either permanent or takes
+			// a while to get solved...
+			FailureThreshold: 2,
+			FailureBackoff:   60 * time.Second,
+			// Only log restarts in debug mode.
+			Log: func(line string) {
+				if debug {
+					l.Debugln(line)
+				}
+			},
+		}),
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
 	}
 
-	conn, err := net.ListenPacket("udp6", addr)
+	m.mr = &multicastReader{
+		addr:   addr,
+		outbox: m.outbox,
+		stop:   make(chan struct{}),
+	}
+	m.Add(m.mr)
+
+	m.mw = &multicastWriter{
+		addr:  addr,
+		inbox: m.inbox,
+		stop:  make(chan struct{}),
+	}
+	m.Add(m.mw)
+
+	return m
+}
+
+func (m *Multicast) Send(data []byte) {
+	m.inbox <- data
+}
+
+func (m *Multicast) Recv() ([]byte, net.Addr) {
+	recv := <-m.outbox
+	return recv.data, recv.src
+}
+
+func (m *Multicast) Error() error {
+	if err := m.mr.Error(); err != nil {
+		return err
+	}
+	return m.mw.Error()
+}
+
+type multicastWriter struct {
+	addr  string
+	inbox <-chan []byte
+	errorHolder
+	stop chan struct{}
+}
+
+func (w *multicastWriter) Serve() {
+	if debug {
+		l.Debugln(w, "starting")
+		defer l.Debugln(w, "stopping")
+	}
+
+	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
 	if err != nil {
-		return nil, err
+		if debug {
+			l.Debugln(err)
+		}
+		w.setError(err)
+		return
+	}
+
+	conn, err := net.ListenPacket("udp6", ":0")
+	if err != nil {
+		if debug {
+			l.Debugln(err)
+		}
+		w.setError(err)
+		return
+	}
+
+	pconn := ipv6.NewPacketConn(conn)
+
+	wcm := &ipv6.ControlMessage{
+		HopLimit: 1,
+	}
+
+	for bs := range w.inbox {
+		intfs, err := net.Interfaces()
+		if err != nil {
+			if debug {
+				l.Debugln(err)
+			}
+			w.setError(err)
+			return
+		}
+
+		var success int
+
+		for _, intf := range intfs {
+			wcm.IfIndex = intf.Index
+			pconn.SetWriteDeadline(time.Now().Add(time.Second))
+			_, err = pconn.WriteTo(bs, wcm, gaddr)
+			pconn.SetWriteDeadline(time.Time{})
+			if err != nil && debug {
+				l.Debugln(err, "on write to", gaddr, intf.Name)
+			} else if debug {
+				l.Debugf("sent %d bytes to %v on %s", len(bs), gaddr, intf.Name)
+				success++
+			}
+		}
+
+		if success > 0 {
+			w.setError(nil)
+		} else {
+			if debug {
+				l.Debugln(err)
+			}
+			w.setError(err)
+		}
+	}
+}
+
+func (w *multicastWriter) Stop() {
+	close(w.stop)
+}
+
+func (w *multicastWriter) String() string {
+	return fmt.Sprintf("multicastWriter@%p", w)
+}
+
+type multicastReader struct {
+	addr   string
+	outbox chan<- recv
+	errorHolder
+	stop chan struct{}
+}
+
+func (r *multicastReader) Serve() {
+	if debug {
+		l.Debugln(r, "starting")
+		defer l.Debugln(r, "stopping")
+	}
+
+	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
+	if err != nil {
+		if debug {
+			l.Debugln(err)
+		}
+		r.setError(err)
+		return
+	}
+
+	conn, err := net.ListenPacket("udp6", r.addr)
+	if err != nil {
+		if debug {
+			l.Debugln(err)
+		}
+		r.setError(err)
+		return
 	}
 
 	intfs, err := net.Interfaces()
 	if err != nil {
-		return nil, err
+		if debug {
+			l.Debugln(err)
+		}
+		r.setError(err)
+		return
 	}
 
-	p := ipv6.NewPacketConn(conn)
+	pconn := ipv6.NewPacketConn(conn)
 	joined := 0
 	for _, intf := range intfs {
-		err := p.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
+		err := pconn.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
 		if debug {
 			if err != nil {
 				l.Debugln("IPv6 join", intf.Name, "failed:", err)
@@ -52,57 +213,43 @@ func NewMulticast(addr string) (*Multicast, error) {
 	}
 
 	if joined == 0 {
-		return nil, errors.New("no multicast interfaces available")
+		if debug {
+			l.Debugln("no multicast interfaces available")
+		}
+		r.setError(errors.New("no multicast interfaces available"))
+		return
 	}
 
-	b := &Multicast{
-		conn:   p,
-		addr:   gaddr,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-		intfs:  intfs,
-	}
+	bs := make([]byte, 65536)
+	for {
+		n, _, addr, err := pconn.ReadFrom(bs)
+		if err != nil {
+			if debug {
+				l.Debugln(err)
+			}
+			r.setError(err)
+			continue
+		}
+		if debug {
+			l.Debugf("recv %d bytes from %s", n, addr)
+		}
 
-	go genericReader(ipv6ReaderAdapter{b.conn}, b.outbox)
-	go b.writer()
-
-	return b, nil
-}
-
-func (b *Multicast) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Multicast) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Multicast) writer() {
-	wcm := &ipv6.ControlMessage{
-		HopLimit: 1,
-	}
-
-	for bs := range b.inbox {
-		for _, intf := range b.intfs {
-			wcm.IfIndex = intf.Index
-			_, err := b.conn.WriteTo(bs, wcm, b.addr)
-			if err != nil && debug {
-				l.Debugln(err, "on write to", b.addr)
-			} else if debug {
-				l.Debugf("sent %d bytes to %v on %s", len(bs), b.addr, intf.Name)
+		c := make([]byte, n)
+		copy(c, bs)
+		select {
+		case r.outbox <- recv{c, addr}:
+		default:
+			if debug {
+				l.Debugln("dropping message")
 			}
 		}
 	}
 }
 
-// This makes ReadFrom on an *ipv6.PacketConn behave like ReadFrom on a
-// net.PacketConn.
-type ipv6ReaderAdapter struct {
-	c *ipv6.PacketConn
+func (r *multicastReader) Stop() {
+	close(r.stop)
 }
 
-func (i ipv6ReaderAdapter) ReadFrom(bs []byte) (int, net.Addr, error) {
-	n, _, src, err := i.c.ReadFrom(bs)
-	return n, src, err
+func (r *multicastReader) String() string {
+	return fmt.Sprintf("multicastReader@%p", r)
 }

lib/config/config.go

@@ -9,6 +9,7 @@ package config
 
 import (
 	"encoding/xml"
+	"fmt"
 	"io"
 	"math/rand"
 	"os"
@@ -26,10 +27,36 @@ import (
 
 const (
 	OldestHandledVersion = 5
-	CurrentVersion       = 11
+	CurrentVersion       = 12
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
+var (
+	// DefaultDiscoveryServers should be substituted when the configuration
+	// contains <globalAnnounceServer>default</globalAnnounceServer>. This is
+	// done by the "consumer" of the configuration, as we don't want these
+	// saved to the config.
+	DefaultDiscoveryServers = []string{
+		"https://discovery-v4-1.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 194.126.249.5, Sweden
+		"https://discovery-v4-2.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 45.55.230.38, USA
+		"https://discovery-v4-3.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 128.199.95.124, Singapore
+		"https://discovery-v6-1.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 2001:470:28:4d6::5, Sweden
+		"https://discovery-v6-2.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 2604:a880:800:10::182:a001, USA
+		"https://discovery-v6-3.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 2400:6180:0:d0::d9:d001, Singapore
+	}
+
+	// DefaultDiscoveryServersIP is used by the usage reporting.
+	// XXX: Detect Android, and use this is we still don't have working DNS?
+	DefaultDiscoveryServersIP = []string{
+		"https://194.126.249.5/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA",
+		"https://45.55.230.38/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS",
+		"https://128.199.95.124/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4",
+		"https://[2001:470:28:4d6::5]/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA",
+		"https://[2604:a880:800:10::182:a001]/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS",
+		"https://[2400:6180:0:d0::d9:d001]/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4",
+	}
+)
+
 type Configuration struct {
 	Version        int                   `xml:"version,attr" json:"version"`
 	Folders        []FolderConfiguration `xml:"folder" json:"folders"`
@@ -67,20 +94,21 @@ func (cfg Configuration) Copy() Configuration {
 }
 
 type FolderConfiguration struct {
-	ID              string                      `xml:"id,attr" json:"id"`
-	RawPath         string                      `xml:"path,attr" json:"path"`
-	Devices         []FolderDeviceConfiguration `xml:"device" json:"devices"`
-	ReadOnly        bool                        `xml:"ro,attr" json:"readOnly"`
-	RescanIntervalS int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
-	IgnorePerms     bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
-	AutoNormalize   bool                        `xml:"autoNormalize,attr" json:"autoNormalize"`
-	MinDiskFreePct  float64                     `xml:"minDiskFreePct" json:"minDiskFreePct"`
-	Versioning      VersioningConfiguration     `xml:"versioning" json:"versioning"`
-	Copiers         int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
-	Pullers         int                         `xml:"pullers" json:"pullers"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
-	Hashers         int                         `xml:"hashers" json:"hashers"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
-	Order           PullOrder                   `xml:"order" json:"order"`
-	IgnoreDelete    bool                        `xml:"ignoreDelete" json:"ignoreDelete"`
+	ID                    string                      `xml:"id,attr" json:"id"`
+	RawPath               string                      `xml:"path,attr" json:"path"`
+	Devices               []FolderDeviceConfiguration `xml:"device" json:"devices"`
+	ReadOnly              bool                        `xml:"ro,attr" json:"readOnly"`
+	RescanIntervalS       int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
+	IgnorePerms           bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
+	AutoNormalize         bool                        `xml:"autoNormalize,attr" json:"autoNormalize"`
+	MinDiskFreePct        float64                     `xml:"minDiskFreePct" json:"minDiskFreePct"`
+	Versioning            VersioningConfiguration     `xml:"versioning" json:"versioning"`
+	Copiers               int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
+	Pullers               int                         `xml:"pullers" json:"pullers"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
+	Hashers               int                         `xml:"hashers" json:"hashers"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
+	Order                 PullOrder                   `xml:"order" json:"order"`
+	IgnoreDelete          bool                        `xml:"ignoreDelete" json:"ignoreDelete"`
+	ScanProgressIntervalS int                         `xml:"scanProgressInterval" json:"scanProgressInterval"` // Set to a negative value to disable. Value of 0 will get replaced with value of 2 (default value)
 
 	Invalid string `xml:"-" json:"invalid"` // Set at runtime when there is an error, not saved
 }
@@ -212,15 +240,19 @@ type FolderDeviceConfiguration struct {
 }
 
 type OptionsConfiguration struct {
-	ListenAddress           []string `xml:"listenAddress" json:"listenAddress" default:"0.0.0.0:22000"`
-	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"udp4://announce.syncthing.net:22026, udp6://announce-v6.syncthing.net:22026"`
+	ListenAddress           []string `xml:"listenAddress" json:"listenAddress" default:"tcp://0.0.0.0:22000"`
+	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default"`
 	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true"`
 	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true"`
-	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21025"`
-	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff32::5222]:21026"`
+	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027"`
+	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027"`
+	RelayServers            []string `xml:"relayServer" json:"relayServers" default:"dynamic+https://relays.syncthing.net"`
 	MaxSendKbps             int      `xml:"maxSendKbps" json:"maxSendKbps"`
 	MaxRecvKbps             int      `xml:"maxRecvKbps" json:"maxRecvKbps"`
 	ReconnectIntervalS      int      `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
+	RelaysEnabled           bool     `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
+	RelayReconnectIntervalM int      `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
+	RelayWithoutGlobalAnn   bool     `xml:"relayWithoutGlobalAnn" json:"relayWithoutGlobalAnn" default:"false"`
 	StartBrowser            bool     `xml:"startBrowser" json:"startBrowser" default:"true"`
 	UPnPEnabled             bool     `xml:"upnpEnabled" json:"upnpEnabled" default:"true"`
 	UPnPLeaseM              int      `xml:"upnpLeaseMinutes" json:"upnpLeaseMinutes" default:"60"`
@@ -228,6 +260,9 @@ type OptionsConfiguration struct {
 	UPnPTimeoutS            int      `xml:"upnpTimeoutSeconds" json:"upnpTimeoutSeconds" default:"10"`
 	URAccepted              int      `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
 	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
+	URURL                   string   `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
+	URPostInsecurely        bool     `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
+	URInitialDelayS         int      `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
 	RestartOnWakeup         bool     `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true"`
 	AutoUpgradeIntervalH    int      `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12"` // 0 for off
 	KeepTemporariesH        int      `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"`         // 0 for off
@@ -236,9 +271,9 @@ type OptionsConfiguration struct {
 	SymlinksEnabled         bool     `xml:"symlinksEnabled" json:"symlinksEnabled" default:"true"`
 	LimitBandwidthInLan     bool     `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
 	DatabaseBlockCacheMiB   int      `xml:"databaseBlockCacheMiB" json:"databaseBlockCacheMiB" default:"0"`
-	PingTimeoutS            int      `xml:"pingTimeoutS" json:"pingTimeoutS" default:"30"`
-	PingIdleTimeS           int      `xml:"pingIdleTimeS" json:"pingIdleTimeS" default:"60"`
 	MinHomeDiskFreePct      float64  `xml:"minHomeDiskFreePct" json:"minHomeDiskFreePct" default:"1"`
+	ReleasesURL             string   `xml:"releasesURL" json:"releasesURL" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=30"`
+	AlwaysLocalNets         []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
 }
 
 func (orig OptionsConfiguration) Copy() OptionsConfiguration {
@@ -346,6 +381,9 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		}
 	}
 
+	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
+	cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
+
 	if cfg.Version < OldestHandledVersion {
 		l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version)
 	}
@@ -369,6 +407,9 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 	if cfg.Version == 10 {
 		convertV10V11(cfg)
 	}
+	if cfg.Version == 11 {
+		convertV11V12(cfg)
+	}
 
 	// Hash old cleartext passwords
 	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
@@ -420,9 +461,6 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		cfg.Options.ReconnectIntervalS = 5
 	}
 
-	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
-	cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
-
 	if cfg.GUI.APIKey == "" {
 		cfg.GUI.APIKey = randomString(32)
 	}
@@ -467,6 +505,52 @@ func convertV10V11(cfg *Configuration) {
 	cfg.Version = 11
 }
 
+func convertV11V12(cfg *Configuration) {
+	// Change listen address schema
+	for i, addr := range cfg.Options.ListenAddress {
+		if len(addr) > 0 && !strings.HasPrefix(addr, "tcp://") {
+			cfg.Options.ListenAddress[i] = fmt.Sprintf("tcp://%s", addr)
+		}
+	}
+
+	for i, device := range cfg.Devices {
+		for j, addr := range device.Addresses {
+			if addr != "dynamic" && addr != "" {
+				cfg.Devices[i].Addresses[j] = fmt.Sprintf("tcp://%s", addr)
+			}
+		}
+	}
+
+	// Use new discovery server
+	var newDiscoServers []string
+	var useDefault bool
+	for _, addr := range cfg.Options.GlobalAnnServers {
+		if addr == "udp4://announce.syncthing.net:22026" {
+			useDefault = true
+		} else if addr == "udp6://announce-v6.syncthing.net:22026" {
+			useDefault = true
+		} else {
+			newDiscoServers = append(newDiscoServers, addr)
+		}
+	}
+	if useDefault {
+		newDiscoServers = append(newDiscoServers, "default")
+	}
+	cfg.Options.GlobalAnnServers = newDiscoServers
+
+	// Use new multicast group
+	if cfg.Options.LocalAnnMCAddr == "[ff32::5222]:21026" {
+		cfg.Options.LocalAnnMCAddr = "[ff12::8384]:21027"
+	}
+
+	// Use new local discovery port
+	if cfg.Options.LocalAnnPort == 21025 {
+		cfg.Options.LocalAnnPort = 21027
+	}
+
+	cfg.Version = 12
+}
+
 func convertV9V10(cfg *Configuration) {
 	// Enable auto normalization on existing folders.
 	for i := range cfg.Folders {

lib/config/config_test.go

@@ -31,15 +31,19 @@ func init() {
 
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddress:           []string{"0.0.0.0:22000"},
-		GlobalAnnServers:        []string{"udp4://announce.syncthing.net:22026", "udp6://announce-v6.syncthing.net:22026"},
+		ListenAddress:           []string{"tcp://0.0.0.0:22000"},
+		GlobalAnnServers:        []string{"default"},
 		GlobalAnnEnabled:        true,
 		LocalAnnEnabled:         true,
-		LocalAnnPort:            21025,
-		LocalAnnMCAddr:          "[ff32::5222]:21026",
+		LocalAnnPort:            21027,
+		LocalAnnMCAddr:          "[ff12::8384]:21027",
+		RelayServers:            []string{"dynamic+https://relays.syncthing.net"},
 		MaxSendKbps:             0,
 		MaxRecvKbps:             0,
 		ReconnectIntervalS:      60,
+		RelaysEnabled:           true,
+		RelayReconnectIntervalM: 10,
+		RelayWithoutGlobalAnn:   false,
 		StartBrowser:            true,
 		UPnPEnabled:             true,
 		UPnPLeaseM:              60,
@@ -53,9 +57,11 @@ func TestDefaultValues(t *testing.T) {
 		SymlinksEnabled:         true,
 		LimitBandwidthInLan:     false,
 		DatabaseBlockCacheMiB:   0,
-		PingTimeoutS:            30,
-		PingIdleTimeS:           60,
 		MinHomeDiskFreePct:      1,
+		URURL:                   "https://data.syncthing.net/newdata",
+		URInitialDelayS:         1800,
+		URPostInsecurely:        false,
+		ReleasesURL:             "https://api.github.com/repos/syncthing/syncthing/releases?per_page=30",
 	}
 
 	cfg := New(device1)
@@ -100,13 +106,13 @@ func TestDeviceConfig(t *testing.T) {
 			{
 				DeviceID:    device1,
 				Name:        "node one",
-				Addresses:   []string{"a"},
+				Addresses:   []string{"tcp://a"},
 				Compression: protocol.CompressMetadata,
 			},
 			{
 				DeviceID:    device4,
 				Name:        "node two",
-				Addresses:   []string{"b"},
+				Addresses:   []string{"tcp://b"},
 				Compression: protocol.CompressMetadata,
 			},
 		}
@@ -142,15 +148,19 @@ func TestNoListenAddress(t *testing.T) {
 
 func TestOverriddenValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddress:           []string{":23000"},
+		ListenAddress:           []string{"tcp://:23000"},
 		GlobalAnnServers:        []string{"udp4://syncthing.nym.se:22026"},
 		GlobalAnnEnabled:        false,
 		LocalAnnEnabled:         false,
 		LocalAnnPort:            42123,
 		LocalAnnMCAddr:          "quux:3232",
+		RelayServers:            []string{"relay://123.123.123.123:1234", "relay://125.125.125.125:1255"},
 		MaxSendKbps:             1234,
 		MaxRecvKbps:             2341,
 		ReconnectIntervalS:      6000,
+		RelaysEnabled:           false,
+		RelayReconnectIntervalM: 20,
+		RelayWithoutGlobalAnn:   true,
 		StartBrowser:            false,
 		UPnPEnabled:             false,
 		UPnPLeaseM:              90,
@@ -164,9 +174,11 @@ func TestOverriddenValues(t *testing.T) {
 		SymlinksEnabled:         false,
 		LimitBandwidthInLan:     true,
 		DatabaseBlockCacheMiB:   42,
-		PingTimeoutS:            60,
-		PingIdleTimeS:           120,
 		MinHomeDiskFreePct:      5.2,
+		URURL:                   "https://localhost/newdata",
+		URInitialDelayS:         800,
+		URPostInsecurely:        true,
+		ReleasesURL:             "https://localhost/releases",
 	}
 
 	cfg, err := Load("testdata/overridenvalues.xml", device1)
@@ -255,15 +267,15 @@ func TestDeviceAddressesStatic(t *testing.T) {
 	expected := map[protocol.DeviceID]DeviceConfiguration{
 		device1: {
 			DeviceID:  device1,
-			Addresses: []string{"192.0.2.1", "192.0.2.2"},
+			Addresses: []string{"tcp://192.0.2.1", "tcp://192.0.2.2"},
 		},
 		device2: {
 			DeviceID:  device2,
-			Addresses: []string{"192.0.2.3:6070", "[2001:db8::42]:4242"},
+			Addresses: []string{"tcp://192.0.2.3:6070", "tcp://[2001:db8::42]:4242"},
 		},
 		device3: {
 			DeviceID:  device3,
-			Addresses: []string{"[2001:db8::44]:4444", "192.0.2.4:6090"},
+			Addresses: []string{"tcp://[2001:db8::44]:4444", "tcp://192.0.2.4:6090"},
 		},
 		device4: {
 			DeviceID:    device4,
@@ -330,12 +342,12 @@ func TestIssue1750(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if cfg.Options().ListenAddress[0] != ":23000" {
-		t.Errorf("%q != %q", cfg.Options().ListenAddress[0], ":23000")
+	if cfg.Options().ListenAddress[0] != "tcp://:23000" {
+		t.Errorf("%q != %q", cfg.Options().ListenAddress[0], "tcp://:23000")
 	}
 
-	if cfg.Options().ListenAddress[1] != ":23001" {
-		t.Errorf("%q != %q", cfg.Options().ListenAddress[1], ":23001")
+	if cfg.Options().ListenAddress[1] != "tcp://:23001" {
+		t.Errorf("%q != %q", cfg.Options().ListenAddress[1], "tcp://:23001")
 	}
 
 	if cfg.Options().GlobalAnnServers[0] != "udp4://syncthing.nym.se:22026" {

lib/config/testdata/overridenvalues.xml

@@ -7,10 +7,15 @@
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>42123</localAnnouncePort>
         <localAnnounceMCAddr>quux:3232</localAnnounceMCAddr>
+        <relayServer>relay://123.123.123.123:1234</relayServer>
+        <relayServer>relay://125.125.125.125:1255</relayServer>
         <parallelRequests>32</parallelRequests>
         <maxSendKbps>1234</maxSendKbps>
         <maxRecvKbps>2341</maxRecvKbps>
         <reconnectionIntervalS>6000</reconnectionIntervalS>
+        <relaysEnabled>false</relaysEnabled>
+        <relayReconnectIntervalM>20</relayReconnectIntervalM>
+        <relayWithoutGlobalAnn>true</relayWithoutGlobalAnn>
         <startBrowser>false</startBrowser>
         <upnpEnabled>false</upnpEnabled>
         <upnpLeaseMinutes>90</upnpLeaseMinutes>
@@ -24,8 +29,10 @@
         <symlinksEnabled>false</symlinksEnabled>
         <limitBandwidthInLan>true</limitBandwidthInLan>
         <databaseBlockCacheMiB>42</databaseBlockCacheMiB>
-        <pingTimeoutS>60</pingTimeoutS>
-        <pingIdleTimeS>120</pingIdleTimeS>
         <minHomeDiskFreePct>5.2</minHomeDiskFreePct>
+        <urURL>https://localhost/newdata</urURL>
+        <urInitialDelayS>800</urInitialDelayS>
+        <urPostInsecurely>true</urPostInsecurely>
+        <releasesURL>https://localhost/releases</releasesURL>
     </options>
 </configuration>

lib/config/testdata/v12.xml

@@ -0,0 +1,13 @@
+<configuration version="12">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFreePct>1</minDiskFreePct>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/config/wrapper.go

@@ -60,10 +60,8 @@ type Wrapper struct {
 	deviceMap map[protocol.DeviceID]DeviceConfiguration
 	folderMap map[string]FolderConfiguration
 	replaces  chan Configuration
+	subs      []Committer
 	mut       sync.Mutex
-
-	subs []Committer
-	sMut sync.Mutex
 }
 
 // Wrap wraps an existing Configuration structure and ties it to a file on
@@ -73,7 +71,6 @@ func Wrap(path string, cfg Configuration) *Wrapper {
 		cfg:  cfg,
 		path: path,
 		mut:  sync.NewMutex(),
-		sMut: sync.NewMutex(),
 	}
 	w.replaces = make(chan Configuration)
 	return w
@@ -109,9 +106,24 @@ func (w *Wrapper) Stop() {
 // Subscribe registers the given handler to be called on any future
 // configuration changes.
 func (w *Wrapper) Subscribe(c Committer) {
-	w.sMut.Lock()
+	w.mut.Lock()
 	w.subs = append(w.subs, c)
-	w.sMut.Unlock()
+	w.mut.Unlock()
+}
+
+// Unsubscribe de-registers the given handler from any future calls to
+// configuration changes
+func (w *Wrapper) Unsubscribe(c Committer) {
+	w.mut.Lock()
+	for i := range w.subs {
+		if w.subs[i] == c {
+			copy(w.subs[i:], w.subs[i+1:])
+			w.subs[len(w.subs)-1] = nil
+			w.subs = w.subs[:len(w.subs)-1]
+			break
+		}
+	}
+	w.mut.Unlock()
 }
 
 // Raw returns the currently wrapped Configuration object.
@@ -302,3 +314,15 @@ func (w *Wrapper) Save() error {
 	events.Default.Log(events.ConfigSaved, w.cfg)
 	return nil
 }
+
+func (w *Wrapper) GlobalDiscoveryServers() []string {
+	var servers []string
+	for _, srv := range w.cfg.Options.GlobalAnnServers {
+		if srv == "default" {
+			servers = append(servers, DefaultDiscoveryServers...)
+		} else {
+			servers = append(servers, srv)
+		}
+	}
+	return uniqueStrings(servers)
+}

lib/db/blockmap.go

@@ -16,12 +16,9 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"sort"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/osutil"
-	"github.com/syncthing/syncthing/lib/sync"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -112,48 +109,21 @@ func (m *BlockMap) blockKey(hash []byte, file string) []byte {
 }
 
 type BlockFinder struct {
-	db      *leveldb.DB
-	folders []string
-	mut     sync.RWMutex
+	db *leveldb.DB
 }
 
-func NewBlockFinder(db *leveldb.DB, cfg *config.Wrapper) *BlockFinder {
+func NewBlockFinder(db *leveldb.DB) *BlockFinder {
 	if blockFinder != nil {
 		return blockFinder
 	}
 
 	f := &BlockFinder{
-		db:  db,
-		mut: sync.NewRWMutex(),
+		db: db,
 	}
 
-	f.CommitConfiguration(config.Configuration{}, cfg.Raw())
-	cfg.Subscribe(f)
-
 	return f
 }
 
-// VerifyConfiguration implementes the config.Committer interface
-func (f *BlockFinder) VerifyConfiguration(from, to config.Configuration) error {
-	return nil
-}
-
-// CommitConfiguration implementes the config.Committer interface
-func (f *BlockFinder) CommitConfiguration(from, to config.Configuration) bool {
-	folders := make([]string, len(to.Folders))
-	for i, folder := range to.Folders {
-		folders[i] = folder.ID
-	}
-
-	sort.Strings(folders)
-
-	f.mut.Lock()
-	f.folders = folders
-	f.mut.Unlock()
-
-	return true
-}
-
 func (f *BlockFinder) String() string {
 	return fmt.Sprintf("BlockFinder@%p", f)
 }
@@ -163,10 +133,7 @@ func (f *BlockFinder) String() string {
 // they are happy with the block) or false to continue iterating for whatever
 // reason. The iterator finally returns the result, whether or not a
 // satisfying block was eventually found.
-func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, int32) bool) bool {
-	f.mut.RLock()
-	folders := f.folders
-	f.mut.RUnlock()
+func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string, string, int32) bool) bool {
 	for _, folder := range folders {
 		key := toBlockKey(hash, folder, "")
 		iter := f.db.NewIterator(util.BytesPrefix(key), nil)

lib/db/blockmap_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/config"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
@@ -30,6 +29,7 @@ func genBlocks(n int) []protocol.BlockInfo {
 }
 
 var f1, f2, f3 protocol.FileInfo
+var folders = []string{"folder1", "folder2"}
 
 func init() {
 	blocks := genBlocks(30)
@@ -57,16 +57,7 @@ func setup() (*leveldb.DB, *BlockFinder) {
 	if err != nil {
 		panic(err)
 	}
-
-	wrapper := config.Wrap("", config.Configuration{})
-	wrapper.SetFolder(config.FolderConfiguration{
-		ID: "folder1",
-	})
-	wrapper.SetFolder(config.FolderConfiguration{
-		ID: "folder2",
-	})
-
-	return db, NewBlockFinder(db, wrapper)
+	return db, NewBlockFinder(db)
 }
 
 func dbEmpty(db *leveldb.DB) bool {
@@ -94,21 +85,21 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		if folder != "folder1" || file != "f1" || index != 0 {
 			t.Fatal("Mismatch")
 		}
 		return true
 	})
 
-	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		if folder != "folder1" || file != "f2" || index != 0 {
 			t.Fatal("Mismatch")
 		}
 		return true
 	})
 
-	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		t.Fatal("Unexpected block")
 		return true
 	})
@@ -123,17 +114,17 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		t.Fatal("Unexpected block")
 		return false
 	})
 
-	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		t.Fatal("Unexpected block")
 		return false
 	})
 
-	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		if folder != "folder1" || file != "f3" || index != 0 {
 			t.Fatal("Mismatch")
 		}
@@ -180,7 +171,7 @@ func TestBlockFinderLookup(t *testing.T) {
 	}
 
 	counter := 0
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		counter++
 		switch counter {
 		case 1:
@@ -196,6 +187,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		}
 		return false
 	})
+
 	if counter != 2 {
 		t.Fatal("Incorrect count", counter)
 	}
@@ -208,7 +200,7 @@ func TestBlockFinderLookup(t *testing.T) {
 	}
 
 	counter = 0
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		counter++
 		switch counter {
 		case 1:
@@ -220,6 +212,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		}
 		return false
 	})
+
 	if counter != 1 {
 		t.Fatal("Incorrect count")
 	}
@@ -240,7 +233,7 @@ func TestBlockFinderFix(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !f.Iterate(f1.Blocks[0].Hash, iterFn) {
+	if !f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
 		t.Fatal("Block not found")
 	}
 
@@ -249,11 +242,11 @@ func TestBlockFinderFix(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if f.Iterate(f1.Blocks[0].Hash, iterFn) {
+	if f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
 		t.Fatal("Unexpected block")
 	}
 
-	if !f.Iterate(f2.Blocks[0].Hash, iterFn) {
+	if !f.Iterate(folders, f2.Blocks[0].Hash, iterFn) {
 		t.Fatal("Block not found")
 	}
 }

lib/discover/cache.go

@@ -0,0 +1,198 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	stdsync "sync"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/thejerf/suture"
+)
+
+// The CachingMux aggregates results from multiple Finders. Each Finder has
+// an associated cache time and negative cache time. The cache time sets how
+// long we cache and return successfull lookup results, the negative cache
+// time sets how long we refrain from asking about the same device ID after
+// receiving a negative answer. The value of zero disables caching (positive
+// or negative).
+type CachingMux struct {
+	*suture.Supervisor
+	finders []cachedFinder
+	caches  []*cache
+	mut     sync.Mutex
+}
+
+// A cachedFinder is a Finder with associated cache timeouts.
+type cachedFinder struct {
+	Finder
+	cacheTime    time.Duration
+	negCacheTime time.Duration
+}
+
+func NewCachingMux() *CachingMux {
+	return &CachingMux{
+		Supervisor: suture.NewSimple("discover.cachingMux"),
+		mut:        sync.NewMutex(),
+	}
+}
+
+// Add registers a new Finder, with associated cache timeouts.
+func (m *CachingMux) Add(finder Finder, cacheTime, negCacheTime time.Duration) {
+	m.mut.Lock()
+	m.finders = append(m.finders, cachedFinder{finder, cacheTime, negCacheTime})
+	m.caches = append(m.caches, newCache())
+	m.mut.Unlock()
+
+	if svc, ok := finder.(suture.Service); ok {
+		m.Supervisor.Add(svc)
+	}
+}
+
+// Lookup attempts to resolve the device ID using any of the added Finders,
+// while obeying the cache settings.
+func (m *CachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	m.mut.Lock()
+	for i, finder := range m.finders {
+		if cacheEntry, ok := m.caches[i].Get(deviceID); ok {
+			// We have a cache entry. Lets see what it says.
+
+			if cacheEntry.found && time.Since(cacheEntry.when) < finder.cacheTime {
+				// It's a positive, valid entry. Use it.
+				if debug {
+					l.Debugln("cached discovery entry for", deviceID, "at", finder.String())
+					l.Debugln("   ", cacheEntry)
+				}
+				direct = append(direct, cacheEntry.Direct...)
+				relays = append(relays, cacheEntry.Relays...)
+				continue
+			}
+
+			if !cacheEntry.found && time.Since(cacheEntry.when) < finder.negCacheTime {
+				// It's a negative, valid entry. We should not make another
+				// attempt right now.
+				if debug {
+					l.Debugln("negative cache entry for", deviceID, "at", finder.String())
+				}
+				continue
+			}
+
+			// It's expired. Ignore and continue.
+		}
+
+		// Perform the actual lookup and cache the result.
+		if td, tr, err := finder.Lookup(deviceID); err == nil {
+			if debug {
+				l.Debugln("lookup for", deviceID, "at", finder.String())
+				l.Debugln("   ", td)
+				l.Debugln("   ", tr)
+			}
+			direct = append(direct, td...)
+			relays = append(relays, tr...)
+			m.caches[i].Set(deviceID, CacheEntry{
+				Direct: td,
+				Relays: tr,
+				when:   time.Now(),
+				found:  len(td)+len(tr) > 0,
+			})
+		}
+	}
+	m.mut.Unlock()
+
+	if debug {
+		l.Debugln("lookup results for", deviceID)
+		l.Debugln("   ", direct)
+		l.Debugln("   ", relays)
+	}
+
+	return direct, relays, nil
+}
+
+func (m *CachingMux) String() string {
+	return "discovery cache"
+}
+
+func (m *CachingMux) Error() error {
+	return nil
+}
+
+func (m *CachingMux) ChildErrors() map[string]error {
+	m.mut.Lock()
+	children := make(map[string]error, len(m.finders))
+	for _, f := range m.finders {
+		children[f.String()] = f.Error()
+	}
+	m.mut.Unlock()
+	return children
+}
+
+func (m *CachingMux) Cache() map[protocol.DeviceID]CacheEntry {
+	// Res will be the "total" cache, i.e. the union of our cache and all our
+	// children's caches.
+	res := make(map[protocol.DeviceID]CacheEntry)
+
+	m.mut.Lock()
+	for i := range m.finders {
+		// Each finder[i] has a corresponding cache at cache[i]. Go through it
+		// and populate the total, if it's newer than what's already in there.
+		// We skip any negative cache entries.
+		for k, v := range m.caches[i].Cache() {
+			if v.found && v.when.After(res[k].when) {
+				res[k] = v
+			}
+		}
+
+		// Then ask the finder itself for it's cache and do the same. If this
+		// finder is a global discovery client, it will have no cache. If it's
+		// a local discovery client, this will be it's current state.
+		for k, v := range m.finders[i].Cache() {
+			if v.found && v.when.After(res[k].when) {
+				res[k] = v
+			}
+		}
+	}
+	m.mut.Unlock()
+
+	return res
+}
+
+// A cache can be embedded wherever useful
+
+type cache struct {
+	entries map[protocol.DeviceID]CacheEntry
+	mut     stdsync.Mutex
+}
+
+func newCache() *cache {
+	return &cache{
+		entries: make(map[protocol.DeviceID]CacheEntry),
+	}
+}
+
+func (c *cache) Set(id protocol.DeviceID, ce CacheEntry) {
+	c.mut.Lock()
+	c.entries[id] = ce
+	c.mut.Unlock()
+}
+
+func (c *cache) Get(id protocol.DeviceID) (CacheEntry, bool) {
+	c.mut.Lock()
+	ce, ok := c.entries[id]
+	c.mut.Unlock()
+	return ce, ok
+}
+
+func (c *cache) Cache() map[protocol.DeviceID]CacheEntry {
+	c.mut.Lock()
+	m := make(map[protocol.DeviceID]CacheEntry, len(c.entries))
+	for k, v := range c.entries {
+		m[k] = v
+	}
+	c.mut.Unlock()
+	return m
+}

lib/discover/client.go

@@ -1,50 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"fmt"
-	"net/url"
-	"time"
-
-	"github.com/syncthing/protocol"
-)
-
-type Factory func(*url.URL, *Announce) (Client, error)
-
-var (
-	factories                      = make(map[string]Factory)
-	DefaultErrorRetryInternval     = 60 * time.Second
-	DefaultGlobalBroadcastInterval = 1800 * time.Second
-)
-
-func Register(proto string, factory Factory) {
-	factories[proto] = factory
-}
-
-func New(addr string, pkt *Announce) (Client, error) {
-	uri, err := url.Parse(addr)
-	if err != nil {
-		return nil, err
-	}
-	factory, ok := factories[uri.Scheme]
-	if !ok {
-		return nil, fmt.Errorf("Unsupported scheme: %s", uri.Scheme)
-	}
-	client, err := factory(uri, pkt)
-	if err != nil {
-		return nil, err
-	}
-	return client, nil
-}
-
-type Client interface {
-	Lookup(device protocol.DeviceID) []string
-	StatusOK() bool
-	Address() string
-	Stop()
-}

lib/discover/client_test.go

@@ -1,219 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"testing"
-
-	"github.com/syncthing/protocol"
-
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-var device protocol.DeviceID
-
-func init() {
-	device, _ = protocol.DeviceIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
-}
-
-func TestUDP4Success(t *testing.T) {
-	conn, err := net.ListenUDP("udp4", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-
-	address := fmt.Sprintf("udp4://127.0.0.1:%d", port)
-	pkt := &Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			device[:],
-			[]Address{{
-				IP:   net.IPv4(123, 123, 123, 123),
-				Port: 1234,
-			}},
-		},
-	}
-
-	client, err := New(address, pkt)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	udpclient := client.(*UDPClient)
-	if udpclient.errorRetryInterval != DefaultErrorRetryInternval {
-		t.Fatal("Incorrect retry interval")
-	}
-
-	if udpclient.listenAddress.IP != nil || udpclient.listenAddress.Port != 0 {
-		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
-	}
-
-	if client.Address() != address {
-		t.Fatal("Incorrect address")
-	}
-
-	buf := make([]byte, 2048)
-
-	// First announcement
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Announcement verification
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
-	_, addr, err := conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Reply to it.
-	_, err = conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// We should get nothing else
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-
-	// Status should be ok
-	if !client.StatusOK() {
-		t.Fatal("Wrong status")
-	}
-
-	// Do a lookup in a separate routine
-	addrs := []string{}
-	wg := sync.NewWaitGroup()
-	wg.Add(1)
-	go func() {
-		addrs = client.Lookup(device)
-		wg.Done()
-	}()
-
-	// Receive the lookup and reply
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, addr, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
-
-	// Wait for the lookup to arrive, verify that the number of answers is correct
-	wg.Wait()
-
-	if len(addrs) != 1 || addrs[0] != "123.123.123.123:1234" {
-		t.Fatal("Wrong number of answers")
-	}
-
-	client.Stop()
-}
-
-func TestUDP4Failure(t *testing.T) {
-	conn, err := net.ListenUDP("udp4", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-
-	address := fmt.Sprintf("udp4://127.0.0.1:%d/?listenaddress=127.0.0.1&retry=5", port)
-
-	pkt := &Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			device[:],
-			[]Address{{
-				IP:   net.IPv4(123, 123, 123, 123),
-				Port: 1234,
-			}},
-		},
-	}
-
-	client, err := New(address, pkt)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	udpclient := client.(*UDPClient)
-	if udpclient.errorRetryInterval != time.Second*5 {
-		t.Fatal("Incorrect retry interval")
-	}
-
-	if !udpclient.listenAddress.IP.Equal(net.IPv4(127, 0, 0, 1)) || udpclient.listenAddress.Port != 0 {
-		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
-	}
-
-	if client.Address() != address {
-		t.Fatal("Incorrect address")
-	}
-
-	buf := make([]byte, 2048)
-
-	// First announcement
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Announcement verification
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
-	_, _, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Don't reply
-	// We should get nothing else
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-
-	// Status should be failure
-	if client.StatusOK() {
-		t.Fatal("Wrong status")
-	}
-
-	// Do a lookup in a separate routine
-	addrs := []string{}
-	wg := sync.NewWaitGroup()
-	wg.Add(1)
-	go func() {
-		addrs = client.Lookup(device)
-		wg.Done()
-	}()
-
-	// Receive the lookup and don't reply
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, _, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Wait for the lookup to timeout, verify that the number of answers is none
-	wg.Wait()
-
-	if len(addrs) != 0 {
-		t.Fatal("Wrong number of answers")
-	}
-
-	client.Stop()
-}

lib/discover/client_udp.go

@@ -1,243 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"encoding/hex"
-	"io"
-	"net"
-	"net/url"
-	"strconv"
-	"time"
-
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-func init() {
-	for _, proto := range []string{"udp", "udp4", "udp6"} {
-		Register(proto, func(uri *url.URL, pkt *Announce) (Client, error) {
-			c := &UDPClient{
-				wg:  sync.NewWaitGroup(),
-				mut: sync.NewRWMutex(),
-			}
-			err := c.Start(uri, pkt)
-			if err != nil {
-				return nil, err
-			}
-			return c, nil
-		})
-	}
-}
-
-type UDPClient struct {
-	url *url.URL
-
-	id protocol.DeviceID
-
-	stop          chan struct{}
-	wg            sync.WaitGroup
-	listenAddress *net.UDPAddr
-
-	globalBroadcastInterval time.Duration
-	errorRetryInterval      time.Duration
-
-	status bool
-	mut    sync.RWMutex
-}
-
-func (d *UDPClient) Start(uri *url.URL, pkt *Announce) error {
-	d.url = uri
-	d.id = protocol.DeviceIDFromBytes(pkt.This.ID)
-	d.stop = make(chan struct{})
-
-	params := uri.Query()
-	// The address must not have a port, as otherwise both announce and lookup
-	// sockets would try to bind to the same port.
-	addr, err := net.ResolveUDPAddr(d.url.Scheme, params.Get("listenaddress")+":0")
-	if err != nil {
-		return err
-	}
-	d.listenAddress = addr
-
-	broadcastSeconds, err := strconv.ParseUint(params.Get("broadcast"), 0, 0)
-	if err != nil {
-		d.globalBroadcastInterval = DefaultGlobalBroadcastInterval
-	} else {
-		d.globalBroadcastInterval = time.Duration(broadcastSeconds) * time.Second
-	}
-
-	retrySeconds, err := strconv.ParseUint(params.Get("retry"), 0, 0)
-	if err != nil {
-		d.errorRetryInterval = DefaultErrorRetryInternval
-	} else {
-		d.errorRetryInterval = time.Duration(retrySeconds) * time.Second
-	}
-
-	d.wg.Add(1)
-	go d.broadcast(pkt.MustMarshalXDR())
-	return nil
-}
-
-func (d *UDPClient) broadcast(pkt []byte) {
-	defer d.wg.Done()
-
-	conn, err := net.ListenUDP(d.url.Scheme, d.listenAddress)
-	for err != nil {
-		if debug {
-			l.Debugf("discover %s: broadcast listen: %v; trying again in %v", d.url, err, d.errorRetryInterval)
-		}
-		select {
-		case <-d.stop:
-			return
-		case <-time.After(d.errorRetryInterval):
-		}
-		conn, err = net.ListenUDP(d.url.Scheme, d.listenAddress)
-	}
-	defer conn.Close()
-
-	remote, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	for err != nil {
-		if debug {
-			l.Debugf("discover %s: broadcast resolve: %v; trying again in %v", d.url, err, d.errorRetryInterval)
-		}
-		select {
-		case <-d.stop:
-			return
-		case <-time.After(d.errorRetryInterval):
-		}
-		remote, err = net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	}
-
-	timer := time.NewTimer(0)
-	for {
-		select {
-		case <-d.stop:
-			return
-
-		case <-timer.C:
-			var ok bool
-
-			if debug {
-				l.Debugf("discover %s: broadcast: Sending self announcement to %v", d.url, remote)
-			}
-
-			_, err := conn.WriteTo(pkt, remote)
-			if err != nil {
-				if debug {
-					l.Debugf("discover %s: broadcast: Failed to send self announcement: %s", d.url, err)
-				}
-				ok = false
-			} else {
-				// Verify that the announce server responds positively for our device ID
-
-				time.Sleep(1 * time.Second)
-
-				res := d.Lookup(d.id)
-				if debug {
-					l.Debugf("discover %s: broadcast: Self-lookup returned: %v", d.url, res)
-				}
-				ok = len(res) > 0
-			}
-
-			d.mut.Lock()
-			d.status = ok
-			d.mut.Unlock()
-
-			if ok {
-				timer.Reset(d.globalBroadcastInterval)
-			} else {
-				timer.Reset(d.errorRetryInterval)
-			}
-		}
-	}
-}
-
-func (d *UDPClient) Lookup(device protocol.DeviceID) []string {
-	extIP, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	conn, err := net.DialUDP(d.url.Scheme, d.listenAddress, extIP)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-	defer conn.Close()
-
-	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	buf := Query{QueryMagic, device[:]}.MustMarshalXDR()
-	_, err = conn.Write(buf)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	buf = make([]byte, 2048)
-	n, err := conn.Read(buf)
-	if err != nil {
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			// Expected if the server doesn't know about requested device ID
-			return nil
-		}
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	var pkt Announce
-	err = pkt.UnmarshalXDR(buf[:n])
-	if err != nil && err != io.EOF {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s\n%s", d.url, device, err, hex.Dump(buf[:n]))
-		}
-		return nil
-	}
-
-	var addrs []string
-	for _, a := range pkt.This.Addresses {
-		deviceAddr := net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
-		addrs = append(addrs, deviceAddr)
-	}
-	if debug {
-		l.Debugf("discover %s: Lookup(%s) result: %v", d.url, device, addrs)
-	}
-	return addrs
-}
-
-func (d *UDPClient) Stop() {
-	if d.stop != nil {
-		close(d.stop)
-		d.wg.Wait()
-	}
-}
-
-func (d *UDPClient) StatusOK() bool {
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-	return d.status
-}
-
-func (d *UDPClient) Address() string {
-	return d.url.String()
-}

lib/discover/discover.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2014 The Syncthing Authors.
+// Copyright (C) 2015 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
@@ -7,424 +7,48 @@
 package discover
 
 import (
-	"bytes"
-	"encoding/hex"
-	"errors"
-	"io"
-	"net"
-	"strconv"
 	"time"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/beacon"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/thejerf/suture"
 )
 
-type Discoverer struct {
-	myID            protocol.DeviceID
-	listenAddrs     []string
-	localBcastIntv  time.Duration
-	localBcastStart time.Time
-	cacheLifetime   time.Duration
-	negCacheCutoff  time.Duration
-	beacons         []beacon.Interface
-	extPort         uint16
-	localBcastTick  <-chan time.Time
-	forcedBcastTick chan time.Time
-
-	registryLock sync.RWMutex
-	registry     map[protocol.DeviceID][]CacheEntry
-	lastLookup   map[protocol.DeviceID]time.Time
-
-	clients []Client
-	mut     sync.RWMutex
+// A Finder provides lookup services of some kind.
+type Finder interface {
+	Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error)
+	Error() error
+	String() string
+	Cache() map[protocol.DeviceID]CacheEntry
 }
 
 type CacheEntry struct {
-	Address string
-	Seen    time.Time
+	Direct []string  `json:"direct"`
+	Relays []Relay   `json:"relays"`
+	when   time.Time // When did we get the result
+	found  bool      // Is it a success (cacheTime applies) or a failure (negCacheTime applies)?
 }
 
-var (
-	ErrIncorrectMagic = errors.New("incorrect magic number")
-)
-
-func NewDiscoverer(id protocol.DeviceID, addresses []string) *Discoverer {
-	return &Discoverer{
-		myID:           id,
-		listenAddrs:    addresses,
-		localBcastIntv: 30 * time.Second,
-		cacheLifetime:  5 * time.Minute,
-		negCacheCutoff: 3 * time.Minute,
-		registry:       make(map[protocol.DeviceID][]CacheEntry),
-		lastLookup:     make(map[protocol.DeviceID]time.Time),
-		registryLock:   sync.NewRWMutex(),
-		mut:            sync.NewRWMutex(),
-	}
+// A FinderService is a Finder that has background activity and must be run as
+// a suture.Service.
+type FinderService interface {
+	Finder
+	suture.Service
 }
 
-func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
-	if localPort > 0 {
-		d.startLocalIPv4Broadcasts(localPort)
-	}
-
-	if len(localMCAddr) > 0 {
-		d.startLocalIPv6Multicasts(localMCAddr)
-	}
-
-	if len(d.beacons) == 0 {
-		l.Warnln("Local discovery unavailable")
-		return
-	}
-
-	d.localBcastTick = time.Tick(d.localBcastIntv)
-	d.forcedBcastTick = make(chan time.Time)
-	d.localBcastStart = time.Now()
-	go d.sendLocalAnnouncements()
+type FinderMux interface {
+	Finder
+	ChildStatus() map[string]error
 }
 
-func (d *Discoverer) startLocalIPv4Broadcasts(localPort int) {
-	bb := beacon.NewBroadcast(localPort)
-	d.beacons = append(d.beacons, bb)
-	go d.recvAnnouncements(bb)
-	bb.ServeBackground()
+// The RelayStatusProvider answers questions about current relay status.
+type RelayStatusProvider interface {
+	Relays() []string
+	RelayStatus(uri string) (time.Duration, bool)
 }
 
-func (d *Discoverer) startLocalIPv6Multicasts(localMCAddr string) {
-	mb, err := beacon.NewMulticast(localMCAddr)
-	if err != nil {
-		if debug {
-			l.Debugln("beacon.NewMulticast:", err)
-		}
-		l.Infoln("Local discovery over IPv6 unavailable")
-		return
-	}
-	d.beacons = append(d.beacons, mb)
-	go d.recvAnnouncements(mb)
-}
-
-func (d *Discoverer) StartGlobal(servers []string, extPort uint16) {
-	d.mut.Lock()
-	defer d.mut.Unlock()
-
-	if len(d.clients) > 0 {
-		d.stopGlobal()
-	}
-
-	d.extPort = extPort
-	pkt := d.announcementPkt()
-	wg := sync.NewWaitGroup()
-	clients := make(chan Client, len(servers))
-	for _, address := range servers {
-		wg.Add(1)
-		go func(addr string) {
-			defer wg.Done()
-			client, err := New(addr, pkt)
-			if err != nil {
-				l.Infoln("Error creating discovery client", addr, err)
-				return
-			}
-			clients <- client
-		}(address)
-	}
-
-	wg.Wait()
-	close(clients)
-
-	for client := range clients {
-		d.clients = append(d.clients, client)
-	}
-}
-
-func (d *Discoverer) StopGlobal() {
-	d.mut.Lock()
-	defer d.mut.Unlock()
-	d.stopGlobal()
-}
-
-func (d *Discoverer) stopGlobal() {
-	for _, client := range d.clients {
-		client.Stop()
-	}
-	d.clients = []Client{}
-}
-
-func (d *Discoverer) ExtAnnounceOK() map[string]bool {
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-
-	ret := make(map[string]bool)
-	for _, client := range d.clients {
-		ret[client.Address()] = client.StatusOK()
-	}
-	return ret
-}
-
-func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
-	d.registryLock.RLock()
-	cached := d.filterCached(d.registry[device])
-	lastLookup := d.lastLookup[device]
-	d.registryLock.RUnlock()
-
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-
-	if len(cached) > 0 {
-		// There are cached address entries.
-		addrs := make([]string, len(cached))
-		for i := range cached {
-			addrs[i] = cached[i].Address
-		}
-		return addrs
-	}
-
-	if time.Since(lastLookup) < d.negCacheCutoff {
-		// We have recently tried to lookup this address and failed. Lets
-		// chill for a while.
-		return nil
-	}
-
-	if len(d.clients) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
-		// Only perform external lookups if we have at least one external
-		// server client and one local announcement interval has passed. This is
-		// to avoid finding local peers on their remote address at startup.
-		results := make(chan []string, len(d.clients))
-		wg := sync.NewWaitGroup()
-		for _, client := range d.clients {
-			wg.Add(1)
-			go func(c Client) {
-				defer wg.Done()
-				results <- c.Lookup(device)
-			}(client)
-		}
-
-		wg.Wait()
-		close(results)
-
-		cached := []CacheEntry{}
-		seen := make(map[string]struct{})
-		now := time.Now()
-
-		var addrs []string
-		for result := range results {
-			for _, addr := range result {
-				_, ok := seen[addr]
-				if !ok {
-					cached = append(cached, CacheEntry{
-						Address: addr,
-						Seen:    now,
-					})
-					seen[addr] = struct{}{}
-					addrs = append(addrs, addr)
-				}
-			}
-		}
-
-		d.registryLock.Lock()
-		d.registry[device] = cached
-		d.lastLookup[device] = time.Now()
-		d.registryLock.Unlock()
-
-		return addrs
-	}
-
-	return nil
-}
-
-func (d *Discoverer) Hint(device string, addrs []string) {
-	resAddrs := resolveAddrs(addrs)
-	var id protocol.DeviceID
-	id.UnmarshalText([]byte(device))
-	d.registerDevice(nil, Device{
-		Addresses: resAddrs,
-		ID:        id[:],
-	})
-}
-
-func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
-	d.registryLock.RLock()
-	devices := make(map[protocol.DeviceID][]CacheEntry, len(d.registry))
-	for device, addrs := range d.registry {
-		addrsCopy := make([]CacheEntry, len(addrs))
-		copy(addrsCopy, addrs)
-		devices[device] = addrsCopy
-	}
-	d.registryLock.RUnlock()
-	return devices
-}
-
-func (d *Discoverer) announcementPkt() *Announce {
-	var addrs []Address
-	if d.extPort != 0 {
-		addrs = []Address{{Port: d.extPort}}
-	} else {
-		for _, astr := range d.listenAddrs {
-			addr, err := net.ResolveTCPAddr("tcp", astr)
-			if err != nil {
-				l.Warnln("discover: %v: not announcing %s", err, astr)
-				continue
-			} else if debug {
-				l.Debugf("discover: resolved %s as %#v", astr, addr)
-			}
-			if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
-				addrs = append(addrs, Address{Port: uint16(addr.Port)})
-			} else if bs := addr.IP.To4(); bs != nil {
-				addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
-			} else if bs := addr.IP.To16(); bs != nil {
-				addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
-			}
-		}
-	}
-	return &Announce{
-		Magic: AnnouncementMagic,
-		This:  Device{d.myID[:], addrs},
-	}
-}
-
-func (d *Discoverer) sendLocalAnnouncements() {
-	var addrs = resolveAddrs(d.listenAddrs)
-
-	var pkt = Announce{
-		Magic: AnnouncementMagic,
-		This:  Device{d.myID[:], addrs},
-	}
-	msg := pkt.MustMarshalXDR()
-
-	for {
-		for _, b := range d.beacons {
-			b.Send(msg)
-		}
-
-		select {
-		case <-d.localBcastTick:
-		case <-d.forcedBcastTick:
-		}
-	}
-}
-
-func (d *Discoverer) recvAnnouncements(b beacon.Interface) {
-	for {
-		buf, addr := b.Recv()
-
-		var pkt Announce
-		err := pkt.UnmarshalXDR(buf)
-		if err != nil && err != io.EOF {
-			if debug {
-				l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
-			}
-			continue
-		}
-
-		if debug {
-			l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
-		}
-
-		var newDevice bool
-		if bytes.Compare(pkt.This.ID, d.myID[:]) != 0 {
-			newDevice = d.registerDevice(addr, pkt.This)
-		}
-
-		if newDevice {
-			select {
-			case d.forcedBcastTick <- time.Now():
-			}
-		}
-	}
-}
-
-func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
-	var id protocol.DeviceID
-	copy(id[:], device.ID)
-
-	d.registryLock.Lock()
-	defer d.registryLock.Unlock()
-
-	current := d.filterCached(d.registry[id])
-
-	orig := current
-
-	for _, a := range device.Addresses {
-		var deviceAddr string
-		if len(a.IP) > 0 {
-			deviceAddr = net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
-		} else if addr != nil {
-			ua := addr.(*net.UDPAddr)
-			ua.Port = int(a.Port)
-			deviceAddr = ua.String()
-		}
-		for i := range current {
-			if current[i].Address == deviceAddr {
-				current[i].Seen = time.Now()
-				goto done
-			}
-		}
-		current = append(current, CacheEntry{
-			Address: deviceAddr,
-			Seen:    time.Now(),
-		})
-	done:
-	}
-
-	if debug {
-		l.Debugf("discover: Caching %s addresses: %v", id, current)
-	}
-
-	d.registry[id] = current
-
-	if len(current) > len(orig) {
-		addrs := make([]string, len(current))
-		for i := range current {
-			addrs[i] = current[i].Address
-		}
-		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
-			"device": id.String(),
-			"addrs":  addrs,
-		})
-	}
-
-	return len(current) > len(orig)
-}
-
-func (d *Discoverer) filterCached(c []CacheEntry) []CacheEntry {
-	for i := 0; i < len(c); {
-		if ago := time.Since(c[i].Seen); ago > d.cacheLifetime {
-			if debug {
-				l.Debugf("discover: Removing cached address %s - seen %v ago", c[i].Address, ago)
-			}
-			c[i] = c[len(c)-1]
-			c = c[:len(c)-1]
-		} else {
-			i++
-		}
-	}
-	return c
-}
-
-func addrToAddr(addr *net.TCPAddr) Address {
-	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
-		return Address{Port: uint16(addr.Port)}
-	} else if bs := addr.IP.To4(); bs != nil {
-		return Address{IP: bs, Port: uint16(addr.Port)}
-	} else if bs := addr.IP.To16(); bs != nil {
-		return Address{IP: bs, Port: uint16(addr.Port)}
-	}
-	return Address{}
-}
-
-func resolveAddrs(addrs []string) []Address {
-	var raddrs []Address
-	for _, addrStr := range addrs {
-		addrRes, err := net.ResolveTCPAddr("tcp", addrStr)
-		if err != nil {
-			continue
-		}
-		addr := addrToAddr(addrRes)
-		if len(addr.IP) > 0 {
-			raddrs = append(raddrs, addr)
-		} else {
-			raddrs = append(raddrs, Address{Port: addr.Port})
-		}
-	}
-	return raddrs
+// The AddressLister answers questions about what addresses we are listening
+// on.
+type AddressLister interface {
+	ExternalAddresses() []string
+	AllAddresses() []string
 }

lib/discover/discover_test.go

@@ -1,139 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"net/url"
-	"time"
-
-	"testing"
-
-	"github.com/syncthing/protocol"
-)
-
-type DummyClient struct {
-	url          *url.URL
-	lookups      []protocol.DeviceID
-	lookupRet    []string
-	stops        int
-	statusRet    bool
-	statusChecks int
-}
-
-func (c *DummyClient) Lookup(device protocol.DeviceID) []string {
-	c.lookups = append(c.lookups, device)
-	return c.lookupRet
-}
-
-func (c *DummyClient) StatusOK() bool {
-	c.statusChecks++
-	return c.statusRet
-}
-
-func (c *DummyClient) Stop() {
-	c.stops++
-}
-
-func (c *DummyClient) Address() string {
-	return c.url.String()
-}
-
-func TestGlobalDiscovery(t *testing.T) {
-	c1 := &DummyClient{
-		statusRet: false,
-		lookupRet: []string{"test.com:1234"},
-	}
-
-	c2 := &DummyClient{
-		statusRet: true,
-		lookupRet: []string{},
-	}
-
-	c3 := &DummyClient{
-		statusRet: true,
-		lookupRet: []string{"best.com:2345"},
-	}
-
-	clients := []*DummyClient{c1, c2}
-
-	Register("test1", func(uri *url.URL, pkt *Announce) (Client, error) {
-		c := clients[0]
-		clients = clients[1:]
-		c.url = uri
-		return c, nil
-	})
-
-	Register("test2", func(uri *url.URL, pkt *Announce) (Client, error) {
-		c3.url = uri
-		return c3, nil
-	})
-
-	d := NewDiscoverer(device, []string{})
-	d.localBcastStart = time.Time{}
-	servers := []string{
-		"test1://123.123.123.123:1234",
-		"test1://23.23.23.23:234",
-		"test2://234.234.234.234.2345",
-	}
-	d.StartGlobal(servers, 1234)
-
-	if len(d.clients) != 3 {
-		t.Fatal("Wrong number of clients")
-	}
-
-	status := d.ExtAnnounceOK()
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if status[c.url.String()] != c.statusRet || c.statusChecks != 1 {
-			t.Fatal("Wrong status")
-		}
-	}
-
-	addrs := d.Lookup(device)
-	if len(addrs) != 2 {
-		t.Fatal("Wrong number of addresses", addrs)
-	}
-
-	for _, addr := range []string{"test.com:1234", "best.com:2345"} {
-		found := false
-		for _, laddr := range addrs {
-			if laddr == addr {
-				found = true
-				break
-			}
-		}
-		if !found {
-			t.Fatal("Couldn't find", addr)
-		}
-	}
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if len(c.lookups) != 1 || c.lookups[0] != device {
-			t.Fatal("Wrong lookups")
-		}
-	}
-
-	addrs = d.Lookup(device)
-	if len(addrs) != 2 {
-		t.Fatal("Wrong number of addresses", addrs)
-	}
-
-	// Answer should be cached, so number of lookups should have not increased
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if len(c.lookups) != 1 || c.lookups[0] != device {
-			t.Fatal("Wrong lookups")
-		}
-	}
-
-	d.StopGlobal()
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if c.stops != 1 {
-			t.Fatal("Wrong number of stops")
-		}
-	}
-}

lib/discover/doc.go

@@ -1,8 +1,75 @@
-// Copyright (C) 2014 The Syncthing Authors.
+// Copyright (C) 2015 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-// Package discover implements the device discovery protocol.
+/*
+Package discover implements the local and global device discovery protocols.
+
+Global Discovery
+================
+
+Announcements
+-------------
+
+A device should announce itself at startup. It does this by an HTTPS POST to
+the announce server URL (with the path usually being "/", but this is of
+course up to the discovery server). The POST has a JSON payload listing direct
+connection addresses (if any) and relay addresses (if any).
+
+	{
+		direct: ["tcp://192.0.2.45:22000", "tcp://:22202"],
+		relays: [{"url": "relay://192.0.2.99:22028", "latency": 142}]
+	}
+
+It's OK for either of the "direct" or "relays" fields to be either the empty
+list ([]), null, or missing entirely. An announcment with both fields missing
+or empty is however not useful...
+
+Any empty or unspecified IP addresses (i.e. addresses like tcp://:22000,
+tcp://0.0.0.0:22000, tcp://[::]:22000) are interpreted as referring to the
+source IP address of the announcement.
+
+The device ID of the announcing device is not part of the announcement.
+Instead, the server requires that the client perform certificate
+authentication. The device ID is deduced from the presented certificate.
+
+The server response is empty, with code 200 (OK) on success. If no certificate
+was presented, status 403 (Forbidden) is returned. If the posted data doesn't
+conform to the expected format, 400 (Bad Request) is returned.
+
+In successfull responses, the server may return a "Reannounce-After" header
+containing the number of seconds after which the client should perform a new
+announcement.
+
+In error responses, the server may return a "Retry-After" header containing
+the number of seconds after which the client should retry.
+
+Performing announcements significantly more often than indicated by the
+Reannounce-After or Retry-After headers may result in the client being
+throttled. In such cases the server may respond with status code 429 (Too Many
+Requests).
+
+Queries
+=======
+
+Queries are performed as HTTPS GET requests to the announce server URL. The
+requested device ID is passed as the query parameter "device", in canonical
+string form, i.e. https://announce.syncthing.net/?device=ABC12345-....
+
+Successfull responses will have status code 200 (OK) and carry a JSON payload
+of the same format as the announcement above. The response will not contain
+empty or unspecified addresses.
+
+If the "device" query parameter is missing or malformed, the status code 400
+(Bad Request) is returned.
+
+If the device ID is of a valid format but not found in the registry, 404 (Not
+Found) is returned.
+
+If the client has exceeded a rate limit, the server may respond with 429 (Too
+Many Requests).
+
+*/
 package discover

lib/discover/global.go

@@ -0,0 +1,391 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	stdsync "sync"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/events"
+)
+
+type globalClient struct {
+	server         string
+	addrList       AddressLister
+	relayStat      RelayStatusProvider
+	announceClient httpClient
+	queryClient    httpClient
+	noAnnounce     bool
+	stop           chan struct{}
+	errorHolder
+}
+
+type httpClient interface {
+	Get(url string) (*http.Response, error)
+	Post(url, ctype string, data io.Reader) (*http.Response, error)
+}
+
+const (
+	defaultReannounceInterval  = 30 * time.Minute
+	announceErrorRetryInterval = 5 * time.Minute
+)
+
+type announcement struct {
+	Direct []string `json:"direct"`
+	Relays []Relay  `json:"relays"`
+}
+
+type serverOptions struct {
+	insecure   bool   // don't check certificate
+	noAnnounce bool   // don't announce
+	id         string // expected server device ID
+}
+
+func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
+	server, opts, err := parseOptions(server)
+	if err != nil {
+		return nil, err
+	}
+
+	var devID protocol.DeviceID
+	if opts.id != "" {
+		devID, err = protocol.DeviceIDFromString(opts.id)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// The http.Client used for announcements. It needs to have our
+	// certificate to prove our identity, and may or may not verify the server
+	// certificate depending on the insecure setting.
+	var announceClient httpClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: opts.insecure,
+				Certificates:       []tls.Certificate{cert},
+			},
+		},
+	}
+	if opts.id != "" {
+		announceClient = newIDCheckingHTTPClient(announceClient, devID)
+	}
+
+	// The http.Client used for queries. We don't need to present our
+	// certificate here, so lets not include it. May be insecure if requested.
+	var queryClient httpClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: opts.insecure,
+			},
+		},
+	}
+	if opts.id != "" {
+		queryClient = newIDCheckingHTTPClient(queryClient, devID)
+	}
+
+	cl := &globalClient{
+		server:         server,
+		addrList:       addrList,
+		relayStat:      relayStat,
+		announceClient: announceClient,
+		queryClient:    queryClient,
+		noAnnounce:     opts.noAnnounce,
+		stop:           make(chan struct{}),
+	}
+	cl.setError(errors.New("not announced"))
+
+	return cl, nil
+}
+
+// Lookup returns the list of addresses where the given device is available;
+// direct, and via relays.
+func (c *globalClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	qURL, err := url.Parse(c.server)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	q := qURL.Query()
+	q.Set("device", device.String())
+	qURL.RawQuery = q.Encode()
+
+	resp, err := c.queryClient.Get(qURL.String())
+	if err != nil {
+		if debug {
+			l.Debugln("globalClient.Lookup", qURL.String(), err)
+		}
+		return nil, nil, err
+	}
+	if resp.StatusCode != 200 {
+		resp.Body.Close()
+		if debug {
+			l.Debugln("globalClient.Lookup", qURL.String(), resp.Status)
+		}
+		return nil, nil, errors.New(resp.Status)
+	}
+
+	// TODO: Handle 429 and Retry-After?
+
+	var ann announcement
+	err = json.NewDecoder(resp.Body).Decode(&ann)
+	resp.Body.Close()
+	return ann.Direct, ann.Relays, err
+}
+
+func (c *globalClient) String() string {
+	return "global@" + c.server
+}
+
+func (c *globalClient) Serve() {
+	if c.noAnnounce {
+		// We're configured to not do announcements, only lookups. To maintain
+		// the same interface, we just pause here if Serve() is run.
+		<-c.stop
+		return
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+
+	eventSub := events.Default.Subscribe(events.ExternalPortMappingChanged | events.RelayStateChanged)
+	defer events.Default.Unsubscribe(eventSub)
+
+	for {
+		select {
+		case <-eventSub.C():
+			c.sendAnnouncement(timer)
+
+		case <-timer.C:
+			c.sendAnnouncement(timer)
+
+		case <-c.stop:
+			return
+		}
+	}
+}
+
+func (c *globalClient) sendAnnouncement(timer *time.Timer) {
+
+	var ann announcement
+	if c.addrList != nil {
+		ann.Direct = c.addrList.ExternalAddresses()
+	}
+
+	if c.relayStat != nil {
+		for _, relay := range c.relayStat.Relays() {
+			latency, ok := c.relayStat.RelayStatus(relay)
+			if ok {
+				ann.Relays = append(ann.Relays, Relay{
+					URL:     relay,
+					Latency: int32(latency / time.Millisecond),
+				})
+			}
+		}
+	}
+
+	if len(ann.Direct)+len(ann.Relays) == 0 {
+		c.setError(errors.New("nothing to announce"))
+		if debug {
+			l.Debugln("Nothing to announce")
+		}
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+
+	// The marshal doesn't fail, I promise.
+	postData, _ := json.Marshal(ann)
+
+	if debug {
+		l.Debugf("Announcement: %s", postData)
+	}
+
+	resp, err := c.announceClient.Post(c.server, "application/json", bytes.NewReader(postData))
+	if err != nil {
+		if debug {
+			l.Debugln("announce POST:", err)
+		}
+		c.setError(err)
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+	if debug {
+		l.Debugln("announce POST:", resp.Status)
+	}
+	resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		if debug {
+			l.Debugln("announce POST:", resp.Status)
+		}
+		c.setError(errors.New(resp.Status))
+
+		if h := resp.Header.Get("Retry-After"); h != "" {
+			// The server has a recommendation on when we should
+			// retry. Follow it.
+			if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
+				if debug {
+					l.Debugln("announce Retry-After:", secs, err)
+				}
+				timer.Reset(time.Duration(secs) * time.Second)
+				return
+			}
+		}
+
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+
+	c.setError(nil)
+
+	if h := resp.Header.Get("Reannounce-After"); h != "" {
+		// The server has a recommendation on when we should
+		// reannounce. Follow it.
+		if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
+			if debug {
+				l.Debugln("announce Reannounce-After:", secs, err)
+			}
+			timer.Reset(time.Duration(secs) * time.Second)
+			return
+		}
+	}
+
+	timer.Reset(defaultReannounceInterval)
+}
+
+func (c *globalClient) Stop() {
+	close(c.stop)
+}
+
+func (c *globalClient) Cache() map[protocol.DeviceID]CacheEntry {
+	// The globalClient doesn't do caching
+	return nil
+}
+
+// parseOptions parses and strips away any ?query=val options, setting the
+// corresponding field in the serverOptions struct. Unknown query options are
+// ignored and removed.
+func parseOptions(dsn string) (server string, opts serverOptions, err error) {
+	p, err := url.Parse(dsn)
+	if err != nil {
+		return "", serverOptions{}, err
+	}
+
+	// Grab known options from the query string
+	q := p.Query()
+	opts.id = q.Get("id")
+	opts.insecure = opts.id != "" || queryBool(q, "insecure")
+	opts.noAnnounce = queryBool(q, "noannounce")
+
+	// Check for disallowed combinations
+	if p.Scheme == "http" {
+		if !opts.insecure {
+			return "", serverOptions{}, errors.New("http without insecure not supported")
+		}
+		if !opts.noAnnounce {
+			return "", serverOptions{}, errors.New("http without noannounce not supported")
+		}
+	} else if p.Scheme != "https" {
+		return "", serverOptions{}, errors.New("unsupported scheme " + p.Scheme)
+	}
+
+	// Remove the query string
+	p.RawQuery = ""
+	server = p.String()
+
+	return
+}
+
+// queryBool returns the query parameter parsed as a boolean. An empty value
+// ("?foo") is considered true, as is any value string except false
+// ("?foo=false").
+func queryBool(q url.Values, key string) bool {
+	if _, ok := q[key]; !ok {
+		return false
+	}
+
+	return q.Get(key) != "false"
+}
+
+type idCheckingHTTPClient struct {
+	httpClient
+	id protocol.DeviceID
+}
+
+func newIDCheckingHTTPClient(client httpClient, id protocol.DeviceID) *idCheckingHTTPClient {
+	return &idCheckingHTTPClient{
+		httpClient: client,
+		id:         id,
+	}
+}
+
+func (c *idCheckingHTTPClient) check(resp *http.Response) error {
+	if resp.TLS == nil {
+		return errors.New("security: not TLS")
+	}
+
+	if len(resp.TLS.PeerCertificates) == 0 {
+		return errors.New("security: no certificates")
+	}
+
+	id := protocol.NewDeviceID(resp.TLS.PeerCertificates[0].Raw)
+	if !id.Equals(c.id) {
+		return errors.New("security: incorrect device id")
+	}
+
+	return nil
+}
+
+func (c *idCheckingHTTPClient) Get(url string) (*http.Response, error) {
+	resp, err := c.httpClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if err := c.check(resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+func (c *idCheckingHTTPClient) Post(url, ctype string, data io.Reader) (*http.Response, error) {
+	resp, err := c.httpClient.Post(url, ctype, data)
+	if err != nil {
+		return nil, err
+	}
+	if err := c.check(resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+type errorHolder struct {
+	err error
+	mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
+}
+
+func (e *errorHolder) setError(err error) {
+	e.mut.Lock()
+	e.err = err
+	e.mut.Unlock()
+}
+
+func (e *errorHolder) Error() error {
+	e.mut.Lock()
+	err := e.err
+	e.mut.Unlock()
+	return err
+}

lib/discover/global_test.go

@@ -0,0 +1,259 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"crypto/tls"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+)
+
+func TestParseOptions(t *testing.T) {
+	testcases := []struct {
+		in   string
+		out  string
+		opts serverOptions
+	}{
+		{"https://example.com/", "https://example.com/", serverOptions{}},
+		{"https://example.com/?insecure", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=true", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=yes", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=false&noannounce", "https://example.com/", serverOptions{noAnnounce: true}},
+		{"https://example.com/?id=abc", "https://example.com/", serverOptions{id: "abc", insecure: true}},
+	}
+
+	for _, tc := range testcases {
+		res, opts, err := parseOptions(tc.in)
+		if err != nil {
+			t.Errorf("Unexpected err %v for %v", err, tc.in)
+			continue
+		}
+		if res != tc.out {
+			t.Errorf("Incorrect server, %v!= %v for %v", res, tc.out, tc.in)
+		}
+		if opts != tc.opts {
+			t.Errorf("Incorrect options, %v!= %v for %v", opts, tc.opts, tc.in)
+		}
+	}
+}
+
+func TestGlobalOverHTTP(t *testing.T) {
+	// HTTP works for queries, but is obviously insecure and we can't do
+	// announces over it (as we don't present a certificate). As such, http://
+	// is only allowed in combination with the "insecure" and "noannounce"
+	// parameters.
+
+	if _, err := NewGlobal("http://192.0.2.42/", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without insecure and noannounce")
+	}
+
+	if _, err := NewGlobal("http://192.0.2.42/?insecure", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without noannounce")
+	}
+
+	if _, err := NewGlobal("http://192.0.2.42/?noannounce", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without insecure")
+	}
+
+	// Now lets check that lookups work over HTTP, given the correct options.
+
+	list, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	direct, relays, err := testLookup("http://" + list.Addr().String() + "?insecure&noannounce")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+		t.Errorf("incorrect direct list: %+v", direct)
+	}
+	if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+		t.Errorf("incorrect relays list: %+v", direct)
+	}
+}
+
+func TestGlobalOverHTTPS(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate a server certificate, using fewer bits than usual to hurry the
+	// process along a bit.
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	// With default options the lookup code expects the server certificate to
+	// check out according to the usual CA chains etc. That won't be the case
+	// here so we expect the lookup to fail.
+
+	url := "https://" + list.Addr().String()
+	if _, _, err := testLookup(url); err == nil {
+		t.Fatalf("unexpected nil error when we should have got a certificate error")
+	}
+
+	// With "insecure" set, whatever certificate is on the other side should
+	// be accepted.
+
+	url = "https://" + list.Addr().String() + "?insecure"
+	if direct, relays, err := testLookup(url); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	} else {
+		if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+			t.Errorf("incorrect direct list: %+v", direct)
+		}
+		if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+			t.Errorf("incorrect relays list: %+v", direct)
+		}
+	}
+
+	// With "id" set to something incorrect, the checks should fail again.
+
+	url = "https://" + list.Addr().String() + "?id=" + protocol.LocalDeviceID.String()
+	if _, _, err := testLookup(url); err == nil {
+		t.Fatalf("unexpected nil error for incorrect discovery server ID")
+	}
+
+	// With the correct device ID, the check should pass and we should get a
+	// lookup response.
+
+	id := protocol.NewDeviceID(cert.Certificate[0])
+	url = "https://" + list.Addr().String() + "?id=" + id.String()
+	if direct, relays, err := testLookup(url); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	} else {
+		if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+			t.Errorf("incorrect direct list: %+v", direct)
+		}
+		if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+			t.Errorf("incorrect relays list: %+v", direct)
+		}
+	}
+}
+
+func TestGlobalAnnounce(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate a server certificate, using fewer bits than usual to hurry the
+	// process along a bit.
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	url := "https://" + list.Addr().String() + "?insecure"
+	disco, err := NewGlobal(url, cert, new(fakeAddressLister), new(fakeRelayStatus))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go disco.Serve()
+	defer disco.Stop()
+
+	// The discovery thing should attempt an announcement immediately. We wait
+	// for it to succeed, a while.
+	t0 := time.Now()
+	for err := disco.Error(); err != nil; err = disco.Error() {
+		if time.Since(t0) > 10*time.Second {
+			t.Fatal("announce failed:", err)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	if !strings.Contains(string(s.announce), "tcp://0.0.0.0:22000") {
+		t.Errorf("announce missing direct address: %s", s.announce)
+	}
+	if !strings.Contains(string(s.announce), "relay://192.0.2.42:443") {
+		t.Errorf("announce missing relay address: %s", s.announce)
+	}
+}
+
+func testLookup(url string) ([]string, []Relay, error) {
+	disco, err := NewGlobal(url, tls.Certificate{}, nil, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	go disco.Serve()
+	defer disco.Stop()
+
+	return disco.Lookup(protocol.LocalDeviceID)
+}
+
+type fakeDiscoveryServer struct {
+	announce []byte
+}
+
+func (s *fakeDiscoveryServer) handler(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "POST" {
+		s.announce, _ = ioutil.ReadAll(r.Body)
+		w.WriteHeader(204)
+	} else {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"direct":["tcp://192.0.2.42::22000"], "relays":[{"url": "relay://192.0.2.43:443", "latency": 42}]}`))
+	}
+}
+
+type fakeAddressLister struct{}
+
+func (f *fakeAddressLister) ExternalAddresses() []string {
+	return []string{"tcp://0.0.0.0:22000"}
+}
+func (f *fakeAddressLister) AllAddresses() []string {
+	return []string{"tcp://0.0.0.0:22000", "tcp://192.168.0.1:22000"}
+}
+
+type fakeRelayStatus struct{}
+
+func (f *fakeRelayStatus) Relays() []string {
+	return []string{"relay://192.0.2.42:443"}
+}
+func (f *fakeRelayStatus) RelayStatus(uri string) (time.Duration, bool) {
+	return 42 * time.Millisecond, true
+}

lib/discover/local.go

@@ -0,0 +1,270 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/beacon"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/thejerf/suture"
+)
+
+type localClient struct {
+	*suture.Supervisor
+	myID      protocol.DeviceID
+	addrList  AddressLister
+	relayStat RelayStatusProvider
+	name      string
+
+	beacon          beacon.Interface
+	localBcastStart time.Time
+	localBcastTick  <-chan time.Time
+	forcedBcastTick chan time.Time
+
+	*cache
+}
+
+const (
+	BroadcastInterval = 30 * time.Second
+	CacheLifeTime     = 3 * BroadcastInterval
+)
+
+var (
+	ErrIncorrectMagic = errors.New("incorrect magic number")
+)
+
+func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
+	c := &localClient{
+		Supervisor:      suture.NewSimple("local"),
+		myID:            id,
+		addrList:        addrList,
+		relayStat:       relayStat,
+		localBcastTick:  time.Tick(BroadcastInterval),
+		forcedBcastTick: make(chan time.Time),
+		localBcastStart: time.Now(),
+		cache:           newCache(),
+	}
+
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(host) == 0 {
+		// A broadcast client
+		c.name = "IPv4 local"
+		bcPort, err := strconv.Atoi(port)
+		if err != nil {
+			return nil, err
+		}
+		c.startLocalIPv4Broadcasts(bcPort)
+	} else {
+		// A multicast client
+		c.name = "IPv6 local"
+		c.startLocalIPv6Multicasts(addr)
+	}
+
+	go c.sendLocalAnnouncements()
+
+	return c, nil
+}
+
+func (c *localClient) startLocalIPv4Broadcasts(localPort int) {
+	c.beacon = beacon.NewBroadcast(localPort)
+	c.Add(c.beacon)
+	go c.recvAnnouncements(c.beacon)
+}
+
+func (c *localClient) startLocalIPv6Multicasts(localMCAddr string) {
+	c.beacon = beacon.NewMulticast(localMCAddr)
+	c.Add(c.beacon)
+	go c.recvAnnouncements(c.beacon)
+}
+
+// Lookup returns a list of addresses the device is available at. Local
+// discovery never returns relays.
+func (c *localClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	if cache, ok := c.Get(device); ok {
+		if time.Since(cache.when) < CacheLifeTime {
+			direct = cache.Direct
+			relays = cache.Relays
+		}
+	}
+
+	return
+}
+
+func (c *localClient) String() string {
+	return c.name
+}
+
+func (c *localClient) Error() error {
+	return c.beacon.Error()
+}
+
+func (c *localClient) announcementPkt() Announce {
+	addrs := c.addrList.AllAddresses()
+
+	var relays []Relay
+	for _, relay := range c.relayStat.Relays() {
+		latency, ok := c.relayStat.RelayStatus(relay)
+		if ok {
+			relays = append(relays, Relay{
+				URL:     relay,
+				Latency: int32(latency / time.Millisecond),
+			})
+		}
+	}
+
+	return Announce{
+		Magic: AnnouncementMagic,
+		This: Device{
+			ID:        c.myID[:],
+			Addresses: addrs,
+			Relays:    relays,
+		},
+	}
+}
+
+func (c *localClient) sendLocalAnnouncements() {
+	var pkt = c.announcementPkt()
+	msg := pkt.MustMarshalXDR()
+
+	for {
+		c.beacon.Send(msg)
+
+		select {
+		case <-c.localBcastTick:
+		case <-c.forcedBcastTick:
+		}
+	}
+}
+
+func (c *localClient) recvAnnouncements(b beacon.Interface) {
+	for {
+		buf, addr := b.Recv()
+
+		var pkt Announce
+		err := pkt.UnmarshalXDR(buf)
+		if err != nil && err != io.EOF {
+			if debug {
+				l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
+			}
+			continue
+		}
+
+		if debug {
+			l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
+		}
+
+		var newDevice bool
+		if bytes.Compare(pkt.This.ID, c.myID[:]) != 0 {
+			newDevice = c.registerDevice(addr, pkt.This)
+		}
+
+		if newDevice {
+			select {
+			case c.forcedBcastTick <- time.Now():
+			}
+		}
+	}
+}
+
+func (c *localClient) registerDevice(src net.Addr, device Device) bool {
+	var id protocol.DeviceID
+	copy(id[:], device.ID)
+
+	// Remember whether we already had a valid cache entry for this device.
+
+	ce, existsAlready := c.Get(id)
+	isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime
+
+	// Any empty or unspecified addresses should be set to the source address
+	// of the announcement. We also skip any addresses we can't parse.
+
+	var validAddresses []string
+	for _, addr := range device.Addresses {
+		u, err := url.Parse(addr)
+		if err != nil {
+			continue
+		}
+
+		tcpAddr, err := net.ResolveTCPAddr("tcp", u.Host)
+		if err != nil {
+			continue
+		}
+
+		if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
+			host, _, err := net.SplitHostPort(src.String())
+			if err != nil {
+				continue
+			}
+			u.Host = fmt.Sprintf("%s:%d", host, tcpAddr.Port)
+			validAddresses = append(validAddresses, u.String())
+		} else {
+			validAddresses = append(validAddresses, addr)
+		}
+	}
+
+	c.Set(id, CacheEntry{
+		Direct: validAddresses,
+		Relays: device.Relays,
+		when:   time.Now(),
+		found:  true,
+	})
+
+	if isNewDevice {
+		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
+			"device": id.String(),
+			"addrs":  device.Addresses,
+			"relays": device.Relays,
+		})
+	}
+
+	return isNewDevice
+}
+
+func addrToAddr(addr *net.TCPAddr) string {
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		return fmt.Sprintf(":%c", addr.Port)
+	} else if bs := addr.IP.To4(); bs != nil {
+		return fmt.Sprintf("%s:%c", bs.String(), addr.Port)
+	} else if bs := addr.IP.To16(); bs != nil {
+		return fmt.Sprintf("[%s]:%c", bs.String(), addr.Port)
+	}
+	return ""
+}
+
+func resolveAddrs(addrs []string) []string {
+	var raddrs []string
+	for _, addrStr := range addrs {
+		uri, err := url.Parse(addrStr)
+		if err != nil {
+			continue
+		}
+		addrRes, err := net.ResolveTCPAddr("tcp", uri.Host)
+		if err != nil {
+			continue
+		}
+		addr := addrToAddr(addrRes)
+		if len(addr) > 0 {
+			uri.Host = addr
+			raddrs = append(raddrs, uri.String())
+		}
+	}
+	return raddrs
+}

lib/discover/localpackets.go

@@ -5,13 +5,13 @@
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
 //go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o packets_xdr.go packets.go
+//go:generate genxdr -o localpackets_xdr.go localpackets.go
 
 package discover
 
 const (
-	AnnouncementMagic = 0x9D79BC39
-	QueryMagic        = 0x2CA856F5
+	AnnouncementMagic = 0x9D79BC40
+	QueryMagic        = 0x2CA856F6
 )
 
 type Query struct {
@@ -25,12 +25,13 @@ type Announce struct {
 	Extra []Device // max:16
 }
 
-type Device struct {
-	ID        []byte    // max:32
-	Addresses []Address // max:16
+type Relay struct {
+	URL     string `json:"url"` // max:2083
+	Latency int32  `json:"latency"`
 }
 
-type Address struct {
-	IP   []byte // max:16
-	Port uint16
+type Device struct {
+	ID        []byte   // max:32
+	Addresses []string // max:16
+	Relays    []Relay  // max:16
 }

lib/discover/localpackets_xdr.go

@@ -187,6 +187,80 @@ func (o *Announce) DecodeXDRFrom(xr *xdr.Reader) error {
 
 /*
 
+Relay Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of URL                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     URL (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                            Latency                            |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Relay {
+	string URL<256>;
+	int Latency;
+}
+
+*/
+
+func (o Relay) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Relay) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Relay) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Relay) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Relay) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	if l := len(o.URL); l > 256 {
+		return xw.Tot(), xdr.ElementSizeExceeded("URL", l, 256)
+	}
+	xw.WriteString(o.URL)
+	xw.WriteUint32(uint32(o.Latency))
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Relay) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Relay) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Relay) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.URL = xr.ReadStringMax(256)
+	o.Latency = int32(xr.ReadUint32())
+	return xr.Error()
+}
+
+/*
+
 Device Structure:
 
  0                   1                   2                   3
@@ -200,15 +274,24 @@ Device Structure:
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                      Number of Addresses                      |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                      Length of Addresses                      |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 /                                                               /
-\                Zero or more Address Structures                \
+\                  Addresses (variable length)                  \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Relays                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                 Zero or more Relay Structures                 \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct Device {
 	opaque ID<32>;
-	Address Addresses<16>;
+	string Addresses<16>;
+	Relay Relays<16>;
 }
 
 */
@@ -247,7 +330,14 @@ func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Addresses)))
 	for i := range o.Addresses {
-		_, err := o.Addresses[i].EncodeXDRInto(xw)
+		xw.WriteString(o.Addresses[i])
+	}
+	if l := len(o.Relays); l > 16 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Relays", l, 16)
+	}
+	xw.WriteUint32(uint32(len(o.Relays)))
+	for i := range o.Relays {
+		_, err := o.Relays[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -275,83 +365,20 @@ func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error {
 	if _AddressesSize > 16 {
 		return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
 	}
-	o.Addresses = make([]Address, _AddressesSize)
+	o.Addresses = make([]string, _AddressesSize)
 	for i := range o.Addresses {
-		(&o.Addresses[i]).DecodeXDRFrom(xr)
+		o.Addresses[i] = xr.ReadString()
+	}
+	_RelaysSize := int(xr.ReadUint32())
+	if _RelaysSize < 0 {
+		return xdr.ElementSizeExceeded("Relays", _RelaysSize, 16)
+	}
+	if _RelaysSize > 16 {
+		return xdr.ElementSizeExceeded("Relays", _RelaysSize, 16)
+	}
+	o.Relays = make([]Relay, _RelaysSize)
+	for i := range o.Relays {
+		(&o.Relays[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
-
-/*
-
-Address Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of IP                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     IP (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |             Port              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Address {
-	opaque IP<16>;
-	unsigned int Port;
-}
-
-*/
-
-func (o Address) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
-}
-
-func (o Address) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o Address) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Address) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
-}
-
-func (o Address) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	if l := len(o.IP); l > 16 {
-		return xw.Tot(), xdr.ElementSizeExceeded("IP", l, 16)
-	}
-	xw.WriteBytes(o.IP)
-	xw.WriteUint16(o.Port)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *Address) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *Address) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *Address) DecodeXDRFrom(xr *xdr.Reader) error {
-	o.IP = xr.ReadBytesMax(16)
-	o.Port = xr.ReadUint16()
-	return xr.Error()
-}

lib/events/events.go

@@ -25,6 +25,8 @@ const (
 	DeviceConnected
 	DeviceDisconnected
 	DeviceRejected
+	DevicePaused
+	DeviceResumed
 	LocalIndexUpdated
 	RemoteIndexUpdated
 	ItemStarted
@@ -36,6 +38,9 @@ const (
 	FolderSummary
 	FolderCompletion
 	FolderErrors
+	FolderScanProgress
+	ExternalPortMappingChanged
+	RelayStateChanged
 
 	AllEvents = (1 << iota) - 1
 )
@@ -78,6 +83,16 @@ func (t EventType) String() string {
 		return "FolderCompletion"
 	case FolderErrors:
 		return "FolderErrors"
+	case DevicePaused:
+		return "DevicePaused"
+	case DeviceResumed:
+		return "DeviceResumed"
+	case FolderScanProgress:
+		return "FolderScanProgress"
+	case ExternalPortMappingChanged:
+		return "ExternalPortMappingChanged"
+	case RelayStateChanged:
+		return "RelayStateChanged"
 	default:
 		return "Unknown"
 	}

lib/fnmatch/fnmatch.go

@@ -27,43 +27,43 @@ func Convert(pattern string, flags int) (*regexp.Regexp, error) {
 		flags |= NoEscape | CaseFold
 		pattern = filepath.FromSlash(pattern)
 		if flags&PathName != 0 {
-			any = "[^\\\\]"
+			any = `[^\\]`
 		}
 	case "darwin":
 		flags |= CaseFold
 		fallthrough
 	default:
 		if flags&PathName != 0 {
-			any = "[^/]"
+			any = `[^/]`
 		}
 	}
 
 	if flags&NoEscape != 0 {
-		pattern = strings.Replace(pattern, "\\", "\\\\", -1)
+		pattern = strings.Replace(pattern, `\`, `\\`, -1)
 	} else {
-		pattern = strings.Replace(pattern, "\\*", "[:escapedstar:]", -1)
-		pattern = strings.Replace(pattern, "\\?", "[:escapedques:]", -1)
-		pattern = strings.Replace(pattern, "\\.", "[:escapeddot:]", -1)
+		pattern = strings.Replace(pattern, `\*`, "[:escapedstar:]", -1)
+		pattern = strings.Replace(pattern, `\?`, "[:escapedques:]", -1)
+		pattern = strings.Replace(pattern, `\.`, "[:escapeddot:]", -1)
 	}
 
 	// Characters that are special in regexps but not in glob, must be
 	// escaped.
-	for _, char := range []string{".", "+", "$", "^", "(", ")", "|"} {
-		pattern = strings.Replace(pattern, char, "\\"+char, -1)
+	for _, char := range []string{`.`, `+`, `$`, `^`, `(`, `)`, `|`} {
+		pattern = strings.Replace(pattern, char, `\`+char, -1)
 	}
 
-	pattern = strings.Replace(pattern, "**", "[:doublestar:]", -1)
-	pattern = strings.Replace(pattern, "*", any+"*", -1)
-	pattern = strings.Replace(pattern, "[:doublestar:]", ".*", -1)
-	pattern = strings.Replace(pattern, "?", any, -1)
+	pattern = strings.Replace(pattern, `**`, `[:doublestar:]`, -1)
+	pattern = strings.Replace(pattern, `*`, any+`*`, -1)
+	pattern = strings.Replace(pattern, `[:doublestar:]`, `.*`, -1)
+	pattern = strings.Replace(pattern, `?`, any, -1)
 
-	pattern = strings.Replace(pattern, "[:escapedstar:]", "\\*", -1)
-	pattern = strings.Replace(pattern, "[:escapedques:]", "\\?", -1)
-	pattern = strings.Replace(pattern, "[:escapeddot:]", "\\.", -1)
+	pattern = strings.Replace(pattern, `[:escapedstar:]`, `\*`, -1)
+	pattern = strings.Replace(pattern, `[:escapedques:]`, `\?`, -1)
+	pattern = strings.Replace(pattern, `[:escapeddot:]`, `\.`, -1)
 
-	pattern = "^" + pattern + "$"
+	pattern = `^` + pattern + `$`
 	if flags&CaseFold != 0 {
-		pattern = "(?i)" + pattern
+		pattern = `(?i)` + pattern
 	}
 	return regexp.Compile(pattern)
 }

lib/ignore/ignore.go

@@ -12,6 +12,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -236,6 +237,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
 			includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
 			includes, err := loadIgnoreFile(includeFile, seen)
 			if err != nil {
+				log.Println(err)
 				return err
 			}
 			patterns = append(patterns, includes...)

lib/model/connection.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"crypto/tls"
+	"net"
+
+	"github.com/syncthing/protocol"
+)
+
+type IntermediateConnection struct {
+	*tls.Conn
+	Type ConnectionType
+}
+
+type Connection struct {
+	net.Conn
+	protocol.Connection
+	Type ConnectionType
+}
+
+const (
+	ConnectionTypeDirectAccept ConnectionType = iota
+	ConnectionTypeDirectDial
+	ConnectionTypeRelayAccept
+	ConnectionTypeRelayDial
+)
+
+type ConnectionType int
+
+func (t ConnectionType) String() string {
+	switch t {
+	case ConnectionTypeDirectAccept:
+		return "direct-accept"
+	case ConnectionTypeDirectDial:
+		return "direct-dial"
+	case ConnectionTypeRelayAccept:
+		return "relay-accept"
+	case ConnectionTypeRelayDial:
+		return "relay-dial"
+	}
+	return "unknown"
+}
+
+func (t ConnectionType) IsDirect() bool {
+	return t == ConnectionTypeDirectAccept || t == ConnectionTypeDirectDial
+}

lib/model/model.go

@@ -87,10 +87,10 @@ type Model struct {
 	folderStatRefs map[string]*stats.FolderStatisticsReference            // folder -> statsRef
 	fmut           sync.RWMutex                                           // protects the above
 
-	protoConn map[protocol.DeviceID]protocol.Connection
-	rawConn   map[protocol.DeviceID]io.Closer
-	deviceVer map[protocol.DeviceID]string
-	pmut      sync.RWMutex // protects protoConn and rawConn
+	conn         map[protocol.DeviceID]Connection
+	deviceVer    map[protocol.DeviceID]string
+	devicePaused map[protocol.DeviceID]bool
+	pmut         sync.RWMutex // protects the above
 
 	reqValidationCache map[string]time.Time // folder / file name => time when confirmed to exist
 	rvmut              sync.RWMutex         // protects reqValidationCache
@@ -114,7 +114,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		}),
 		cfg:                cfg,
 		db:                 ldb,
-		finder:             db.NewBlockFinder(ldb, cfg),
+		finder:             db.NewBlockFinder(ldb),
 		progressEmitter:    NewProgressEmitter(cfg),
 		id:                 id,
 		shortID:            id.Short(),
@@ -130,9 +130,9 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		folderIgnores:      make(map[string]*ignore.Matcher),
 		folderRunners:      make(map[string]service),
 		folderStatRefs:     make(map[string]*stats.FolderStatisticsReference),
-		protoConn:          make(map[protocol.DeviceID]protocol.Connection),
-		rawConn:            make(map[protocol.DeviceID]io.Closer),
+		conn:               make(map[protocol.DeviceID]Connection),
 		deviceVer:          make(map[protocol.DeviceID]string),
+		devicePaused:       make(map[protocol.DeviceID]bool),
 		reqValidationCache: make(map[string]time.Time),
 
 		fmut:  sync.NewRWMutex(),
@@ -219,8 +219,11 @@ func (m *Model) StartFolderRO(folder string) {
 
 type ConnectionInfo struct {
 	protocol.Statistics
+	Connected     bool
+	Paused        bool
 	Address       string
 	ClientVersion string
+	Type          ConnectionType
 }
 
 func (info ConnectionInfo) MarshalJSON() ([]byte, error) {
@@ -228,8 +231,11 @@ func (info ConnectionInfo) MarshalJSON() ([]byte, error) {
 		"at":            info.At,
 		"inBytesTotal":  info.InBytesTotal,
 		"outBytesTotal": info.OutBytesTotal,
+		"connected":     info.Connected,
+		"paused":        info.Paused,
 		"address":       info.Address,
 		"clientVersion": info.ClientVersion,
+		"type":          info.Type.String(),
 	})
 }
 
@@ -242,15 +248,21 @@ func (m *Model) ConnectionStats() map[string]interface{} {
 	m.pmut.RLock()
 	m.fmut.RLock()
 
-	var res = make(map[string]interface{})
-	conns := make(map[string]ConnectionInfo, len(m.protoConn))
-	for device, conn := range m.protoConn {
+	res := make(map[string]interface{})
+	devs := m.cfg.Devices()
+	conns := make(map[string]ConnectionInfo, len(devs))
+	for device := range devs {
 		ci := ConnectionInfo{
-			Statistics:    conn.Statistics(),
 			ClientVersion: m.deviceVer[device],
+			Paused:        m.devicePaused[device],
 		}
-		if nc, ok := m.rawConn[device].(remoteAddrer); ok {
-			ci.Address = nc.RemoteAddr().String()
+		if conn, ok := m.conn[device]; ok {
+			ci.Type = conn.Type
+			ci.Connected = ok
+			ci.Statistics = conn.Statistics()
+			if addr := conn.RemoteAddr(); addr != nil {
+				ci.Address = addr.String()
+			}
 		}
 
 		conns[device.String()] = ci
@@ -586,8 +598,12 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 		"clientVersion": cm.ClientVersion,
 	}
 
-	if conn, ok := m.rawConn[deviceID].(*tls.Conn); ok {
-		event["addr"] = conn.RemoteAddr().String()
+	if conn, ok := m.conn[deviceID]; ok {
+		event["type"] = conn.Type.String()
+		addr := conn.RemoteAddr()
+		if addr != nil {
+			event["addr"] = addr.String()
+		}
 	}
 
 	m.pmut.Unlock()
@@ -693,12 +709,11 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 	}
 	m.fmut.RUnlock()
 
-	conn, ok := m.rawConn[device]
+	conn, ok := m.conn[device]
 	if ok {
 		closeRawConn(conn)
 	}
-	delete(m.protoConn, device)
-	delete(m.rawConn, device)
+	delete(m.conn, device)
 	delete(m.deviceVer, device)
 	m.pmut.Unlock()
 }
@@ -860,7 +875,7 @@ func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
 // ConnectedTo returns true if we are connected to the named device.
 func (m *Model) ConnectedTo(deviceID protocol.DeviceID) bool {
 	m.pmut.RLock()
-	_, ok := m.protoConn[deviceID]
+	_, ok := m.conn[deviceID]
 	m.pmut.RUnlock()
 	if ok {
 		m.deviceWasSeen(deviceID)
@@ -906,7 +921,9 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 		return fmt.Errorf("Folder %s does not exist", folder)
 	}
 
-	fd, err := osutil.CreateAtomic(filepath.Join(cfg.Path(), ".stignore"), 0644)
+	path := filepath.Join(cfg.Path(), ".stignore")
+
+	fd, err := osutil.CreateAtomic(path, 0644)
 	if err != nil {
 		l.Warnln("Saving .stignore:", err)
 		return err
@@ -920,6 +937,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 		l.Warnln("Saving .stignore:", err)
 		return err
 	}
+	osutil.HideFile(path)
 
 	return m.ScanFolder(folder)
 }
@@ -927,28 +945,24 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 // AddConnection adds a new peer connection to the model. An initial index will
 // be sent to the connected peer, thereafter index updates whenever the local
 // folder changes.
-func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
-	deviceID := protoConn.ID()
+func (m *Model) AddConnection(conn Connection) {
+	deviceID := conn.ID()
 
 	m.pmut.Lock()
-	if _, ok := m.protoConn[deviceID]; ok {
+	if _, ok := m.conn[deviceID]; ok {
 		panic("add existing device")
 	}
-	m.protoConn[deviceID] = protoConn
-	if _, ok := m.rawConn[deviceID]; ok {
-		panic("add existing device")
-	}
-	m.rawConn[deviceID] = rawConn
+	m.conn[deviceID] = conn
 
-	protoConn.Start()
+	conn.Start()
 
 	cm := m.clusterConfig(deviceID)
-	protoConn.ClusterConfig(cm)
+	conn.ClusterConfig(cm)
 
 	m.fmut.RLock()
 	for _, folder := range m.deviceFolders[deviceID] {
 		fs := m.folderFiles[folder]
-		go sendIndexes(protoConn, folder, fs, m.folderIgnores[folder])
+		go sendIndexes(conn, folder, fs, m.folderIgnores[folder])
 	}
 	m.fmut.RUnlock()
 	m.pmut.Unlock()
@@ -956,6 +970,31 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	m.deviceWasSeen(deviceID)
 }
 
+func (m *Model) PauseDevice(device protocol.DeviceID) {
+	m.pmut.Lock()
+	m.devicePaused[device] = true
+	_, ok := m.conn[device]
+	m.pmut.Unlock()
+	if ok {
+		m.Close(device, errors.New("device paused"))
+	}
+	events.Default.Log(events.DevicePaused, map[string]string{"device": device.String()})
+}
+
+func (m *Model) ResumeDevice(device protocol.DeviceID) {
+	m.pmut.Lock()
+	m.devicePaused[device] = false
+	m.pmut.Unlock()
+	events.Default.Log(events.DeviceResumed, map[string]string{"device": device.String()})
+}
+
+func (m *Model) IsPaused(device protocol.DeviceID) bool {
+	m.pmut.Lock()
+	paused := m.devicePaused[device]
+	m.pmut.Unlock()
+	return paused
+}
+
 func (m *Model) deviceStatRef(deviceID protocol.DeviceID) *stats.DeviceStatisticsReference {
 	m.fmut.Lock()
 	defer m.fmut.Unlock()
@@ -964,7 +1003,7 @@ func (m *Model) deviceStatRef(deviceID protocol.DeviceID) *stats.DeviceStatistic
 		return sr
 	}
 
-	sr := stats.NewDeviceStatisticsReference(m.db, deviceID)
+	sr := stats.NewDeviceStatisticsReference(m.db, deviceID.String())
 	m.deviceStatRefs[deviceID] = sr
 	return sr
 }
@@ -986,7 +1025,7 @@ func (m *Model) folderStatRef(folder string) *stats.FolderStatisticsReference {
 }
 
 func (m *Model) receivedFile(folder string, file protocol.FileInfo) {
-	m.folderStatRef(folder).ReceivedFile(file)
+	m.folderStatRef(folder).ReceivedFile(file.Name, file.IsDeleted())
 }
 
 func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) {
@@ -1114,7 +1153,7 @@ func (m *Model) updateLocals(folder string, fs []protocol.FileInfo) {
 
 func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) {
 	m.pmut.RLock()
-	nc, ok := m.protoConn[deviceID]
+	nc, ok := m.conn[deviceID]
 	m.pmut.RUnlock()
 
 	if !ok {
@@ -1261,18 +1300,20 @@ nextSub:
 	subs = unifySubs
 
 	w := &scanner.Walker{
-		Dir:           folderCfg.Path(),
-		Subs:          subs,
-		Matcher:       ignores,
-		BlockSize:     protocol.BlockSize,
-		TempNamer:     defTempNamer,
-		TempLifetime:  time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
-		CurrentFiler:  cFiler{m, folder},
-		MtimeRepo:     db.NewVirtualMtimeRepo(m.db, folderCfg.ID),
-		IgnorePerms:   folderCfg.IgnorePerms,
-		AutoNormalize: folderCfg.AutoNormalize,
-		Hashers:       m.numHashers(folder),
-		ShortID:       m.shortID,
+		Folder:                folderCfg.ID,
+		Dir:                   folderCfg.Path(),
+		Subs:                  subs,
+		Matcher:               ignores,
+		BlockSize:             protocol.BlockSize,
+		TempNamer:             defTempNamer,
+		TempLifetime:          time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
+		CurrentFiler:          cFiler{m, folder},
+		MtimeRepo:             db.NewVirtualMtimeRepo(m.db, folderCfg.ID),
+		IgnorePerms:           folderCfg.IgnorePerms,
+		AutoNormalize:         folderCfg.AutoNormalize,
+		Hashers:               m.numHashers(folder),
+		ShortID:               m.shortID,
+		ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
 	}
 
 	runner.setState(FolderScanning)
@@ -1648,7 +1689,7 @@ func (m *Model) Availability(folder, file string) []protocol.DeviceID {
 
 	availableDevices := []protocol.DeviceID{}
 	for _, device := range fs.Availability(file) {
-		_, ok := m.protoConn[device]
+		_, ok := m.conn[device]
 		if ok {
 			availableDevices = append(availableDevices, device)
 		}
@@ -1772,7 +1813,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 			// folder.
 			m.pmut.Lock()
 			for _, dev := range cfg.DeviceIDs() {
-				if conn, ok := m.rawConn[dev]; ok {
+				if conn, ok := m.conn[dev]; ok {
 					closeRawConn(conn)
 				}
 			}
@@ -1820,7 +1861,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 				// disconnect it so that we start sharing the folder with it.
 				// We close the underlying connection and let the normal error
 				// handling kick in to clean up and reconnect.
-				if conn, ok := m.rawConn[dev]; ok {
+				if conn, ok := m.conn[dev]; ok {
 					closeRawConn(conn)
 				}
 

lib/model/model_test.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -281,7 +282,11 @@ func BenchmarkRequest(b *testing.B) {
 		id:          device1,
 		requestData: []byte("some data to return"),
 	}
-	m.AddConnection(fc, fc)
+	m.AddConnection(Connection{
+		&net.TCPConn{},
+		fc,
+		ConnectionTypeDirectAccept,
+	})
 	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
@@ -314,6 +319,18 @@ func TestDeviceRename(t *testing.T) {
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+
+	fc := FakeConnection{
+		id:          device1,
+		requestData: []byte("some data to return"),
+	}
+
+	m.AddConnection(Connection{
+		&net.TCPConn{},
+		fc,
+		ConnectionTypeDirectAccept,
+	})
+
 	m.ServeBackground()
 	if cfg.Devices()[device1].Name != "" {
 		t.Errorf("Device already has a name")

lib/model/rwfolder.go

@@ -212,6 +212,12 @@ func (p *rwFolder) Serve() {
 				continue
 			}
 
+			if err := p.model.CheckFolderHealth(p.folder); err != nil {
+				l.Infoln("Skipping folder", p.folder, "pull due to folder error:", err)
+				p.pullTimer.Reset(nextPullIntv)
+				continue
+			}
+
 			p.model.fmut.RLock()
 			curIgnores := p.model.folderIgnores[p.folder]
 			p.model.fmut.RUnlock()
@@ -451,7 +457,6 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 	// !!!
 
 	changed := 0
-	pullFileSize := int64(0)
 
 	fileDeletions := map[string]protocol.FileInfo{}
 	dirDeletions := []protocol.FileInfo{}
@@ -500,7 +505,6 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 		default:
 			// A new or changed file or symlink. This is the only case where we
 			// do stuff concurrently in the background
-			pullFileSize += file.Size()
 			p.queue.Push(file.Name, file.Size(), file.Modified)
 		}
 
@@ -508,16 +512,6 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 		return true
 	})
 
-	// Check if we are able to store all files on disk. Only perform this
-	// check if there is a minimum free space threshold set on the folder.
-	folderCfg := p.model.cfg.Folders()[p.folder]
-	if folderCfg.MinDiskFreePct > 0 || pullFileSize > 0 {
-		if free, err := osutil.DiskFreeBytes(folderCfg.Path()); err == nil && free < pullFileSize {
-			l.Warnf(`Folder "%s": insufficient disk space available to pull %d files (%.2f MiB)`, p.folder, changed, float64(pullFileSize)/1024/1024)
-			return 0
-		}
-	}
-
 	// Reorder the file queue according to configuration
 
 	switch p.order {
@@ -970,6 +964,12 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		return
 	}
 
+	if free, err := osutil.DiskFreeBytes(p.dir); err == nil && free < file.Size() {
+		l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, p.dir, file.Name, float64(free)/1024/1024, float64(file.Size())/1024/1024)
+		p.newError(file.Name, errors.New("insufficient space"))
+		return
+	}
+
 	events.Default.Log(events.ItemStarted, map[string]string{
 		"folder": p.folder,
 		"item":   file.Name,
@@ -988,7 +988,7 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 
 	// Check for an old temporary file which might have some blocks we could
 	// reuse.
-	tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize)
+	tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize, 0, nil)
 	if err == nil {
 		// Check for any reusable blocks in the temp file
 		tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
@@ -1080,7 +1080,11 @@ func (p *rwFolder) shortcutFile(file protocol.FileInfo) error {
 
 // shortcutSymlink changes the symlinks type if necessary.
 func (p *rwFolder) shortcutSymlink(file protocol.FileInfo) (err error) {
-	err = symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
+	tt := symlinks.TargetFile
+	if file.IsDirectory() {
+		tt = symlinks.TargetDirectory
+	}
+	err = symlinks.ChangeType(filepath.Join(p.dir, file.Name), tt)
 	if err != nil {
 		l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
 		p.newError(file.Name, err)
@@ -1106,15 +1110,17 @@ func (p *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
 		}
 
 		folderRoots := make(map[string]string)
+		var folders []string
 		p.model.fmut.RLock()
 		for folder, cfg := range p.model.folderCfgs {
 			folderRoots[folder] = cfg.Path()
+			folders = append(folders, folder)
 		}
 		p.model.fmut.RUnlock()
 
 		for _, block := range state.blocks {
 			buf = buf[:int(block.Size)]
-			found := p.model.finder.Iterate(block.Hash, func(folder, file string, index int32) bool {
+			found := p.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
 				fd, err := os.Open(filepath.Join(folderRoots[folder], file))
 				if err != nil {
 					return false
@@ -1314,7 +1320,11 @@ func (p *rwFolder) performFinish(state *sharedPullerState) error {
 		// Remove the file, and replace it with a symlink.
 		err = osutil.InWritableDir(func(path string) error {
 			os.Remove(path)
-			return symlinks.Create(path, string(content), state.file.Flags)
+			tt := symlinks.TargetFile
+			if state.file.IsDirectory() {
+				tt = symlinks.TargetDirectory
+			}
+			return symlinks.Create(path, string(content), tt)
 		}, state.realName)
 		if err != nil {
 			return err

lib/model/rwfolder_test.go

@@ -42,6 +42,8 @@ var blocks = []protocol.BlockInfo{
 	{Offset: 917504, Size: 0x20000, Hash: []uint8{0x96, 0x6b, 0x15, 0x6b, 0xc4, 0xf, 0x19, 0x18, 0xca, 0xbb, 0x5f, 0xd6, 0xbb, 0xa2, 0xc6, 0x2a, 0xac, 0xbb, 0x8a, 0xb9, 0xce, 0xec, 0x4c, 0xdb, 0x78, 0xec, 0x57, 0x5d, 0x33, 0xf9, 0x8e, 0xaf}},
 }
 
+var folders = []string{"default"}
+
 // Layout of the files: (indexes from the above array)
 // 12345678 - Required file
 // 02005008 - Existing file (currently in the index)
@@ -197,7 +199,7 @@ func TestCopierFinder(t *testing.T) {
 
 	// Verify that the blocks we say exist on file, really exist in the db.
 	for _, idx := range []int{2, 3, 4, 7} {
-		if m.finder.Iterate(blocks[idx].Hash, iterFn) == false {
+		if m.finder.Iterate(folders, blocks[idx].Hash, iterFn) == false {
 			t.Error("Didn't find block")
 		}
 	}
@@ -241,7 +243,7 @@ func TestCopierFinder(t *testing.T) {
 	}
 
 	// Verify that the fetched blocks have actually been written to the temp file
-	blks, err := scanner.HashFile(tempFile, protocol.BlockSize)
+	blks, err := scanner.HashFile(tempFile, protocol.BlockSize, 0, nil)
 	if err != nil {
 		t.Log(err)
 	}
@@ -277,7 +279,7 @@ func TestCopierCleanup(t *testing.T) {
 	// Add file to index
 	m.updateLocals("default", []protocol.FileInfo{file})
 
-	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 
@@ -286,11 +288,11 @@ func TestCopierCleanup(t *testing.T) {
 	// Update index (removing old blocks)
 	m.updateLocals("default", []protocol.FileInfo{file})
 
-	if m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Unexpected block found")
 	}
 
-	if !m.finder.Iterate(blocks[1].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 
@@ -299,11 +301,11 @@ func TestCopierCleanup(t *testing.T) {
 	// Update index (removing old blocks)
 	m.updateLocals("default", []protocol.FileInfo{file})
 
-	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Unexpected block found")
 	}
 
-	if m.finder.Iterate(blocks[1].Hash, iterFn) {
+	if m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 }
@@ -334,7 +336,7 @@ func TestLastResortPulling(t *testing.T) {
 	}
 
 	// Check that that particular block is there
-	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 
@@ -361,11 +363,11 @@ func TestLastResortPulling(t *testing.T) {
 	<-pullChan
 
 	// Verify that it did fix the incorrect hash.
-	if m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Found unexpected block")
 	}
 
-	if !m.finder.Iterate(scanner.SHA256OfNothing, iterFn) {
+	if !m.finder.Iterate(folders, scanner.SHA256OfNothing, iterFn) {
 		t.Error("Expected block not found")
 	}
 

lib/osutil/osutil.go

@@ -11,10 +11,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/calmh/du"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -221,3 +223,21 @@ func DiskFreePercentage(path string) (freePct float64, err error) {
 	u, err := du.Get(path)
 	return (float64(u.FreeBytes) / float64(u.TotalBytes)) * 100, err
 }
+
+// SetTCPOptions sets syncthings default TCP options on a TCP connection
+func SetTCPOptions(conn *net.TCPConn) error {
+	var err error
+	if err = conn.SetLinger(0); err != nil {
+		return err
+	}
+	if err = conn.SetNoDelay(false); err != nil {
+		return err
+	}
+	if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
+		return err
+	}
+	if err = conn.SetKeepAlive(true); err != nil {
+		return err
+	}
+	return nil
+}

lib/osutil/ping.go

@@ -0,0 +1,39 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package osutil
+
+import (
+	"net"
+	"net/url"
+	"time"
+)
+
+// TCPPing returns the duration required to establish a TCP connection
+// to the given host. ICMP packets require root priviledges, hence why we use
+// tcp.
+func TCPPing(address string) (time.Duration, error) {
+	dialer := net.Dialer{
+		Deadline: time.Now().Add(time.Second),
+	}
+	start := time.Now()
+	conn, err := dialer.Dial("tcp", address)
+	if conn != nil {
+		conn.Close()
+	}
+	return time.Since(start), err
+}
+
+// GetLatencyForURL parses the given URL, tries opening a TCP connection to it
+// and returns the time it took to establish the connection.
+func GetLatencyForURL(addr string) (time.Duration, error) {
+	uri, err := url.Parse(addr)
+	if err != nil {
+		return 0, err
+	}
+
+	return TCPPing(uri.Host)
+}

lib/relay/debug.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package relay
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "relay") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

lib/relay/relay.go

@@ -0,0 +1,401 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package relay
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/url"
+	"sort"
+	"time"
+
+	"github.com/syncthing/relaysrv/client"
+	"github.com/syncthing/relaysrv/protocol"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/sync"
+
+	"github.com/thejerf/suture"
+)
+
+const (
+	eventBroadcasterCheckInterval = 10 * time.Second
+)
+
+type Svc struct {
+	*suture.Supervisor
+	cfg    *config.Wrapper
+	tlsCfg *tls.Config
+
+	tokens      map[string]suture.ServiceToken
+	clients     map[string]*client.ProtocolClient
+	mut         sync.RWMutex
+	invitations chan protocol.SessionInvitation
+	conns       chan *tls.Conn
+}
+
+func NewSvc(cfg *config.Wrapper, tlsCfg *tls.Config) *Svc {
+	conns := make(chan *tls.Conn)
+
+	svc := &Svc{
+		Supervisor: suture.New("Svc", suture.Spec{
+			Log: func(log string) {
+				if debug {
+					l.Debugln(log)
+				}
+			},
+			FailureBackoff:   5 * time.Minute,
+			FailureDecay:     float64((10 * time.Minute) / time.Second),
+			FailureThreshold: 5,
+		}),
+		cfg:    cfg,
+		tlsCfg: tlsCfg,
+
+		tokens:      make(map[string]suture.ServiceToken),
+		clients:     make(map[string]*client.ProtocolClient),
+		mut:         sync.NewRWMutex(),
+		invitations: make(chan protocol.SessionInvitation),
+		conns:       conns,
+	}
+
+	rcfg := cfg.Raw()
+	svc.CommitConfiguration(rcfg, rcfg)
+	cfg.Subscribe(svc)
+
+	receiver := &invitationReceiver{
+		tlsCfg:      tlsCfg,
+		conns:       conns,
+		invitations: svc.invitations,
+		stop:        make(chan struct{}),
+	}
+
+	eventBc := &eventBroadcaster{
+		svc: svc,
+	}
+
+	svc.Add(receiver)
+	svc.Add(eventBc)
+
+	return svc
+}
+
+func (s *Svc) VerifyConfiguration(from, to config.Configuration) error {
+	for _, addr := range to.Options.RelayServers {
+		_, err := url.Parse(addr)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
+	existing := make(map[string]*url.URL, len(to.Options.RelayServers))
+
+	for _, addr := range to.Options.RelayServers {
+		uri, err := url.Parse(addr)
+		if err != nil {
+			if debug {
+				l.Debugln("Failed to parse relay address", addr, err)
+			}
+			continue
+		}
+		existing[uri.String()] = uri
+	}
+
+	// Query dynamic addresses, and pick the closest relay from the ones they provide.
+	for key, uri := range existing {
+		if uri.Scheme != "dynamic+http" && uri.Scheme != "dynamic+https" {
+			continue
+		}
+		delete(existing, key)
+
+		// Trim off the `dynamic+` prefix
+		uri.Scheme = uri.Scheme[8:]
+
+		if debug {
+			l.Debugln("Looking up dynamic relays from", uri)
+		}
+
+		data, err := http.Get(uri.String())
+		if err != nil {
+			if debug {
+				l.Debugln("Failed to lookup dynamic relays", err)
+			}
+			continue
+		}
+
+		var ann dynamicAnnouncement
+		err = json.NewDecoder(data.Body).Decode(&ann)
+		data.Body.Close()
+		if err != nil {
+			if debug {
+				l.Debugln("Failed to lookup dynamic relays", err)
+			}
+			continue
+		}
+
+		var dynRelayAddrs []string
+		for _, relayAnn := range ann.Relays {
+			ruri, err := url.Parse(relayAnn.URL)
+			if err != nil {
+				if debug {
+					l.Debugln("Failed to parse dynamic relay address", relayAnn.URL, err)
+				}
+				continue
+			}
+			if debug {
+				l.Debugln("Found", ruri, "via", uri)
+			}
+			dynRelayAddrs = append(dynRelayAddrs, ruri.String())
+		}
+
+		if len(dynRelayAddrs) > 0 {
+			dynRelayAddrs = relayAddressesSortedByLatency(dynRelayAddrs)
+			closestRelay := dynRelayAddrs[0]
+			if debug {
+				l.Debugln("Picking", closestRelay, "as closest dynamic relay from", uri)
+			}
+			ruri, _ := url.Parse(closestRelay)
+			existing[closestRelay] = ruri
+		} else if debug {
+			l.Debugln("No dynamic relay found on", uri)
+		}
+	}
+
+	s.mut.Lock()
+
+	for key, uri := range existing {
+		_, ok := s.tokens[key]
+		if !ok {
+			if debug {
+				l.Debugln("Connecting to relay", uri)
+			}
+			c := client.NewProtocolClient(uri, s.tlsCfg.Certificates, s.invitations)
+			s.tokens[key] = s.Add(c)
+			s.clients[key] = c
+		}
+	}
+
+	for key, token := range s.tokens {
+		_, ok := existing[key]
+		if !ok {
+			err := s.Remove(token)
+			delete(s.tokens, key)
+			delete(s.clients, key)
+			if debug {
+				l.Debugln("Disconnecting from relay", key, err)
+			}
+		}
+	}
+
+	s.mut.Unlock()
+
+	return true
+}
+
+type Status struct {
+	URL     string
+	OK      bool
+	Latency int
+}
+
+// Relays return the list of relays that currently have an OK status.
+func (s *Svc) Relays() []string {
+	if s == nil {
+		// A nil client does not have a status, really. Yet we may be called
+		// this way, for raisins...
+		return nil
+	}
+
+	s.mut.RLock()
+	relays := make([]string, 0, len(s.clients))
+	for uri := range s.clients {
+		relays = append(relays, uri)
+	}
+	s.mut.RUnlock()
+
+	sort.Strings(relays)
+
+	return relays
+}
+
+// RelayStatus returns the latency and OK status for a given relay.
+func (s *Svc) RelayStatus(uri string) (time.Duration, bool) {
+	if s == nil {
+		// A nil client does not have a status, really. Yet we may be called
+		// this way, for raisins...
+		return time.Hour, false
+	}
+
+	s.mut.RLock()
+	client, ok := s.clients[uri]
+	s.mut.RUnlock()
+
+	if !ok || !client.StatusOK() {
+		return time.Hour, false
+	}
+
+	return client.Latency(), true
+}
+
+// Accept returns a new *tls.Conn. The connection is already handshaken.
+func (s *Svc) Accept() *tls.Conn {
+	return <-s.conns
+}
+
+type invitationReceiver struct {
+	invitations chan protocol.SessionInvitation
+	tlsCfg      *tls.Config
+	conns       chan<- *tls.Conn
+	stop        chan struct{}
+}
+
+func (r *invitationReceiver) Serve() {
+	for {
+		select {
+		case inv := <-r.invitations:
+			if debug {
+				l.Debugln("Received relay invitation", inv)
+			}
+			conn, err := client.JoinSession(inv)
+			if err != nil {
+				if debug {
+					l.Debugf("Failed to join relay session %s: %v", inv, err)
+				}
+				continue
+			}
+
+			err = osutil.SetTCPOptions(conn.(*net.TCPConn))
+			if err != nil {
+				l.Infoln(err)
+			}
+
+			var tc *tls.Conn
+
+			if inv.ServerSocket {
+				tc = tls.Server(conn, r.tlsCfg)
+			} else {
+				tc = tls.Client(conn, r.tlsCfg)
+			}
+			err = tc.Handshake()
+			if err != nil {
+				l.Infof("TLS handshake (BEP/relay %s): %v", inv, err)
+				tc.Close()
+				continue
+			}
+			r.conns <- tc
+
+		case <-r.stop:
+			return
+		}
+	}
+}
+
+func (r *invitationReceiver) Stop() {
+	close(r.stop)
+}
+
+// The eventBroadcaster sends a RelayStateChanged event when the relay status
+// changes. We need this somewhat ugly polling mechanism as there's currently
+// no way to get the event feed directly from the relay lib. This may be
+// somethign to revisit later, possibly.
+type eventBroadcaster struct {
+	svc  *Svc
+	stop chan struct{}
+}
+
+func (e *eventBroadcaster) Serve() {
+	timer := time.NewTicker(eventBroadcasterCheckInterval)
+	defer timer.Stop()
+
+	var prevOKRelays []string
+
+	for {
+		select {
+		case <-timer.C:
+			curOKRelays := e.svc.Relays()
+
+			changed := len(curOKRelays) != len(prevOKRelays)
+			if !changed {
+				for i := range curOKRelays {
+					if curOKRelays[i] != prevOKRelays[i] {
+						changed = true
+						break
+					}
+				}
+			}
+
+			if changed {
+				events.Default.Log(events.RelayStateChanged, map[string][]string{
+					"old": prevOKRelays,
+					"new": curOKRelays,
+				})
+			}
+
+			prevOKRelays = curOKRelays
+
+		case <-e.stop:
+			return
+		}
+	}
+}
+
+func (e *eventBroadcaster) Stop() {
+	close(e.stop)
+}
+
+// This is the announcement recieved from the relay server;
+// {"relays": [{"url": "relay://10.20.30.40:5060"}, ...]}
+type dynamicAnnouncement struct {
+	Relays []struct {
+		URL string
+	}
+}
+
+// relayAddressesSortedByLatency adds local latency to the relay, and sorts them
+// by sum latency, and returns the addresses.
+func relayAddressesSortedByLatency(input []string) []string {
+	relays := make(relayList, len(input))
+	for i, relay := range input {
+		if latency, err := osutil.GetLatencyForURL(relay); err == nil {
+			relays[i] = relayWithLatency{relay, int(latency / time.Millisecond)}
+		} else {
+			relays[i] = relayWithLatency{relay, int(time.Hour / time.Millisecond)}
+		}
+	}
+
+	sort.Sort(relays)
+
+	addresses := make([]string, len(relays))
+	for i, relay := range relays {
+		addresses[i] = relay.relay
+	}
+	return addresses
+}
+
+type relayWithLatency struct {
+	relay   string
+	latency int
+}
+
+type relayList []relayWithLatency
+
+func (l relayList) Len() int {
+	return len(l)
+}
+
+func (l relayList) Less(a, b int) bool {
+	return l[a].latency < l[b].latency
+}
+
+func (l relayList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}

lib/scanner/blockqueue.go

@@ -19,24 +19,27 @@ import (
 // workers are used in parallel. The outbox will become closed when the inbox
 // is closed and all items handled.
 
-func newParallelHasher(dir string, blockSize, workers int, outbox, inbox chan protocol.FileInfo) {
+func newParallelHasher(dir string, blockSize, workers int, outbox, inbox chan protocol.FileInfo, counter *int64, done chan struct{}) {
 	wg := sync.NewWaitGroup()
 	wg.Add(workers)
 
 	for i := 0; i < workers; i++ {
 		go func() {
-			hashFiles(dir, blockSize, outbox, inbox)
+			hashFiles(dir, blockSize, outbox, inbox, counter)
 			wg.Done()
 		}()
 	}
 
 	go func() {
 		wg.Wait()
+		if done != nil {
+			close(done)
+		}
 		close(outbox)
 	}()
 }
 
-func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
+func HashFile(path string, blockSize int, sizeHint int64, counter *int64) ([]protocol.BlockInfo, error) {
 	fd, err := os.Open(path)
 	if err != nil {
 		if debug {
@@ -44,27 +47,29 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
 		}
 		return []protocol.BlockInfo{}, err
 	}
-
-	fi, err := fd.Stat()
-	if err != nil {
-		fd.Close()
-		if debug {
-			l.Debugln("stat:", err)
-		}
-		return []protocol.BlockInfo{}, err
-	}
 	defer fd.Close()
-	return Blocks(fd, blockSize, fi.Size())
+
+	if sizeHint == 0 {
+		fi, err := fd.Stat()
+		if err != nil {
+			if debug {
+				l.Debugln("stat:", err)
+			}
+			return []protocol.BlockInfo{}, err
+		}
+		sizeHint = fi.Size()
+	}
+
+	return Blocks(fd, blockSize, sizeHint, counter)
 }
 
-func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
+func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo, counter *int64) {
 	for f := range inbox {
-		if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() {
-			outbox <- f
-			continue
+		if f.IsDirectory() || f.IsDeleted() {
+			panic("Bug. Asked to hash a directory or a deleted file.")
 		}
 
-		blocks, err := HashFile(filepath.Join(dir, f.Name), blockSize)
+		blocks, err := HashFile(filepath.Join(dir, f.Name), blockSize, f.CachedSize, counter)
 		if err != nil {
 			if debug {
 				l.Debugln("hash error:", f.Name, err)

lib/scanner/blocks.go

@@ -11,6 +11,7 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
+	"sync/atomic"
 
 	"github.com/syncthing/protocol"
 )
@@ -18,7 +19,7 @@ import (
 var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}
 
 // Blocks returns the blockwise hash of the reader.
-func Blocks(r io.Reader, blocksize int, sizehint int64) ([]protocol.BlockInfo, error) {
+func Blocks(r io.Reader, blocksize int, sizehint int64, counter *int64) ([]protocol.BlockInfo, error) {
 	var blocks []protocol.BlockInfo
 	if sizehint > 0 {
 		blocks = make([]protocol.BlockInfo, 0, int(sizehint/int64(blocksize)))
@@ -36,6 +37,10 @@ func Blocks(r io.Reader, blocksize int, sizehint int64) ([]protocol.BlockInfo, e
 			break
 		}
 
+		if counter != nil {
+			atomic.AddInt64(counter, int64(n))
+		}
+
 		b := protocol.BlockInfo{
 			Size:   int32(n),
 			Offset: offset,

lib/scanner/blocks_test.go

@@ -51,7 +51,7 @@ var blocksTestData = []struct {
 func TestBlocks(t *testing.T) {
 	for _, test := range blocksTestData {
 		buf := bytes.NewBuffer(test.data)
-		blocks, err := Blocks(buf, test.blocksize, 0)
+		blocks, err := Blocks(buf, test.blocksize, 0, nil)
 
 		if err != nil {
 			t.Fatal(err)
@@ -105,8 +105,8 @@ var diffTestData = []struct {
 
 func TestDiff(t *testing.T) {
 	for i, test := range diffTestData {
-		a, _ := Blocks(bytes.NewBufferString(test.a), test.s, 0)
-		b, _ := Blocks(bytes.NewBufferString(test.b), test.s, 0)
+		a, _ := Blocks(bytes.NewBufferString(test.a), test.s, 0, nil)
+		b, _ := Blocks(bytes.NewBufferString(test.b), test.s, 0, nil)
 		_, d := BlockDiff(a, b)
 		if len(d) != len(test.d) {
 			t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d))

lib/scanner/walk.go

@@ -12,12 +12,13 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"time"
 	"unicode/utf8"
 
 	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/ignore"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/symlinks"
 	"golang.org/x/text/unicode/norm"
@@ -39,6 +40,8 @@ func init() {
 }
 
 type Walker struct {
+	// Folder for which the walker has been created
+	Folder string
 	// Dir is the base directory for the walk
 	Dir string
 	// Limit walking to these paths within Dir, or no limit if Sub is empty
@@ -46,7 +49,7 @@ type Walker struct {
 	// BlockSize controls the size of the block used when hashing.
 	BlockSize int
 	// If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
-	Matcher *ignore.Matcher
+	Matcher IgnoreMatcher
 	// If TempNamer is not nil, it is used to ignore temporary files when walking.
 	TempNamer TempNamer
 	// Number of hours to keep temporary files for
@@ -66,6 +69,9 @@ type Walker struct {
 	Hashers int
 	// Our vector clock id
 	ShortID uint64
+	// Optional progress tick interval which defines how often FolderScanProgress
+	// events are emitted. Negative number means disabled.
+	ProgressTickIntervalS int
 }
 
 type TempNamer interface {
@@ -80,6 +86,11 @@ type CurrentFiler interface {
 	CurrentFile(name string) (protocol.FileInfo, bool)
 }
 
+type IgnoreMatcher interface {
+	// Match returns true if the file should be ignored.
+	Match(filename string) bool
+}
+
 // Walk returns the list of files found in the local folder by scanning the
 // file system. Files are blockwise hashed.
 func (w *Walker) Walk() (chan protocol.FileInfo, error) {
@@ -92,12 +103,13 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 		return nil, err
 	}
 
-	files := make(chan protocol.FileInfo)
-	hashedFiles := make(chan protocol.FileInfo)
-	newParallelHasher(w.Dir, w.BlockSize, w.Hashers, hashedFiles, files)
+	toHashChan := make(chan protocol.FileInfo)
+	finishedChan := make(chan protocol.FileInfo)
 
+	// A routine which walks the filesystem tree, and sends files which have
+	// been modified to the counter routine.
 	go func() {
-		hashFiles := w.walkAndHashFiles(files, hashedFiles)
+		hashFiles := w.walkAndHashFiles(toHashChan, finishedChan)
 		if len(w.Subs) == 0 {
 			filepath.Walk(w.Dir, hashFiles)
 		} else {
@@ -105,10 +117,77 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 				filepath.Walk(filepath.Join(w.Dir, sub), hashFiles)
 			}
 		}
-		close(files)
+		close(toHashChan)
 	}()
 
-	return hashedFiles, nil
+	// We're not required to emit scan progress events, just kick off hashers,
+	// and feed inputs directly from the walker.
+	if w.ProgressTickIntervalS < 0 {
+		newParallelHasher(w.Dir, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil)
+		return finishedChan, nil
+	}
+
+	// Defaults to every 2 seconds.
+	if w.ProgressTickIntervalS == 0 {
+		w.ProgressTickIntervalS = 2
+	}
+
+	ticker := time.NewTicker(time.Duration(w.ProgressTickIntervalS) * time.Second)
+
+	// We need to emit progress events, hence we create a routine which buffers
+	// the list of files to be hashed, counts the total number of
+	// bytes to hash, and once no more files need to be hashed (chan gets closed),
+	// start a routine which periodically emits FolderScanProgress events,
+	// until a stop signal is sent by the parallel hasher.
+	// Parallel hasher is stopped by this routine when we close the channel over
+	// which it receives the files we ask it to hash.
+	go func() {
+		var filesToHash []protocol.FileInfo
+		var total, progress int64
+		for file := range toHashChan {
+			filesToHash = append(filesToHash, file)
+			total += int64(file.CachedSize)
+		}
+
+		realToHashChan := make(chan protocol.FileInfo)
+		done := make(chan struct{})
+		newParallelHasher(w.Dir, w.BlockSize, w.Hashers, finishedChan, realToHashChan, &progress, done)
+
+		// A routine which actually emits the FolderScanProgress events
+		// every w.ProgressTicker ticks, until the hasher routines terminate.
+		go func() {
+			for {
+				select {
+				case <-done:
+					if debug {
+						l.Debugln("Walk progress done", w.Dir, w.Subs, w.BlockSize, w.Matcher)
+					}
+					ticker.Stop()
+					return
+				case <-ticker.C:
+					current := atomic.LoadInt64(&progress)
+					if debug {
+						l.Debugf("Walk %s %s current progress %d/%d (%d%%)", w.Dir, w.Subs, current, total, current*100/total)
+					}
+					events.Default.Log(events.FolderScanProgress, map[string]interface{}{
+						"folder":  w.Folder,
+						"current": current,
+						"total":   total,
+					})
+				}
+			}
+		}()
+
+		for _, file := range filesToHash {
+			if debug {
+				l.Debugln("real to hash:", file.Name)
+			}
+			realToHashChan <- file
+		}
+		close(realToHashChan)
+	}()
+
+	return finishedChan, nil
 }
 
 func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.WalkFunc {
@@ -161,7 +240,7 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 		}
 
 		if sn := filepath.Base(rn); sn == ".stignore" || sn == ".stfolder" ||
-			strings.HasPrefix(rn, ".stversions") || w.Matcher.Match(rn) {
+			strings.HasPrefix(rn, ".stversions") || (w.Matcher != nil && w.Matcher.Match(rn)) {
 			// An ignored file
 			if debug {
 				l.Debugln("ignored:", rn)
@@ -232,8 +311,7 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 			// checking that their existing blocks match with the blocks in
 			// the index.
 
-			target, flags, err := symlinks.Read(p)
-			flags = flags & protocol.SymlinkTypeMask
+			target, targetType, err := symlinks.Read(p)
 			if err != nil {
 				if debug {
 					l.Debugln("readlink error:", p, err)
@@ -241,7 +319,7 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 				return skip
 			}
 
-			blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0)
+			blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0, nil)
 			if err != nil {
 				if debug {
 					l.Debugln("hash link error:", p, err)
@@ -258,7 +336,7 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 				//  - the symlink type (file/dir) was the same
 				//  - the block list (i.e. hash of target) was the same
 				cf, ok = w.CurrentFiler.CurrentFile(rn)
-				if ok && !cf.IsDeleted() && cf.IsSymlink() && !cf.IsInvalid() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) {
+				if ok && !cf.IsDeleted() && cf.IsSymlink() && !cf.IsInvalid() && SymlinkTypeEqual(targetType, cf) && BlocksEqual(cf.Blocks, blocks) {
 					return skip
 				}
 			}
@@ -266,16 +344,16 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 			f := protocol.FileInfo{
 				Name:     rn,
 				Version:  cf.Version.Update(w.ShortID),
-				Flags:    protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666,
+				Flags:    uint32(protocol.FlagSymlink | protocol.FlagNoPermBits | 0666 | SymlinkFlags(targetType)),
 				Modified: 0,
 				Blocks:   blocks,
 			}
 
 			if debug {
-				l.Debugln("symlink to hash:", p, f)
+				l.Debugln("symlink changedb:", p, f)
 			}
 
-			fchan <- f
+			dchan <- f
 
 			return skip
 		}
@@ -349,10 +427,11 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 			}
 
 			f := protocol.FileInfo{
-				Name:     rn,
-				Version:  cf.Version.Update(w.ShortID),
-				Flags:    flags,
-				Modified: mtime.Unix(),
+				Name:       rn,
+				Version:    cf.Version.Update(w.ShortID),
+				Flags:      flags,
+				Modified:   mtime.Unix(),
+				CachedSize: info.Size(),
 			}
 			if debug {
 				l.Debugln("to hash:", p, f)
@@ -387,7 +466,7 @@ func PermsEqual(a, b uint32) bool {
 	}
 }
 
-func SymlinkTypeEqual(disk, index uint32) bool {
+func SymlinkTypeEqual(disk symlinks.TargetType, f protocol.FileInfo) bool {
 	// If the target is missing, Unix never knows what type of symlink it is
 	// and Windows always knows even if there is no target. Which means that
 	// without this special check a Unix node would be fighting with a Windows
@@ -396,8 +475,25 @@ func SymlinkTypeEqual(disk, index uint32) bool {
 	// know means you are on Unix, and on Unix you don't really care what the
 	// target type is. The moment you do know, and if something doesn't match,
 	// that will propagate through the cluster.
-	if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 {
+	switch disk {
+	case symlinks.TargetUnknown:
 		return true
+	case symlinks.TargetDirectory:
+		return f.IsDirectory() && f.Flags&protocol.FlagSymlinkMissingTarget == 0
+	case symlinks.TargetFile:
+		return !f.IsDirectory() && f.Flags&protocol.FlagSymlinkMissingTarget == 0
 	}
-	return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask
+	panic("unknown symlink TargetType")
+}
+
+func SymlinkFlags(t symlinks.TargetType) uint32 {
+	switch t {
+	case symlinks.TargetFile:
+		return 0
+	case symlinks.TargetDirectory:
+		return protocol.FlagDirectory
+	case symlinks.TargetUnknown:
+		return protocol.FlagSymlinkMissingTarget
+	}
+	panic("unknown symlink TargetType")
 }

lib/scanner/walk_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/symlinks"
 	"golang.org/x/text/unicode/norm"
 )
 
@@ -149,8 +150,9 @@ func TestVerify(t *testing.T) {
 	// data should be an even multiple of blocksize long
 	data := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut e")
 	buf := bytes.NewBuffer(data)
+	var progress int64
 
-	blocks, err := Blocks(buf, blocksize, 0)
+	blocks, err := Blocks(buf, blocksize, 0, &progress)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -158,6 +160,10 @@ func TestVerify(t *testing.T) {
 		t.Fatalf("Incorrect number of blocks %d != %d", len(blocks), exp)
 	}
 
+	if int64(len(data)) != progress {
+		t.Fatalf("Incorrect counter value %d  != %d", len(data), progress)
+	}
+
 	buf = bytes.NewBuffer(data)
 	err = Verify(buf, blocksize, blocks)
 	t.Log(err)
@@ -338,3 +344,31 @@ func (l testfileList) String() string {
 	b.WriteString("}")
 	return b.String()
 }
+
+func TestSymlinkTypeEqual(t *testing.T) {
+	testcases := []struct {
+		onDiskType   symlinks.TargetType
+		inIndexFlags uint32
+		equal        bool
+	}{
+		// File is only equal to file
+		{symlinks.TargetFile, 0, true},
+		{symlinks.TargetFile, protocol.FlagDirectory, false},
+		{symlinks.TargetFile, protocol.FlagSymlinkMissingTarget, false},
+		// Directory is only equal to directory
+		{symlinks.TargetDirectory, 0, false},
+		{symlinks.TargetDirectory, protocol.FlagDirectory, true},
+		{symlinks.TargetDirectory, protocol.FlagSymlinkMissingTarget, false},
+		// Unknown is equal to anything
+		{symlinks.TargetUnknown, 0, true},
+		{symlinks.TargetUnknown, protocol.FlagDirectory, true},
+		{symlinks.TargetUnknown, protocol.FlagSymlinkMissingTarget, true},
+	}
+
+	for _, tc := range testcases {
+		res := SymlinkTypeEqual(tc.onDiskType, protocol.FileInfo{Flags: tc.inIndexFlags})
+		if res != tc.equal {
+			t.Errorf("Incorrect result %v for %v, %v", res, tc.onDiskType, tc.inIndexFlags)
+		}
+	}
+}

lib/signature/signature.go

@@ -0,0 +1,202 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package signature provides simple methods to create and verify signatures
+// in PEM format.
+package signature
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/asn1"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+)
+
+// GenerateKeys returns a new key pair, with the private and public key
+// encoded in PEM format.
+func GenerateKeys() (privKey []byte, pubKey []byte, err error) {
+	// Generate a new key pair
+	key, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Marshal the private key
+	bs, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Encode it in PEM format
+	privKey = pem.EncodeToMemory(&pem.Block{
+		Type:  "EC PRIVATE KEY",
+		Bytes: bs,
+	})
+
+	// Marshal the public key
+	bs, err = x509.MarshalPKIXPublicKey(key.Public())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Encode it in PEM format
+	pubKey = pem.EncodeToMemory(&pem.Block{
+		Type:  "EC PUBLIC KEY",
+		Bytes: bs,
+	})
+
+	return
+}
+
+// Sign computes the hash of data and signs it with the private key, returning
+// a signature in PEM format.
+func Sign(privKeyPEM []byte, data io.Reader) ([]byte, error) {
+	// Parse the private key
+	key, err := loadPrivateKey(privKeyPEM)
+	if err != nil {
+		return nil, err
+	}
+
+	// Hash the reader data
+	hash, err := hashReader(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// Sign the hash
+	r, s, err := ecdsa.Sign(rand.Reader, key, hash)
+	if err != nil {
+		return nil, err
+	}
+
+	// Marshal the signature using ASN.1
+	sig, err := marshalSignature(r, s)
+	if err != nil {
+		return nil, err
+	}
+
+	// Encode it in a PEM block
+	bs := pem.EncodeToMemory(&pem.Block{
+		Type:  "SIGNATURE",
+		Bytes: sig,
+	})
+
+	return bs, nil
+}
+
+// Verify computes the hash of data and compares it to the signature using the
+// given public key. Returns nil if the signature is correct.
+func Verify(pubKeyPEM []byte, signature []byte, data io.Reader) error {
+	// Parse the public key
+	key, err := loadPublicKey(pubKeyPEM)
+	if err != nil {
+		return err
+	}
+
+	// Parse the signature
+	block, _ := pem.Decode(signature)
+	if block == nil || block.Bytes == nil {
+		return errors.New("unsupported signature format")
+	}
+
+	r, s, err := unmarshalSignature(block.Bytes)
+	if err != nil {
+		return err
+	}
+
+	// Compute the hash of the data
+	hash, err := hashReader(data)
+	if err != nil {
+		return err
+	}
+
+	// Verify the signature
+	if !ecdsa.Verify(key, hash, r, s) {
+		return errors.New("incorrect signature")
+	}
+
+	return nil
+}
+
+// hashReader returns the SHA256 hash of the reader
+func hashReader(r io.Reader) ([]byte, error) {
+	h := sha256.New()
+	if _, err := io.Copy(h, r); err != nil {
+		return nil, err
+	}
+	hash := []byte(fmt.Sprintf("%x", h.Sum(nil)))
+	return hash, nil
+}
+
+// loadPrivateKey returns the ECDSA private key structure for the given PEM
+// data.
+func loadPrivateKey(bs []byte) (*ecdsa.PrivateKey, error) {
+	block, _ := pem.Decode(bs)
+	return x509.ParseECPrivateKey(block.Bytes)
+}
+
+// loadPublicKey returns the ECDSA public key structure for the given PEM
+// data.
+func loadPublicKey(bs []byte) (*ecdsa.PublicKey, error) {
+	// Decode and parse the public key PEM block
+	block, _ := pem.Decode(bs)
+	if block == nil || block.Bytes == nil {
+		return nil, errors.New("unsupported public key format")
+	}
+	intf, err := x509.ParsePKIXPublicKey(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	// It should be an ECDSA public key
+	pk, ok := intf.(*ecdsa.PublicKey)
+	if !ok {
+		return nil, errors.New("unsupported public key format")
+	}
+
+	return pk, nil
+}
+
+// A wrapper around the signature integers so that we can marshal and
+// unmarshal them.
+type signature struct {
+	R, S *big.Int
+}
+
+// marhalSignature returns ASN.1 encoded bytes for the given integers,
+// suitable for PEM encoding.
+func marshalSignature(r, s *big.Int) ([]byte, error) {
+	sig := signature{
+		R: r,
+		S: s,
+	}
+
+	bs, err := asn1.Marshal(sig)
+	if err != nil {
+		return nil, err
+	}
+
+	return bs, nil
+}
+
+// unmarshalSignature returns the R and S integers from the given ASN.1
+// encoded signature.
+func unmarshalSignature(sig []byte) (r *big.Int, s *big.Int, err error) {
+	var ts signature
+	_, err = asn1.Unmarshal(sig, &ts)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return ts.R, ts.S, nil
+}

lib/signature/signature_test.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package signature_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/signature"
+)
+
+var (
+	// A private key for signing
+	privKey = []byte(`-----BEGIN EC PRIVATE KEY-----
+MIHbAgEBBEFGXB1IgefFF6kSyE17xIAU7fDIn07sPnGf1kLOCVrEZyUbnAmNFk8u
+lUt/knnvo+Gw1i9ucFjmtYtzDevrhSlG5aAHBgUrgQQAI6GBiQOBhgAEASlcbcgJ
+4PN+TSnAYiMlA0I/PRtFrDCgrt27K7hR+U7Afjc4KqW+QYwoRLvxueNh7gUK+zc0
+Aqrk3z+O1epiQTq8ACikHUXsx/bSzEFlPdMygUAAj3hChlgCL6/vOocuRUbtAqc6
+Zr0L9px+J4L0K+uqhyhKya7y6QLJrYPovFq3A7AK
+-----END EC PRIVATE KEY-----`)
+
+	// The matching public key
+	pubKey = []byte(`-----BEGIN EC PUBLIC KEY-----
+MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBKVxtyAng835NKcBiIyUDQj89G0Ws
+MKCu3bsruFH5TsB+Nzgqpb5BjChEu/G542HuBQr7NzQCquTfP47V6mJBOrwAKKQd
+RezH9tLMQWU90zKBQACPeEKGWAIvr+86hy5FRu0CpzpmvQv2nH4ngvQr66qHKErJ
+rvLpAsmtg+i8WrcDsAo=
+-----END EC PUBLIC KEY-----`)
+
+	// A signature of "this is a string to sign" created with the private key
+	// above
+	exampleSig = []byte(`-----BEGIN SIGNATURE-----
+MIGGAkFdHjdarlFOrtcnCqcb0BX7Mjjq/Sbgp4mopCxBwXmfamtCeRGhZJ5MikyD
+VXScaJ2Dq2Ov7L4/gTcYj9fZwcrWgQJBc7+tcw5fpO0/y8DNq0t3g9bqt2MkmoNm
+eSAM8Fze4usVXHEi+QeMuYM2IKeVPyAR3iyl5gflVul9NRXS3OPAH3A=
+-----END SIGNATURE-----`)
+)
+
+func TestGenerateKeys(t *testing.T) {
+	priv, pub, err := signature.GenerateKeys()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Contains(priv, []byte("PRIVATE KEY")) {
+		t.Fatal("should be a private key")
+	}
+	if !bytes.Contains(pub, []byte("PUBLIC KEY")) {
+		t.Fatal("should be a private key")
+	}
+}
+
+func TestSign(t *testing.T) {
+	data := bytes.NewReader([]byte("this is a string to sign"))
+
+	s, err := signature.Sign(privKey, data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Contains(s, []byte("SIGNATURE")) {
+		t.Error("should be a signature")
+	}
+}
+
+func TestVerify(t *testing.T) {
+	data := bytes.NewReader([]byte("this is a string to sign"))
+	err := signature.Verify(pubKey, exampleSig, data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	data = bytes.NewReader([]byte("thus is a string to sign"))
+	err = signature.Verify(pubKey, exampleSig, data)
+	if err == nil {
+		t.Fatal("signature should not match")
+	}
+}

lib/stats/device.go

@@ -9,7 +9,6 @@ package stats
 import (
 	"time"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -20,11 +19,11 @@ type DeviceStatistics struct {
 
 type DeviceStatisticsReference struct {
 	ns     *db.NamespacedKV
-	device protocol.DeviceID
+	device string
 }
 
-func NewDeviceStatisticsReference(ldb *leveldb.DB, device protocol.DeviceID) *DeviceStatisticsReference {
-	prefix := string(db.KeyTypeDeviceStatistic) + device.String()
+func NewDeviceStatisticsReference(ldb *leveldb.DB, device string) *DeviceStatisticsReference {
+	prefix := string(db.KeyTypeDeviceStatistic) + device
 	return &DeviceStatisticsReference{
 		ns:     db.NewNamespacedKV(ldb, prefix),
 		device: device,

lib/stats/folder.go

@@ -9,8 +9,6 @@ package stats
 import (
 	"time"
 
-	"github.com/syncthing/protocol"
-
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -55,13 +53,13 @@ func (s *FolderStatisticsReference) GetLastFile() LastFile {
 	}
 }
 
-func (s *FolderStatisticsReference) ReceivedFile(file protocol.FileInfo) {
+func (s *FolderStatisticsReference) ReceivedFile(file string, deleted bool) {
 	if debug {
 		l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder, file)
 	}
 	s.ns.PutTime("lastFileAt", time.Now())
-	s.ns.PutString("lastFileName", file.Name)
-	s.ns.PutBool("lastFileDeleted", file.IsDeleted())
+	s.ns.PutString("lastFileName", file)
+	s.ns.PutBool("lastFileDeleted", deleted)
 }
 
 func (s *FolderStatisticsReference) GetStatistics() FolderStatistics {

lib/symlinks/symlink_unix.go

@@ -11,7 +11,6 @@ package symlinks
 import (
 	"os"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/osutil"
 )
 
@@ -19,23 +18,24 @@ var (
 	Supported = true
 )
 
-func Read(path string) (string, uint32, error) {
-	var mode uint32
-	stat, err := os.Stat(path)
-	if err != nil {
-		mode = protocol.FlagSymlinkMissingTarget
-	} else if stat.IsDir() {
-		mode = protocol.FlagDirectory
+func Read(path string) (string, TargetType, error) {
+	tt := TargetUnknown
+	if stat, err := os.Stat(path); err == nil {
+		if stat.IsDir() {
+			tt = TargetDirectory
+		} else {
+			tt = TargetFile
+		}
 	}
-	path, err = os.Readlink(path)
+	path, err := os.Readlink(path)
 
-	return osutil.NormalizedFilename(path), mode, err
+	return osutil.NormalizedFilename(path), tt, err
 }
 
-func Create(source, target string, flags uint32) error {
+func Create(source, target string, tt TargetType) error {
 	return os.Symlink(osutil.NativeFilename(target), source)
 }
 
-func ChangeType(path string, flags uint32) error {
+func ChangeType(path string, tt TargetType) error {
 	return nil
 }

lib/symlinks/symlink_windows.go

@@ -101,14 +101,14 @@ func (r *reparseData) SubstituteName() string {
 	return string(utf16.Decode(r.buffer[offset : offset+length]))
 }
 
-func Read(path string) (string, uint32, error) {
+func Read(path string) (string, TargetType, error) {
 	ptr, err := syscall.UTF16PtrFromString(path)
 	if err != nil {
-		return "", protocol.FlagSymlinkMissingTarget, err
+		return "", TargetUnknown, err
 	}
 	handle, err := syscall.CreateFile(ptr, 0, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS|Win32FileFlagOpenReparsePoint, 0)
 	if err != nil || handle == syscall.InvalidHandle {
-		return "", protocol.FlagSymlinkMissingTarget, err
+		return "", TargetUnknown, err
 	}
 	defer syscall.Close(handle)
 	var ret uint16
@@ -116,21 +116,22 @@ func Read(path string) (string, uint32, error) {
 
 	r1, _, err := syscall.Syscall9(procDeviceIoControl.Addr(), 8, uintptr(handle), Win32FsctlGetReparsePoint, 0, 0, uintptr(unsafe.Pointer(&data)), unsafe.Sizeof(data), uintptr(unsafe.Pointer(&ret)), 0, 0)
 	if r1 == 0 {
-		return "", protocol.FlagSymlinkMissingTarget, err
+		return "", TargetUnknown, err
 	}
 
-	var flags uint32
-	attr, err := syscall.GetFileAttributes(ptr)
-	if err != nil {
-		flags = protocol.FlagSymlinkMissingTarget
-	} else if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
-		flags = protocol.FlagDirectory
+	tt := TargetUnknown
+	if attr, err := syscall.GetFileAttributes(ptr); err == nil {
+		if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
+			tt = TargetDirectory
+		} else {
+			tt = TargetFile
+		}
 	}
 
-	return osutil.NormalizedFilename(data.PrintName()), flags, nil
+	return osutil.NormalizedFilename(data.PrintName()), tt, nil
 }
 
-func Create(source, target string, flags uint32) error {
+func Create(source, target string, tt TargetType) error {
 	srcp, err := syscall.UTF16PtrFromString(source)
 	if err != nil {
 		return err
@@ -146,7 +147,7 @@ func Create(source, target string, flags uint32) error {
 	// If the flags doesn't reveal the target type, try to evaluate it
 	// ourselves, and worst case default to the symlink pointing to a file.
 	mode := 0
-	if flags&protocol.FlagSymlinkMissingTarget != 0 {
+	if tt == TargetUnknown {
 		path := target
 		if !filepath.IsAbs(target) {
 			path = filepath.Join(filepath.Dir(source), target)
@@ -156,7 +157,7 @@ func Create(source, target string, flags uint32) error {
 		if err == nil && stat.IsDir() {
 			mode = Win32SymbolicLinkFlagDirectory
 		}
-	} else if flags&protocol.FlagDirectory != 0 {
+	} else if tt == TargetDirectory {
 		mode = Win32SymbolicLinkFlagDirectory
 	}
 
@@ -167,24 +168,24 @@ func Create(source, target string, flags uint32) error {
 	return err
 }
 
-func ChangeType(path string, flags uint32) error {
-	target, cflags, err := Read(path)
+func ChangeType(path string, tt TargetType) error {
+	target, exTt, err := Read(path)
 	if err != nil {
 		return err
 	}
 	// If it's the same type, nothing to do.
-	if cflags&protocol.SymlinkTypeMask == flags&protocol.SymlinkTypeMask {
+	if tt == exTt {
 		return nil
 	}
 
 	// If the actual type is unknown, but the new type is file, nothing to do
-	if cflags&protocol.FlagSymlinkMissingTarget != 0 && flags&protocol.FlagDirectory == 0 {
+	if exTt == TargetUnknown && tt != TargetDirectory {
 		return nil
 	}
 	return osutil.InWritableDir(func(path string) error {
 		// It should be a symlink as well hence no need to change permissions on
 		// the file.
 		os.Remove(path)
-		return Create(path, target, flags)
+		return Create(path, target, tt)
 	}, path)
 }

lib/symlinks/targets.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package symlinks
+
+type TargetType int
+
+const (
+	TargetFile TargetType = iota
+	TargetDirectory
+	TargetUnknown
+)

lib/tlsutil/tlsutil.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-package main
+package tlsutil
 
 import (
 	"bufio"
@@ -14,6 +14,7 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"fmt"
 	"io"
 	"math/big"
 	mr "math/rand"
@@ -22,17 +23,14 @@ import (
 	"time"
 )
 
-const (
-	tlsRSABits           = 3072
-	tlsDefaultCommonName = "syncthing"
+var (
+	ErrIdentificationFailed = fmt.Errorf("failed to identify socket type")
 )
 
-func newCertificate(certFile, keyFile, name string) (tls.Certificate, error) {
-	l.Infof("Generating RSA key and certificate for %s...", name)
-
+func NewCertificate(certFile, keyFile, tlsDefaultCommonName string, tlsRSABits int) (tls.Certificate, error) {
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
 	if err != nil {
-		l.Fatalln("generate key:", err)
+		return tls.Certificate{}, fmt.Errorf("generate key: %s", err)
 	}
 
 	notBefore := time.Now()
@@ -41,7 +39,7 @@ func newCertificate(certFile, keyFile, name string) (tls.Certificate, error) {
 	template := x509.Certificate{
 		SerialNumber: new(big.Int).SetInt64(mr.Int63()),
 		Subject: pkix.Name{
-			CommonName: name,
+			CommonName: tlsDefaultCommonName,
 		},
 		NotBefore: notBefore,
 		NotAfter:  notAfter,
@@ -49,37 +47,38 @@ func newCertificate(certFile, keyFile, name string) (tls.Certificate, error) {
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
 		BasicConstraintsValid: true,
+		SignatureAlgorithm:    x509.SHA256WithRSA,
 	}
 
 	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
 	if err != nil {
-		l.Fatalln("create cert:", err)
+		return tls.Certificate{}, fmt.Errorf("create cert: %s", err)
 	}
 
 	certOut, err := os.Create(certFile)
 	if err != nil {
-		l.Fatalln("save cert:", err)
+		return tls.Certificate{}, fmt.Errorf("save cert: %s", err)
 	}
 	err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
 	if err != nil {
-		l.Fatalln("save cert:", err)
+		return tls.Certificate{}, fmt.Errorf("save cert: %s", err)
 	}
 	err = certOut.Close()
 	if err != nil {
-		l.Fatalln("save cert:", err)
+		return tls.Certificate{}, fmt.Errorf("save cert: %s", err)
 	}
 
 	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		l.Fatalln("save key:", err)
+		return tls.Certificate{}, fmt.Errorf("save key: %s", err)
 	}
 	err = pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
 	if err != nil {
-		l.Fatalln("save key:", err)
+		return tls.Certificate{}, fmt.Errorf("save key: %s", err)
 	}
 	err = keyOut.Close()
 	if err != nil {
-		l.Fatalln("save key:", err)
+		return tls.Certificate{}, fmt.Errorf("save key: %s", err)
 	}
 
 	return tls.LoadX509KeyPair(certFile, keyFile)
@@ -90,15 +89,29 @@ type DowngradingListener struct {
 	TLSConfig *tls.Config
 }
 
-type WrappedConnection struct {
-	io.Reader
-	net.Conn
+func (l *DowngradingListener) Accept() (net.Conn, error) {
+	conn, isTLS, err := l.AcceptNoWrapTLS()
+
+	// We failed to identify the socket type, pretend that everything is fine,
+	// and pass it to the underlying handler, and let them deal with it.
+	if err == ErrIdentificationFailed {
+		return conn, nil
+	}
+
+	if err != nil {
+		return conn, err
+	}
+
+	if isTLS {
+		return tls.Server(conn, l.TLSConfig), nil
+	}
+	return conn, nil
 }
 
-func (l *DowngradingListener) Accept() (net.Conn, error) {
+func (l *DowngradingListener) AcceptNoWrapTLS() (net.Conn, bool, error) {
 	conn, err := l.Listener.Accept()
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	br := bufio.NewReader(conn)
@@ -107,20 +120,19 @@ func (l *DowngradingListener) Accept() (net.Conn, error) {
 	conn.SetReadDeadline(time.Time{})
 	if err != nil {
 		// We hit a read error here, but the Accept() call succeeded so we must not return an error.
-		// We return the connection as is and let whoever tries to use it deal with the error.
-		return conn, nil
+		// We return the connection as is with a special error which handles this
+		// special case in Accept().
+		return conn, false, ErrIdentificationFailed
 	}
 
-	wrapper := &WrappedConnection{br, conn}
-
-	// 0x16 is the first byte of a TLS handshake
-	if bs[0] == 0x16 {
-		return tls.Server(wrapper, l.TLSConfig), nil
-	}
-
-	return wrapper, nil
+	return &UnionedConnection{br, conn}, bs[0] == 0x16, nil
 }
 
-func (c *WrappedConnection) Read(b []byte) (n int, err error) {
+type UnionedConnection struct {
+	io.Reader
+	net.Conn
+}
+
+func (c *UnionedConnection) Read(b []byte) (n int, err error) {
 	return c.Reader.Read(b)
 }

lib/upgrade/signingkey.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package upgrade
+
+// This is the public key used to verify signed upgrades. It must match the
+// private key used to sign binaries for the built in upgrade mechanism to
+// accept an upgrade. Keys and signatures can be created and verified with the
+// stsigtool utility. The build script creates signed binaries when given the
+// -sign option.
+var SigningKey = []byte(`-----BEGIN EC PUBLIC KEY-----
+MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQA1iRk+p+DsmolixxVKcpEVlMDPOeQ
+1dWthURMqsjxoJuDAe5I98P/A0kXSdBI7avm5hXhX2opJ5TAyBZLHPpDTRoBg4WN
+7jUpeAjtPoVVxvOh37qDeDVcjCgJbbDTPKbjxq/Ae3SHlQMRcoes7lVY1+YJ8dPk
+2oPfjA6jtmo9aVbf/uo=
+-----END EC PUBLIC KEY-----`)

lib/upgrade/upgrade_supported.go

@@ -13,7 +13,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
-	"crypto/md5"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -25,12 +25,29 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+
+	"github.com/syncthing/syncthing/lib/signature"
 )
 
+const DisabledByCompilation = false
+
+// This is an HTTP/HTTPS client that does *not* perform certificate
+// validation. We do this because some systems where Syncthing runs have
+// issues with old or missing CA roots. It doesn't actually matter that we
+// load the upgrade insecurely as we verify an ECDSA signature of the actual
+// binary contents before accepting the upgrade.
+var insecureHTTP = &http.Client{
+	Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+		},
+	},
+}
+
 // LatestGithubReleases returns the latest releases, including prereleases or
 // not depending on the argument
-func LatestGithubReleases(version string) ([]Release, error) {
-	resp, err := http.Get("https://api.github.com/repos/syncthing/syncthing/releases?per_page=30")
+func LatestGithubReleases(releasesURL, version string) ([]Release, error) {
+	resp, err := insecureHTTP.Get(releasesURL)
 	if err != nil {
 		return nil, err
 	}
@@ -57,8 +74,8 @@ func (s SortByRelease) Less(i, j int) bool {
 	return CompareVersions(s[i].Tag, s[j].Tag) > 0
 }
 
-func LatestRelease(version string) (Release, error) {
-	rels, _ := LatestGithubReleases(version)
+func LatestRelease(releasesURL, version string) (Release, error) {
+	rels, _ := LatestGithubReleases(releasesURL, version)
 	return SelectLatestRelease(version, rels)
 }
 
@@ -144,7 +161,7 @@ func readRelease(dir, url string) (string, error) {
 	}
 
 	req.Header.Add("Accept", "application/octet-stream")
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := insecureHTTP.Do(req)
 	if err != nil {
 		return "", err
 	}
@@ -166,10 +183,10 @@ func readTarGz(dir string, r io.Reader) (string, error) {
 
 	tr := tar.NewReader(gr)
 
-	var tempName, actualMD5, expectedMD5 string
+	var tempName string
+	var sig []byte
 
 	// Iterate through the files in the archive.
-fileLoop:
 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {
@@ -186,50 +203,21 @@ fileLoop:
 			l.Debugf("considering file %q", shortName)
 		}
 
-		switch shortName {
-		case "syncthing":
-			if debug {
-				l.Debugln("writing and hashing binary")
-			}
-			tempName, actualMD5, err = writeBinary(dir, tr)
-			if err != nil {
-				return "", err
-			}
+		err = archiveFileVisitor(dir, &tempName, &sig, shortName, tr)
+		if err != nil {
+			return "", err
+		}
 
-			if expectedMD5 != "" {
-				// We're done
-				break fileLoop
-			}
-
-		case "syncthing.md5":
-			bs, err := ioutil.ReadAll(tr)
-			if err != nil {
-				return "", err
-			}
-
-			expectedMD5 = strings.TrimSpace(string(bs))
-			if debug {
-				l.Debugln("expected md5 is", actualMD5)
-			}
-
-			if actualMD5 != "" {
-				// We're done
-				break fileLoop
-			}
+		if tempName != "" && sig != nil {
+			break
 		}
 	}
 
-	if tempName != "" {
-		// We found and saved something to disk.
-		if expectedMD5 == "" || actualMD5 == expectedMD5 {
-			return tempName, nil
-		}
-		os.Remove(tempName)
-		// There was an md5 file included in the archive, and it doesn't
-		// match what we just wrote to disk.
-		return "", fmt.Errorf("incorrect MD5 checksum")
+	if err := verifyUpgrade(tempName, sig); err != nil {
+		return "", err
 	}
-	return "", fmt.Errorf("no upgrade found")
+
+	return tempName, nil
 }
 
 func readZip(dir string, r io.Reader) (string, error) {
@@ -243,10 +231,10 @@ func readZip(dir string, r io.Reader) (string, error) {
 		return "", err
 	}
 
-	var tempName, actualMD5, expectedMD5 string
+	var tempName string
+	var sig []byte
 
 	// Iterate through the files in the archive.
-fileLoop:
 	for _, file := range archive.File {
 		shortName := path.Base(file.Name)
 
@@ -254,94 +242,108 @@ fileLoop:
 			l.Debugf("considering file %q", shortName)
 		}
 
-		switch shortName {
-		case "syncthing.exe":
-			if debug {
-				l.Debugln("writing and hashing binary")
-			}
+		inFile, err := file.Open()
+		if err != nil {
+			return "", err
+		}
 
-			inFile, err := file.Open()
-			if err != nil {
-				return "", err
-			}
-			tempName, actualMD5, err = writeBinary(dir, inFile)
-			if err != nil {
-				return "", err
-			}
+		err = archiveFileVisitor(dir, &tempName, &sig, shortName, inFile)
+		inFile.Close()
+		if err != nil {
+			return "", err
+		}
 
-			if expectedMD5 != "" {
-				// We're done
-				break fileLoop
-			}
-
-		case "syncthing.exe.md5":
-			inFile, err := file.Open()
-			if err != nil {
-				return "", err
-			}
-			bs, err := ioutil.ReadAll(inFile)
-			if err != nil {
-				return "", err
-			}
-
-			expectedMD5 = strings.TrimSpace(string(bs))
-			if debug {
-				l.Debugln("expected md5 is", actualMD5)
-			}
-
-			if actualMD5 != "" {
-				// We're done
-				break fileLoop
-			}
+		if tempName != "" && sig != nil {
+			break
 		}
 	}
 
-	if tempName != "" {
-		// We found and saved something to disk.
-		if expectedMD5 == "" || actualMD5 == expectedMD5 {
-			return tempName, nil
-		}
-		os.Remove(tempName)
-		// There was an md5 file included in the archive, and it doesn't
-		// match what we just wrote to disk.
-		return "", fmt.Errorf("incorrect MD5 checksum")
+	if err := verifyUpgrade(tempName, sig); err != nil {
+		return "", err
 	}
-	return "", fmt.Errorf("No upgrade found")
+
+	return tempName, nil
 }
 
-func writeBinary(dir string, inFile io.Reader) (filename, md5sum string, err error) {
-	outFile, err := ioutil.TempFile(dir, "syncthing")
-	if err != nil {
-		return "", "", err
+// archiveFileVisitor is called for each file in an archive. It may set
+// tempFile and signature.
+func archiveFileVisitor(dir string, tempFile *string, signature *[]byte, filename string, filedata io.Reader) error {
+	var err error
+	switch filename {
+	case "syncthing", "syncthing.exe":
+		if debug {
+			l.Debugln("reading binary")
+		}
+		*tempFile, err = writeBinary(dir, filedata)
+		if err != nil {
+			return err
+		}
+
+	case "syncthing.sig", "syncthing.exe.sig":
+		if debug {
+			l.Debugln("reading signature")
+		}
+		*signature, err = ioutil.ReadAll(filedata)
+		if err != nil {
+			return err
+		}
 	}
 
-	// Write the binary both a temporary file and to the MD5 hasher.
+	return nil
+}
 
-	h := md5.New()
-	mw := io.MultiWriter(h, outFile)
+func verifyUpgrade(tempName string, sig []byte) error {
+	if tempName == "" {
+		return fmt.Errorf("no upgrade found")
+	}
+	if sig == nil {
+		return fmt.Errorf("no signature found")
+	}
 
-	_, err = io.Copy(mw, inFile)
+	if debug {
+		l.Debugf("checking signature\n%s", sig)
+	}
+
+	fd, err := os.Open(tempName)
+	if err != nil {
+		return err
+	}
+	err = signature.Verify(SigningKey, sig, fd)
+	fd.Close()
+
+	if err != nil {
+		os.Remove(tempName)
+		return err
+	}
+
+	return nil
+}
+
+func writeBinary(dir string, inFile io.Reader) (filename string, err error) {
+	// Write the binary to a temporary file.
+
+	outFile, err := ioutil.TempFile(dir, "syncthing")
+	if err != nil {
+		return "", err
+	}
+
+	_, err = io.Copy(outFile, inFile)
 	if err != nil {
 		os.Remove(outFile.Name())
-		return "", "", err
+		return "", err
 	}
 
 	err = outFile.Close()
 	if err != nil {
 		os.Remove(outFile.Name())
-		return "", "", err
+		return "", err
 	}
 
 	err = os.Chmod(outFile.Name(), os.FileMode(0755))
 	if err != nil {
 		os.Remove(outFile.Name())
-		return "", "", err
+		return "", err
 	}
 
-	actualMD5 := fmt.Sprintf("%x", h.Sum(nil))
-	if debug {
-		l.Debugln("actual md5 is", actualMD5)
-	}
-
-	return outFile.Name(), actualMD5, nil
+	return outFile.Name(), nil
 }

lib/upgrade/upgrade_unsupp.go

@@ -8,6 +8,8 @@
 
 package upgrade
 
+const DisabledByCompilation = true
+
 func upgradeTo(binary string, rel Release) error {
 	return ErrUpgradeUnsupported
 }
@@ -16,6 +18,6 @@ func upgradeToURL(binary, url string) error {
 	return ErrUpgradeUnsupported
 }
 
-func LatestRelease(version string) (Release, error) {
+func LatestRelease(releasesURL, version string) (Release, error) {
 	return Release{}, ErrUpgradeUnsupported
 }

script/circle-installgo.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -euo pipefail
+
+[ -d ~/go1.5 ] && exit
+
+# Install the version of Go that we want
+
+curl -s https://storage.googleapis.com/golang/go1.5.linux-amd64.tar.gz \
+	| tar -C ~ --transform s/go/go1.5/ -zx
+
+# Build the standard library for all our cross compilation targets. We do that
+# here so that it gets cached and we don't need to repeat it for every build.
+
+for GOOS in darwin dragonfly solaris; do
+	export GOOS
+	export GOARCH=amd64
+	echo $GOOS $GOARCH
+	go install std
+done
+
+for GOOS in freebsd linux netbsd openbsd windows; do
+	for GOARCH in amd64 386; do
+		export GOOS
+		export GOARCH
+		echo $GOOS $GOARCH
+		go install std
+	done
+done
+
+export GOOS=linux
+export GOARCH=arm
+echo $GOOS $GOARCH
+go install std

test/h1/config.xml

@@ -1,4 +1,4 @@
-<configuration version="11">
+<configuration version="12">
     <folder id="default" path="s1" ro="false" rescanIntervalS="10" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
@@ -24,19 +24,19 @@
         <ignoreDelete>false</ignoreDelete>
     </folder>
     <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4" compression="metadata" introducer="false">
-        <address>127.0.0.1:22004</address>
+        <address>tcp://127.0.0.1:22004</address>
     </device>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
-        <address>127.0.0.1:22001</address>
+        <address>tcp://127.0.0.1:22001</address>
     </device>
     <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="metadata" introducer="false">
-        <address>127.0.0.1:22002</address>
+        <address>tcp://127.0.0.1:22002</address>
     </device>
     <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="metadata" introducer="false">
-        <address>127.0.0.1:22003</address>
+        <address>tcp://127.0.0.1:22003</address>
     </device>
     <device id="7PBCTLL-JJRYBSA-MOWZRKL-MSDMN4N-4US4OMX-SYEXUS4-HSBGNRY-CZXRXAT" name="s4" compression="metadata" introducer="false">
-        <address>127.0.0.1:22004</address>
+        <address>tcp://127.0.0.1:22004</address>
     </device>
     <gui enabled="true" tls="false">
         <address>127.0.0.1:8081</address>
@@ -45,16 +45,19 @@
         <apikey>abc123</apikey>
     </gui>
     <options>
-        <listenAddress>127.0.0.1:22001</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22026</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22026</globalAnnounceServer>
+        <listenAddress>tcp://127.0.0.1:22001</listenAddress>
+        <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
-        <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
+        <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
+        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
+        <relaysEnabled>true</relaysEnabled>
+        <relayReconnectIntervalM>10</relayReconnectIntervalM>
+        <relayWithoutGlobalAnn>false</relayWithoutGlobalAnn>
         <startBrowser>false</startBrowser>
         <upnpEnabled>true</upnpEnabled>
         <upnpLeaseMinutes>0</upnpLeaseMinutes>

test/h2/config.xml

@@ -1,4 +1,4 @@
-<configuration version="11">
+<configuration version="12">
     <folder id="default" path="s2" ro="false" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
@@ -12,6 +12,7 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressInterval>0</scanProgressInterval>
     </folder>
     <folder id="s12" path="s12-2" ro="false" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
@@ -23,6 +24,7 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressInterval>0</scanProgressInterval>
     </folder>
     <folder id="s23" path="s23-2" ro="false" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
@@ -34,38 +36,45 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressInterval>0</scanProgressInterval>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
-        <address>127.0.0.1:22001</address>
+        <address>tcp://127.0.0.1:22001</address>
     </device>
     <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="metadata" introducer="false">
-        <address>127.0.0.1:22002</address>
+        <address>tcp://127.0.0.1:22002</address>
     </device>
     <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="metadata" introducer="false">
-        <address>127.0.0.1:22003</address>
+        <address>tcp://127.0.0.1:22003</address>
     </device>
     <gui enabled="true" tls="false">
         <address>127.0.0.1:8082</address>
         <apikey>abc123</apikey>
     </gui>
     <options>
-        <listenAddress>127.0.0.1:22002</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22026</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22026</globalAnnounceServer>
-        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <listenAddress>tcp://127.0.0.1:22002</listenAddress>
+        <globalAnnounceServer>default</globalAnnounceServer>
+        <globalAnnounceEnabled>true</globalAnnounceEnabled>
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
-        <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
+        <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
+        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
+        <relaysEnabled>true</relaysEnabled>
+        <relayReconnectIntervalM>10</relayReconnectIntervalM>
+        <relayWithoutGlobalAnn>false</relayWithoutGlobalAnn>
         <startBrowser>false</startBrowser>
         <upnpEnabled>true</upnpEnabled>
         <upnpLeaseMinutes>0</upnpLeaseMinutes>
         <upnpRenewalMinutes>1</upnpRenewalMinutes>
         <upnpTimeoutSeconds>10</upnpTimeoutSeconds>
-        <urAccepted>-1</urAccepted>
-        <urUniqueID></urUniqueID>
+        <urAccepted>2</urAccepted>
+        <urUniqueID>621mlbJP</urUniqueID>
+        <urURL>https://localhost:8443/newdata</urURL>
+        <urPostInsecurely>true</urPostInsecurely>
+        <urInitialDelayS>10</urInitialDelayS>
         <restartOnWakeup>true</restartOnWakeup>
         <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
         <keepTemporariesH>24</keepTemporariesH>

test/h3/config.xml

@@ -1,4 +1,4 @@
-<configuration version="11">
+<configuration version="12">
     <folder id="s23" path="s23-3" ro="false" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
@@ -25,29 +25,32 @@
         <ignoreDelete>false</ignoreDelete>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
-        <address>127.0.0.1:22001</address>
+        <address>tcp://127.0.0.1:22001</address>
     </device>
     <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="metadata" introducer="false">
-        <address>127.0.0.1:22002</address>
+        <address>tcp://127.0.0.1:22002</address>
     </device>
     <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="metadata" introducer="false">
-        <address>127.0.0.1:22003</address>
+        <address>tcp://127.0.0.1:22003</address>
     </device>
     <gui enabled="true" tls="false">
         <address>127.0.0.1:8083</address>
         <apikey>abc123</apikey>
     </gui>
     <options>
-        <listenAddress>127.0.0.1:22003</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22026</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22026</globalAnnounceServer>
+        <listenAddress>tcp://127.0.0.1:22003</listenAddress>
+        <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
-        <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
+        <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
+        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
+        <relaysEnabled>true</relaysEnabled>
+        <relayReconnectIntervalM>10</relayReconnectIntervalM>
+        <relayWithoutGlobalAnn>false</relayWithoutGlobalAnn>
         <startBrowser>false</startBrowser>
         <upnpEnabled>false</upnpEnabled>
         <upnpLeaseMinutes>0</upnpLeaseMinutes>

test/h4/config.xml

@@ -1,4 +1,4 @@
-<configuration version="11">
+<configuration version="12">
     <folder id="default" path="s4" ro="false" rescanIntervalS="60" ignorePerms="false" autoNormalize="false">
         <device id="7PBCTLL-JJRYBSA-MOWZRKL-MSDMN4N-4US4OMX-SYEXUS4-HSBGNRY-CZXRXAT"></device>
         <minDiskFreePct>1</minDiskFreePct>
@@ -17,16 +17,19 @@
         <apikey>PMA5yUTG-Mw98nJ0YEtWTCHlM5O4aNi0</apikey>
     </gui>
     <options>
-        <listenAddress>127.0.0.1:22004</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22026</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22026</globalAnnounceServer>
+        <listenAddress>tcp://127.0.0.1:22004</listenAddress>
+        <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
-        <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
+        <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
+        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>60</reconnectionIntervalS>
+        <relaysEnabled>true</relaysEnabled>
+        <relayReconnectIntervalM>10</relayReconnectIntervalM>
+        <relayWithoutGlobalAnn>false</relayWithoutGlobalAnn>
         <startBrowser>false</startBrowser>
         <upnpEnabled>false</upnpEnabled>
         <upnpLeaseMinutes>60</upnpLeaseMinutes>