Docs update

Add separate client for dynamic relays (fixes #2368)

README.md

@@ -48,7 +48,7 @@ Please see the [Syncthing documentation site][6].
 
 All code is licensed under the [MPLv2 License][7].
 
-[1]: https://github.com/syncthing/specs/blob/master/BEPv1.md
+[1]: http://docs.syncthing.net/specs/bep-v1.html
 [2]: http://docs.syncthing.net/intro/getting-started.html
 [3]: https://github.com/syncthing/syncthing/blob/master/etc
 [4]: https://webchat.freenode.net/

build.go

@@ -195,7 +195,11 @@ func bench(pkg string) {
 }
 
 func install(pkg string, tags []string) {
-	os.Setenv("GOBIN", "./bin")
+	cwd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 	args := []string{"install", "-v", "-ldflags", ldflags()}
 	if len(tags) > 0 {
 		args = append(args, "-tags", strings.Join(tags, ","))

cmd/syncthing/main.go

@@ -635,6 +635,10 @@ func syncthingMain() {
 		l.Fatalln("Cannot open database:", err, "- Is another copy of Syncthing already running?")
 	}
 
+	protectedFiles := []string{
+		locations[locDatabase], locations[locConfigFile], locations[locCertFile], locations[locKeyFile],
+	}
+
 	// Remove database entries for folders that no longer exist in the config
 	folders := cfg.Folders()
 	for _, folder := range db.ListFolders(ldb) {
@@ -644,7 +648,7 @@ func syncthingMain() {
 		}
 	}
 
-	m := model.NewModel(cfg, myID, myName, "syncthing", Version, ldb)
+	m := model.NewModel(cfg, myID, myName, "syncthing", Version, ldb, protectedFiles)
 	cfg.Subscribe(m)
 
 	if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 {

cmd/syncthing/main_test.go

@@ -42,7 +42,7 @@ func TestFolderErrors(t *testing.T) {
 
 	// Case 1 - new folder, directory and marker created
 
-	m := model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m := model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
 
 	if err := m.CheckFolderHealth("folder"); err != nil {
@@ -73,7 +73,7 @@ func TestFolderErrors(t *testing.T) {
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
 
 	if err := m.CheckFolderHealth("folder"); err != nil {
@@ -96,7 +96,7 @@ func TestFolderErrors(t *testing.T) {
 		{Name: "dummyfile"},
 	})
 
-	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
 
 	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "folder marker missing" {
@@ -127,7 +127,7 @@ func TestFolderErrors(t *testing.T) {
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
 
 	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "folder path missing" {

lib/auto/gui.files.go

@@ -5,7 +5,7 @@ import (
 )
 
 const (
-	AssetsBuildDate = "Tue, 20 Oct 2015 07:59:38 GMT"
+	AssetsBuildDate = "Thu, 22 Oct 2015 06:14:16 GMT"
 )
 
 func Assets() map[string][]byte {

lib/beacon/broadcast.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/thejerf/suture"
 )
 
@@ -42,13 +43,15 @@ func NewBroadcast(port int) *Broadcast {
 	}
 
 	b.br = &broadcastReader{
-		port:   port,
-		outbox: b.outbox,
+		port:    port,
+		outbox:  b.outbox,
+		connMut: sync.NewMutex(),
 	}
 	b.Add(b.br)
 	b.bw = &broadcastWriter{
-		port:  port,
-		inbox: b.inbox,
+		port:    port,
+		inbox:   b.inbox,
+		connMut: sync.NewMutex(),
 	}
 	b.Add(b.bw)
 
@@ -72,9 +75,10 @@ func (b *Broadcast) Error() error {
 }
 
 type broadcastWriter struct {
-	port  int
-	inbox chan []byte
-	conn  *net.UDPConn
+	port    int
+	inbox   chan []byte
+	conn    *net.UDPConn
+	connMut sync.Mutex
 	errorHolder
 }
 
@@ -82,14 +86,17 @@ func (w *broadcastWriter) Serve() {
 	l.Debugln(w, "starting")
 	defer l.Debugln(w, "stopping")
 
-	var err error
-	w.conn, err = net.ListenUDP("udp4", nil)
+	conn, err := net.ListenUDP("udp4", nil)
 	if err != nil {
 		l.Debugln(err)
 		w.setError(err)
 		return
 	}
-	defer w.conn.Close()
+	defer conn.Close()
+
+	w.connMut.Lock()
+	w.conn = conn
+	w.connMut.Unlock()
 
 	for bs := range w.inbox {
 		addrs, err := net.InterfaceAddrs()
@@ -118,9 +125,9 @@ func (w *broadcastWriter) Serve() {
 		for _, ip := range dsts {
 			dst := &net.UDPAddr{IP: ip, Port: w.port}
 
-			w.conn.SetWriteDeadline(time.Now().Add(time.Second))
-			_, err := w.conn.WriteTo(bs, dst)
-			w.conn.SetWriteDeadline(time.Time{})
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			_, err := conn.WriteTo(bs, dst)
+			conn.SetWriteDeadline(time.Time{})
 
 			if err, ok := err.(net.Error); ok && err.Timeout() {
 				// Write timeouts should not happen. We treat it as a fatal
@@ -154,7 +161,11 @@ func (w *broadcastWriter) Serve() {
 }
 
 func (w *broadcastWriter) Stop() {
-	w.conn.Close()
+	w.connMut.Lock()
+	if w.conn != nil {
+		w.conn.Close()
+	}
+	w.connMut.Unlock()
 }
 
 func (w *broadcastWriter) String() string {
@@ -162,9 +173,10 @@ func (w *broadcastWriter) String() string {
 }
 
 type broadcastReader struct {
-	port   int
-	outbox chan recv
-	conn   *net.UDPConn
+	port    int
+	outbox  chan recv
+	conn    *net.UDPConn
+	connMut sync.Mutex
 	errorHolder
 }
 
@@ -172,18 +184,21 @@ func (r *broadcastReader) Serve() {
 	l.Debugln(r, "starting")
 	defer l.Debugln(r, "stopping")
 
-	var err error
-	r.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
+	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
 	if err != nil {
 		l.Debugln(err)
 		r.setError(err)
 		return
 	}
-	defer r.conn.Close()
+	defer conn.Close()
+
+	r.connMut.Lock()
+	r.conn = conn
+	r.connMut.Unlock()
 
 	bs := make([]byte, 65536)
 	for {
-		n, addr, err := r.conn.ReadFrom(bs)
+		n, addr, err := conn.ReadFrom(bs)
 		if err != nil {
 			l.Debugln(err)
 			r.setError(err)
@@ -206,7 +221,11 @@ func (r *broadcastReader) Serve() {
 }
 
 func (r *broadcastReader) Stop() {
-	r.conn.Close()
+	r.connMut.Lock()
+	if r.conn != nil {
+		r.conn.Close()
+	}
+	r.connMut.Unlock()
 }
 
 func (r *broadcastReader) String() string {

lib/db/benchmark_test.go

@@ -0,0 +1,188 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build benchmark
+
+package db_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+var files, oneFile, firstHalf, secondHalf []protocol.FileInfo
+var fs *db.FileSet
+
+func init() {
+	for i := 0; i < 1000; i++ {
+		files = append(files, protocol.FileInfo{
+			Name:    fmt.Sprintf("file%d", i),
+			Version: protocol.Vector{{ID: myID, Value: 1000}},
+			Blocks:  genBlocks(i),
+		})
+	}
+
+	middle := len(files) / 2
+	firstHalf = files[:middle]
+	secondHalf = files[middle:]
+	oneFile = firstHalf[middle-1 : middle]
+
+	ldb, _ := tempDB()
+	fs = db.NewFileSet("test", ldb)
+	fs.Replace(remoteDevice0, files)
+	fs.Replace(protocol.LocalDeviceID, firstHalf)
+}
+
+func tempDB() (*leveldb.DB, string) {
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		panic(err)
+	}
+	db, err := leveldb.OpenFile(filepath.Join(dir, "db"), &opt.Options{OpenFilesCacheCapacity: 100})
+	if err != nil {
+		panic(err)
+	}
+	return db, dir
+}
+
+func BenchmarkReplaceAll(b *testing.B) {
+	ldb, dir := tempDB()
+	defer func() {
+		ldb.Close()
+		os.RemoveAll(dir)
+	}()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m := db.NewFileSet("test", ldb)
+		m.Replace(protocol.LocalDeviceID, files)
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkUpdateOneChanged(b *testing.B) {
+	changed := make([]protocol.FileInfo, 1)
+	changed[0] = oneFile[0]
+	changed[0].Version = changed[0].Version.Update(myID)
+	changed[0].Blocks = genBlocks(len(changed[0].Blocks))
+
+	for i := 0; i < b.N; i++ {
+		if i%1 == 0 {
+			fs.Update(protocol.LocalDeviceID, changed)
+		} else {
+			fs.Update(protocol.LocalDeviceID, oneFile)
+		}
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkUpdateOneUnchanged(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		fs.Update(protocol.LocalDeviceID, oneFile)
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkNeedHalf(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		count := 0
+		fs.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+			count++
+			return true
+		})
+		if count != len(secondHalf) {
+			b.Errorf("wrong length %d != %d", count, len(secondHalf))
+		}
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkHave(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		count := 0
+		fs.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+			count++
+			return true
+		})
+		if count != len(firstHalf) {
+			b.Errorf("wrong length %d != %d", count, len(firstHalf))
+		}
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkGlobal(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		count := 0
+		fs.WithGlobal(func(fi db.FileIntf) bool {
+			count++
+			return true
+		})
+		if count != len(files) {
+			b.Errorf("wrong length %d != %d", count, len(files))
+		}
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkNeedHalfTruncated(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		count := 0
+		fs.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+			count++
+			return true
+		})
+		if count != len(secondHalf) {
+			b.Errorf("wrong length %d != %d", count, len(secondHalf))
+		}
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkHaveTruncated(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		count := 0
+		fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+			count++
+			return true
+		})
+		if count != len(firstHalf) {
+			b.Errorf("wrong length %d != %d", count, len(firstHalf))
+		}
+	}
+
+	b.ReportAllocs()
+}
+
+func BenchmarkGlobalTruncated(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		count := 0
+		fs.WithGlobalTruncated(func(fi db.FileIntf) bool {
+			count++
+			return true
+		})
+		if count != len(files) {
+			b.Errorf("wrong length %d != %d", count, len(files))
+		}
+	}
+
+	b.ReportAllocs()
+}

lib/db/blockmap.go

@@ -26,6 +26,8 @@ import (
 
 var blockFinder *BlockFinder
 
+const maxBatchSize = 256 << 10
+
 type BlockMap struct {
 	db     *leveldb.DB
 	folder string
@@ -42,14 +44,23 @@ func NewBlockMap(db *leveldb.DB, folder string) *BlockMap {
 func (m *BlockMap) Add(files []protocol.FileInfo) error {
 	batch := new(leveldb.Batch)
 	buf := make([]byte, 4)
+	var key []byte
 	for _, file := range files {
+		if batch.Len() > maxBatchSize {
+			if err := m.db.Write(batch, nil); err != nil {
+				return err
+			}
+			batch.Reset()
+		}
+
 		if file.IsDirectory() || file.IsDeleted() || file.IsInvalid() {
 			continue
 		}
 
 		for i, block := range file.Blocks {
 			binary.BigEndian.PutUint32(buf, uint32(i))
-			batch.Put(m.blockKey(block.Hash, file.Name), buf)
+			key = m.blockKeyInto(key, block.Hash, file.Name)
+			batch.Put(key, buf)
 		}
 	}
 	return m.db.Write(batch, nil)
@@ -59,21 +70,31 @@ func (m *BlockMap) Add(files []protocol.FileInfo) error {
 func (m *BlockMap) Update(files []protocol.FileInfo) error {
 	batch := new(leveldb.Batch)
 	buf := make([]byte, 4)
+	var key []byte
 	for _, file := range files {
+		if batch.Len() > maxBatchSize {
+			if err := m.db.Write(batch, nil); err != nil {
+				return err
+			}
+			batch.Reset()
+		}
+
 		if file.IsDirectory() {
 			continue
 		}
 
 		if file.IsDeleted() || file.IsInvalid() {
 			for _, block := range file.Blocks {
-				batch.Delete(m.blockKey(block.Hash, file.Name))
+				key = m.blockKeyInto(key, block.Hash, file.Name)
+				batch.Delete(key)
 			}
 			continue
 		}
 
 		for i, block := range file.Blocks {
 			binary.BigEndian.PutUint32(buf, uint32(i))
-			batch.Put(m.blockKey(block.Hash, file.Name), buf)
+			key = m.blockKeyInto(key, block.Hash, file.Name)
+			batch.Put(key, buf)
 		}
 	}
 	return m.db.Write(batch, nil)
@@ -82,9 +103,18 @@ func (m *BlockMap) Update(files []protocol.FileInfo) error {
 // Discard block map state, removing the given files
 func (m *BlockMap) Discard(files []protocol.FileInfo) error {
 	batch := new(leveldb.Batch)
+	var key []byte
 	for _, file := range files {
+		if batch.Len() > maxBatchSize {
+			if err := m.db.Write(batch, nil); err != nil {
+				return err
+			}
+			batch.Reset()
+		}
+
 		for _, block := range file.Blocks {
-			batch.Delete(m.blockKey(block.Hash, file.Name))
+			key = m.blockKeyInto(key, block.Hash, file.Name)
+			batch.Delete(key)
 		}
 	}
 	return m.db.Write(batch, nil)
@@ -93,9 +123,16 @@ func (m *BlockMap) Discard(files []protocol.FileInfo) error {
 // Drop block map, removing all entries related to this block map from the db.
 func (m *BlockMap) Drop() error {
 	batch := new(leveldb.Batch)
-	iter := m.db.NewIterator(util.BytesPrefix(m.blockKey(nil, "")[:1+64]), nil)
+	iter := m.db.NewIterator(util.BytesPrefix(m.blockKeyInto(nil, nil, "")[:1+64]), nil)
 	defer iter.Release()
 	for iter.Next() {
+		if batch.Len() > maxBatchSize {
+			if err := m.db.Write(batch, nil); err != nil {
+				return err
+			}
+			batch.Reset()
+		}
+
 		batch.Delete(iter.Key())
 	}
 	if iter.Error() != nil {
@@ -104,8 +141,8 @@ func (m *BlockMap) Drop() error {
 	return m.db.Write(batch, nil)
 }
 
-func (m *BlockMap) blockKey(hash []byte, file string) []byte {
-	return toBlockKey(hash, m.folder, file)
+func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte {
+	return blockKeyInto(o, hash, m.folder, file)
 }
 
 type BlockFinder struct {
@@ -134,8 +171,9 @@ func (f *BlockFinder) String() string {
 // reason. The iterator finally returns the result, whether or not a
 // satisfying block was eventually found.
 func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string, string, int32) bool) bool {
+	var key []byte
 	for _, folder := range folders {
-		key := toBlockKey(hash, folder, "")
+		key = blockKeyInto(key, hash, folder, "")
 		iter := f.db.NewIterator(util.BytesPrefix(key), nil)
 		defer iter.Release()
 
@@ -157,8 +195,8 @@ func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []b
 	binary.BigEndian.PutUint32(buf, uint32(index))
 
 	batch := new(leveldb.Batch)
-	batch.Delete(toBlockKey(oldHash, folder, file))
-	batch.Put(toBlockKey(newHash, folder, file), buf)
+	batch.Delete(blockKeyInto(nil, oldHash, folder, file))
+	batch.Put(blockKeyInto(nil, newHash, folder, file), buf)
 	return f.db.Write(batch, nil)
 }
 
@@ -167,8 +205,13 @@ func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []b
 //	   folder (64 bytes)
 //	   block hash (32 bytes)
 //	   file name (variable size)
-func toBlockKey(hash []byte, folder, file string) []byte {
-	o := make([]byte, 1+64+32+len(file))
+func blockKeyInto(o, hash []byte, folder, file string) []byte {
+	reqLen := 1 + 64 + 32 + len(file)
+	if cap(o) < reqLen {
+		o = make([]byte, reqLen)
+	} else {
+		o = o[:reqLen]
+	}
 	o[0] = KeyTypeBlock
 	copy(o[1:], []byte(folder))
 	copy(o[1+64:], []byte(hash))

lib/db/leveldb.go

@@ -12,7 +12,6 @@ package db
 import (
 	"bytes"
 	"fmt"
-	"runtime"
 	"sort"
 
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -169,9 +168,7 @@ func globalKeyFolder(key []byte) []byte {
 
 type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64
 
-func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) int64 {
-	runtime.GC()
-
+func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 {
 	sort.Sort(fileList(fs)) // sort list on name, same as in the database
 
 	start := deviceKey(folder, device, nil)                            // before all folder/device files
@@ -196,6 +193,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 	fsi := 0
 	var maxLocalVer int64
 
+	isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
 	for {
 		var newName, oldName []byte
 		moreFs := fsi < len(fs)
@@ -223,10 +221,13 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 			if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
+			if isLocalDevice {
+				localSize.addFile(fs[fsi])
+			}
 			if fs[fsi].IsInvalid() {
-				ldbRemoveFromGlobal(snap, batch, folder, device, newName)
+				ldbRemoveFromGlobal(snap, batch, folder, device, newName, globalSize)
 			} else {
-				ldbUpdateGlobal(snap, batch, folder, device, fs[fsi])
+				ldbUpdateGlobal(snap, batch, folder, device, fs[fsi], globalSize)
 			}
 			fsi++
 
@@ -242,10 +243,14 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 				if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
 					maxLocalVer = lv
 				}
+				if isLocalDevice {
+					localSize.removeFile(ef)
+					localSize.addFile(fs[fsi])
+				}
 				if fs[fsi].IsInvalid() {
-					ldbRemoveFromGlobal(snap, batch, folder, device, newName)
+					ldbRemoveFromGlobal(snap, batch, folder, device, newName, globalSize)
 				} else {
-					ldbUpdateGlobal(snap, batch, folder, device, fs[fsi])
+					ldbUpdateGlobal(snap, batch, folder, device, fs[fsi], globalSize)
 				}
 			} else {
 				l.Debugln("generic replace; equal - ignore")
@@ -285,21 +290,19 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 	return maxLocalVer
 }
 
-func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 {
+func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
 	// TODO: Return the remaining maxLocalVer?
-	return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
+	return ldbGenericReplace(db, folder, device, fs, localSize, globalSize, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
 		// Database has a file that we are missing. Remove it.
 		l.Debugf("delete; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name)
-		ldbRemoveFromGlobal(db, batch, folder, device, name)
+		ldbRemoveFromGlobal(db, batch, folder, device, name, globalSize)
 		l.Debugf("batch.Delete %p %x", batch, dbi.Key())
 		batch.Delete(dbi.Key())
 		return 0
 	})
 }
 
-func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 {
-	runtime.GC()
-
+func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
 	batch := new(leveldb.Batch)
 	l.Debugf("new batch %p", batch)
 	snap, err := db.GetSnapshot()
@@ -314,19 +317,24 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 
 	var maxLocalVer int64
 	var fk []byte
+	isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
 	for _, f := range fs {
 		name := []byte(f.Name)
 		fk = deviceKeyInto(fk[:cap(fk)], folder, device, name)
 		l.Debugf("snap.Get %p %x", snap, fk)
 		bs, err := snap.Get(fk, nil)
 		if err == leveldb.ErrNotFound {
+			if isLocalDevice {
+				localSize.addFile(f)
+			}
+
 			if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
 			if f.IsInvalid() {
-				ldbRemoveFromGlobal(snap, batch, folder, device, name)
+				ldbRemoveFromGlobal(snap, batch, folder, device, name, globalSize)
 			} else {
-				ldbUpdateGlobal(snap, batch, folder, device, f)
+				ldbUpdateGlobal(snap, batch, folder, device, f, globalSize)
 			}
 			continue
 		}
@@ -339,13 +347,18 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 		// Flags might change without the version being bumped when we set the
 		// invalid flag on an existing file.
 		if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
+			if isLocalDevice {
+				localSize.removeFile(ef)
+				localSize.addFile(f)
+			}
+
 			if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
 			if f.IsInvalid() {
-				ldbRemoveFromGlobal(snap, batch, folder, device, name)
+				ldbRemoveFromGlobal(snap, batch, folder, device, name, globalSize)
 			} else {
-				ldbUpdateGlobal(snap, batch, folder, device, f)
+				ldbUpdateGlobal(snap, batch, folder, device, f, globalSize)
 			}
 		}
 
@@ -390,7 +403,7 @@ func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) in
 // ldbUpdateGlobal adds this device+version to the version list for the given
 // file. If the device is already present in the list, the version is updated.
 // If the file does not have an entry in the global list, it is created.
-func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file protocol.FileInfo) bool {
+func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file protocol.FileInfo, globalSize *sizeTracker) bool {
 	l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version)
 	name := []byte(file.Name)
 	gk := globalKey(folder, name)
@@ -400,7 +413,8 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 	}
 
 	var fl versionList
-
+	var oldFile protocol.FileInfo
+	var hasOldFile bool
 	// Remove the device from the current version list
 	if svl != nil {
 		err = fl.UnmarshalXDR(svl)
@@ -414,6 +428,13 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 					// No need to do anything
 					return false
 				}
+
+				if i == 0 {
+					// Keep the current newest file around so we can subtract it from
+					// the globalSize if we replace it.
+					oldFile, hasOldFile = ldbGet(db, folder, fl.versions[0].device, name)
+				}
+
 				fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
 				break
 			}
@@ -425,6 +446,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 		version: file.Version,
 	}
 
+	insertedAt := -1
 	// Find a position in the list to insert this file. The file at the front
 	// of the list is the newer, the "global".
 	for i := range fl.versions {
@@ -433,6 +455,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 			// The version at this point in the list is equal to or lesser
 			// ("older") than us. We insert ourselves in front of it.
 			fl.versions = insertVersion(fl.versions, i, nv)
+			insertedAt = i
 			goto done
 
 		case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
@@ -448,6 +471,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 			}
 			if file.WinsConflict(of) {
 				fl.versions = insertVersion(fl.versions, i, nv)
+				insertedAt = i
 				goto done
 			}
 		}
@@ -455,8 +479,28 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 
 	// We didn't find a position for an insert above, so append to the end.
 	fl.versions = append(fl.versions, nv)
+	insertedAt = len(fl.versions) - 1
 
 done:
+	if insertedAt == 0 {
+		// We just inserted a new newest version. Fixup the global size
+		// calculation.
+		if !file.Version.Equal(oldFile.Version) {
+			globalSize.addFile(file)
+			if hasOldFile {
+				// We have the old file that was removed at the head of the list.
+				globalSize.removeFile(oldFile)
+			} else if len(fl.versions) > 1 {
+				// The previous newest version is now at index 1, grab it from there.
+				oldFile, ok := ldbGet(db, folder, fl.versions[1].device, name)
+				if !ok {
+					panic("file referenced in version list does not exist")
+				}
+				globalSize.removeFile(oldFile)
+			}
+		}
+	}
+
 	l.Debugf("batch.Put %p %x", batch, gk)
 	l.Debugf("new global after update: %v", fl)
 	batch.Put(gk, fl.MustMarshalXDR())
@@ -474,7 +518,7 @@ func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
 // ldbRemoveFromGlobal removes the device from the global version list for the
 // given file. If the version list is empty after this, the file entry is
 // removed entirely.
-func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byte) {
+func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byte, globalSize *sizeTracker) {
 	l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
 
 	gk := globalKey(folder, file)
@@ -491,8 +535,17 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
 		panic(err)
 	}
 
+	removed := false
 	for i := range fl.versions {
 		if bytes.Compare(fl.versions[i].device, device) == 0 {
+			if i == 0 && globalSize != nil {
+				f, ok := ldbGet(db, folder, device, file)
+				if !ok {
+					panic("removing nonexistent file")
+				}
+				globalSize.removeFile(f)
+				removed = true
+			}
 			fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
 			break
 		}
@@ -505,6 +558,13 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
 		l.Debugf("batch.Put %p %x", batch, gk)
 		l.Debugf("new global after remove: %v", fl)
 		batch.Put(gk, fl.MustMarshalXDR())
+		if removed {
+			f, ok := ldbGet(db, folder, fl.versions[0].device, file)
+			if !ok {
+				panic("new global is nonexistent file")
+			}
+			globalSize.addFile(f)
+		}
 	}
 }
 
@@ -536,8 +596,6 @@ func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterat
 }
 
 func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
-	runtime.GC()
-
 	start := deviceKey(folder, nil, nil)                                                  // before all folder/device files
 	limit := deviceKey(folder, protocol.LocalDeviceID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
 	snap, err := db.GetSnapshot()
@@ -565,7 +623,7 @@ func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []b
 		case "", ".", "..", "/": // A few obviously invalid filenames
 			l.Infof("Dropping invalid filename %q from database", f.Name)
 			batch := new(leveldb.Batch)
-			ldbRemoveFromGlobal(db, batch, folder, device, nil)
+			ldbRemoveFromGlobal(db, batch, folder, device, nil, nil)
 			batch.Delete(dbi.Key())
 			db.Write(batch, nil)
 			continue
@@ -641,8 +699,6 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf,
 }
 
 func ldbWithGlobal(db *leveldb.DB, folder, prefix []byte, truncate bool, fn Iterator) {
-	runtime.GC()
-
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -722,8 +778,6 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
 }
 
 func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
-	runtime.GC()
-
 	start := globalKey(folder, nil)
 	limit := globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff})
 	snap, err := db.GetSnapshot()
@@ -822,8 +876,6 @@ nextFile:
 }
 
 func ldbListFolders(db *leveldb.DB) []string {
-	runtime.GC()
-
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -855,8 +907,6 @@ func ldbListFolders(db *leveldb.DB) []string {
 }
 
 func ldbDropFolder(db *leveldb.DB, folder []byte) {
-	runtime.GC()
-
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -900,9 +950,7 @@ func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	return tf, err
 }
 
-func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
-	defer runtime.GC()
-
+func ldbCheckGlobals(db *leveldb.DB, folder []byte, globalSize *sizeTracker) {
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -937,8 +985,9 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 
 		name := globalKeyName(gk)
 		var newVL versionList
-		for _, version := range vl.versions {
+		for i, version := range vl.versions {
 			fk = deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
+
 			l.Debugf("snap.Get %p %x", snap, fk)
 			_, err := snap.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
@@ -948,6 +997,14 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 				panic(err)
 			}
 			newVL.versions = append(newVL.versions, version)
+
+			if i == 0 {
+				fi, ok := ldbGet(snap, folder, version.device, name)
+				if !ok {
+					panic("nonexistent global master file")
+				}
+				globalSize.addFile(fi)
+			}
 		}
 
 		if len(newVL.versions) != len(vl.versions) {

lib/db/set.go

@@ -13,6 +13,8 @@
 package db
 
 import (
+	stdsync "sync"
+
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -25,6 +27,8 @@ type FileSet struct {
 	folder       string
 	db           *leveldb.DB
 	blockmap     *BlockMap
+	localSize    sizeTracker
+	globalSize   sizeTracker
 }
 
 // FileIntf is the set of methods implemented by both protocol.FileInfo and
@@ -43,6 +47,52 @@ type FileIntf interface {
 // continue iteration, false to stop.
 type Iterator func(f FileIntf) bool
 
+type sizeTracker struct {
+	files   int
+	deleted int
+	bytes   int64
+	mut     stdsync.Mutex
+}
+
+func (s *sizeTracker) addFile(f FileIntf) {
+	if f.IsInvalid() {
+		return
+	}
+
+	s.mut.Lock()
+	if f.IsDeleted() {
+		s.deleted++
+	} else {
+		s.files++
+	}
+	s.bytes += f.Size()
+	s.mut.Unlock()
+}
+
+func (s *sizeTracker) removeFile(f FileIntf) {
+	if f.IsInvalid() {
+		return
+	}
+
+	s.mut.Lock()
+	if f.IsDeleted() {
+		s.deleted--
+	} else {
+		s.files--
+	}
+	s.bytes -= f.Size()
+	if s.deleted < 0 || s.files < 0 {
+		panic("bug: removed more than added")
+	}
+	s.mut.Unlock()
+}
+
+func (s *sizeTracker) Size() (files, deleted int, bytes int64) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+	return s.files, s.deleted, s.bytes
+}
+
 func NewFileSet(folder string, db *leveldb.DB) *FileSet {
 	var s = FileSet{
 		localVersion: make(map[protocol.DeviceID]int64),
@@ -52,7 +102,7 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
 		mutex:        sync.NewMutex(),
 	}
 
-	ldbCheckGlobals(db, []byte(folder))
+	ldbCheckGlobals(db, []byte(folder), &s.globalSize)
 
 	var deviceID protocol.DeviceID
 	ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f FileInfoTruncated) bool {
@@ -60,6 +110,9 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
 		if f.LocalVersion > s.localVersion[deviceID] {
 			s.localVersion[deviceID] = f.LocalVersion
 		}
+		if deviceID == protocol.LocalDeviceID {
+			s.localSize.addFile(f)
+		}
 		return true
 	})
 	l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion)
@@ -73,7 +126,7 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
 	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
-	s.localVersion[device] = ldbReplace(s.db, []byte(s.folder), device[:], fs)
+	s.localVersion[device] = ldbReplace(s.db, []byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
 	if len(fs) == 0 {
 		// Reset the local version if all files were removed.
 		s.localVersion[device] = 0
@@ -102,7 +155,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 		s.blockmap.Discard(discards)
 		s.blockmap.Update(updates)
 	}
-	if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs); lv > s.localVersion[device] {
+	if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs, &s.localSize, &s.globalSize); lv > s.localVersion[device] {
 		s.localVersion[device] = lv
 	}
 }
@@ -178,6 +231,14 @@ func (s *FileSet) LocalVersion(device protocol.DeviceID) int64 {
 	return s.localVersion[device]
 }
 
+func (s *FileSet) LocalSize() (files, deleted int, bytes int64) {
+	return s.localSize.Size()
+}
+
+func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
+	return s.globalSize.Size()
+}
+
 // ListFolders returns the folder IDs seen in the database.
 func ListFolders(db *leveldb.DB) []string {
 	return ldbListFolders(db)

lib/db/set_test.go

@@ -173,6 +173,29 @@ func TestGlobalSet(t *testing.T) {
 		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
 	}
 
+	globalFiles, globalDeleted, globalBytes := 0, 0, int64(0)
+	for _, f := range g {
+		if f.IsInvalid() {
+			continue
+		}
+		if f.IsDeleted() {
+			globalDeleted++
+		} else {
+			globalFiles++
+		}
+		globalBytes += f.Size()
+	}
+	gsFiles, gsDeleted, gsBytes := m.GlobalSize()
+	if gsFiles != globalFiles {
+		t.Errorf("Incorrect GlobalSize files; %d != %d", gsFiles, globalFiles)
+	}
+	if gsDeleted != globalDeleted {
+		t.Errorf("Incorrect GlobalSize deleted; %d != %d", gsDeleted, globalDeleted)
+	}
+	if gsBytes != globalBytes {
+		t.Errorf("Incorrect GlobalSize bytes; %d != %d", gsBytes, globalBytes)
+	}
+
 	h := fileList(haveList(m, protocol.LocalDeviceID))
 	sort.Sort(h)
 
@@ -180,6 +203,29 @@ func TestGlobalSet(t *testing.T) {
 		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, localTot)
 	}
 
+	haveFiles, haveDeleted, haveBytes := 0, 0, int64(0)
+	for _, f := range h {
+		if f.IsInvalid() {
+			continue
+		}
+		if f.IsDeleted() {
+			haveDeleted++
+		} else {
+			haveFiles++
+		}
+		haveBytes += f.Size()
+	}
+	lsFiles, lsDeleted, lsBytes := m.LocalSize()
+	if lsFiles != haveFiles {
+		t.Errorf("Incorrect LocalSize files; %d != %d", lsFiles, haveFiles)
+	}
+	if lsDeleted != haveDeleted {
+		t.Errorf("Incorrect LocalSize deleted; %d != %d", lsDeleted, haveDeleted)
+	}
+	if lsBytes != haveBytes {
+		t.Errorf("Incorrect LocalSize bytes; %d != %d", lsBytes, haveBytes)
+	}
+
 	h = fileList(haveList(m, remoteDevice0))
 	sort.Sort(h)
 
@@ -371,180 +417,6 @@ func TestInvalidAvailability(t *testing.T) {
 		t.Error("Incorrect availability for 'none':", av)
 	}
 }
-func Benchmark10kReplace(b *testing.B) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	var local []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m := db.NewFileSet("test", ldb)
-		m.Replace(protocol.LocalDeviceID, local)
-	}
-}
-
-func Benchmark10kUpdateChg(b *testing.B) {
-	var remote []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	m := db.NewFileSet("test", ldb)
-	m.Replace(remoteDevice0, remote)
-
-	var local []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	m.Replace(protocol.LocalDeviceID, local)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		for j := range local {
-			local[j].Version = local[j].Version.Update(myID)
-		}
-		b.StartTimer()
-		m.Update(protocol.LocalDeviceID, local)
-	}
-}
-
-func Benchmark10kUpdateSme(b *testing.B) {
-	var remote []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-	m := db.NewFileSet("test", ldb)
-	m.Replace(remoteDevice0, remote)
-
-	var local []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	m.Replace(protocol.LocalDeviceID, local)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.Update(protocol.LocalDeviceID, local)
-	}
-}
-
-func Benchmark10kNeed2k(b *testing.B) {
-	var remote []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	m := db.NewFileSet("test", ldb)
-	m.Replace(remoteDevice0, remote)
-
-	var local []protocol.FileInfo
-	for i := 0; i < 8000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-	for i := 8000; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
-	}
-
-	m.Replace(protocol.LocalDeviceID, local)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		fs := needList(m, protocol.LocalDeviceID)
-		if l := len(fs); l != 2000 {
-			b.Errorf("wrong length %d != 2k", l)
-		}
-	}
-}
-
-func Benchmark10kHaveFullList(b *testing.B) {
-	var remote []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	m := db.NewFileSet("test", ldb)
-	m.Replace(remoteDevice0, remote)
-
-	var local []protocol.FileInfo
-	for i := 0; i < 2000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-	for i := 2000; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
-	}
-
-	m.Replace(protocol.LocalDeviceID, local)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		fs := haveList(m, protocol.LocalDeviceID)
-		if l := len(fs); l != 10000 {
-			b.Errorf("wrong length %d != 10k", l)
-		}
-	}
-}
-
-func Benchmark10kGlobal(b *testing.B) {
-	var remote []protocol.FileInfo
-	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	m := db.NewFileSet("test", ldb)
-	m.Replace(remoteDevice0, remote)
-
-	var local []protocol.FileInfo
-	for i := 0; i < 2000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
-	}
-	for i := 2000; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
-	}
-
-	m.Replace(protocol.LocalDeviceID, local)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		fs := globalList(m)
-		if l := len(fs); l != 10000 {
-			b.Errorf("wrong length %d != 10k", l)
-		}
-	}
-}
 
 func TestGlobalReset(t *testing.T) {
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)

lib/db/truncated.go

@@ -6,22 +6,44 @@
 
 package db
 
-import "github.com/syncthing/syncthing/lib/protocol"
+import (
+	"bytes"
+
+	"github.com/calmh/xdr"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
 
 type FileInfoTruncated struct {
 	protocol.FileInfo
-	ActualSize int64
 }
 
-func (f *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
-	err := f.FileInfo.UnmarshalXDR(bs)
-	f.ActualSize = f.FileInfo.Size()
-	f.FileInfo.Blocks = nil
-	return err
+func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (f FileInfoTruncated) Size() int64 {
-	return f.ActualSize
+func (o *FileInfoTruncated) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.Name = xr.ReadStringMax(8192)
+	o.Flags = xr.ReadUint32()
+	o.Modified = int64(xr.ReadUint64())
+	(&o.Version).DecodeXDRFrom(xr)
+	o.LocalVersion = int64(xr.ReadUint64())
+	_BlocksSize := int(xr.ReadUint32())
+	if _BlocksSize < 0 {
+		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000)
+	}
+	if _BlocksSize > 1000000 {
+		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000)
+	}
+
+	buf := make([]byte, 64)
+	for i := 0; i < _BlocksSize; i++ {
+		size := xr.ReadUint32()
+		o.CachedSize += int64(size)
+		xr.ReadBytesMaxInto(64, buf)
+	}
+	return xr.Error()
 }
 
 func BlocksToSize(num int) int64 {

lib/discover/global.go

@@ -42,6 +42,7 @@ type httpClient interface {
 const (
 	defaultReannounceInterval  = 30 * time.Minute
 	announceErrorRetryInterval = 5 * time.Minute
+	requestTimeout             = 5 * time.Second
 )
 
 type announcement struct {
@@ -73,6 +74,7 @@ func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, rela
 	// certificate to prove our identity, and may or may not verify the server
 	// certificate depending on the insecure setting.
 	var announceClient httpClient = &http.Client{
+		Timeout: requestTimeout,
 		Transport: &http.Transport{
 			Dial:  dialer.Dial,
 			Proxy: http.ProxyFromEnvironment,
@@ -89,6 +91,7 @@ func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, rela
 	// The http.Client used for queries. We don't need to present our
 	// certificate here, so lets not include it. May be insecure if requested.
 	var queryClient httpClient = &http.Client{
+		Timeout: requestTimeout,
 		Transport: &http.Transport{
 			Dial:  dialer.Dial,
 			Proxy: http.ProxyFromEnvironment,

lib/discover/global_test.go

@@ -79,11 +79,26 @@ func TestGlobalOverHTTP(t *testing.T) {
 	mux.HandleFunc("/", s.handler)
 	go http.Serve(list, mux)
 
+	// This should succeed
 	direct, relays, err := testLookup("http://" + list.Addr().String() + "?insecure&noannounce")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
+	if !testing.Short() {
+		// This should time out
+		_, _, err = testLookup("http://" + list.Addr().String() + "/block?insecure&noannounce")
+		if err == nil {
+			t.Fatalf("unexpected nil error, should have been a timeout")
+		}
+	}
+
+	// This should work again
+	_, _, err = testLookup("http://" + list.Addr().String() + "?insecure&noannounce")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
 	if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
 		t.Errorf("incorrect direct list: %+v", direct)
 	}
@@ -231,6 +246,11 @@ type fakeDiscoveryServer struct {
 }
 
 func (s *fakeDiscoveryServer) handler(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path == "/block" {
+		// Never return for requests here
+		select {}
+	}
+
 	if r.Method == "POST" {
 		s.announce, _ = ioutil.ReadAll(r.Body)
 		w.WriteHeader(204)

lib/model/model.go

@@ -70,6 +70,7 @@ type Model struct {
 	id                protocol.DeviceID
 	shortID           uint64
 	cacheIgnoredFiles bool
+	protectedFiles    []string
 
 	deviceName    string
 	clientName    string
@@ -98,7 +99,7 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local folder in any way.
-func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB) *Model {
+func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB, protectedFiles []string) *Model {
 	m := &Model{
 		Supervisor: suture.New("model", suture.Spec{
 			Log: func(line string) {
@@ -112,6 +113,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		id:                id,
 		shortID:           id.Short(),
 		cacheIgnoredFiles: cfg.Options().CacheIgnoredFiles,
+		protectedFiles:    protectedFiles,
 		deviceName:        deviceName,
 		clientName:        clientName,
 		clientVersion:     clientVersion,
@@ -180,11 +182,41 @@ func (m *Model) StartFolderRW(folder string) {
 		p.versioner = versioner
 	}
 
+	m.warnAboutOverwritingProtectedFiles(folder)
+
 	m.Add(p)
 
 	l.Okln("Ready to synchronize", folder, "(read-write)")
 }
 
+func (m *Model) warnAboutOverwritingProtectedFiles(folder string) {
+	if m.folderCfgs[folder].ReadOnly {
+		return
+	}
+
+	folderLocation := m.folderCfgs[folder].Path()
+	ignores := m.folderIgnores[folder]
+
+	var filesAtRisk []string
+	for _, protectedFilePath := range m.protectedFiles {
+		// check if file is synced in this folder
+		if !strings.HasPrefix(protectedFilePath, folderLocation) {
+			continue
+		}
+
+		// check if file is ignored
+		if ignores.Match(protectedFilePath) {
+			continue
+		}
+
+		filesAtRisk = append(filesAtRisk, protectedFilePath)
+	}
+
+	if len(filesAtRisk) > 0 {
+		l.Warnln("Some protected files may be overwritten and cause issues. See http://docs.syncthing.net/users/config.html#syncing-configuration-files for more information. The at risk files are:", strings.Join(filesAtRisk, ", "))
+	}
+}
+
 // StartFolderRO starts read only processing on the current model. When in
 // read only mode the model will announce files to the cluster but not pull in
 // any external changes.
@@ -297,8 +329,6 @@ func (m *Model) FolderStatistics() map[string]stats.FolderStatistics {
 // Completion returns the completion status, in percent, for the given device
 // and folder.
 func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
-	var tot int64
-
 	m.fmut.RLock()
 	rf, ok := m.folderFiles[folder]
 	m.fmut.RUnlock()
@@ -306,29 +336,22 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		return 0 // Folder doesn't exist, so we hardly have any of it
 	}
 
-	rf.WithGlobalTruncated(func(f db.FileIntf) bool {
-		if !f.IsDeleted() {
-			tot += f.Size()
-		}
-		return true
-	})
-
+	_, _, tot := rf.GlobalSize()
 	if tot == 0 {
 		return 100 // Folder is empty, so we have all of it
 	}
 
 	var need int64
 	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
-		if !f.IsDeleted() {
-			need += f.Size()
-		}
+		need += f.Size()
 		return true
 	})
 
-	res := 100 * (1 - float64(need)/float64(tot))
-	l.Debugf("%v Completion(%s, %q): %f (%d / %d)", m, device, folder, res, need, tot)
+	needRatio := float64(need) / float64(tot)
+	completionPct := 100 * (1 - needRatio)
+	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio)
 
-	return res
+	return completionPct
 }
 
 func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
@@ -357,13 +380,7 @@ func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithGlobalTruncated(func(f db.FileIntf) bool {
-			fs, de, by := sizeOfFile(f)
-			nfiles += fs
-			deleted += de
-			bytes += by
-			return true
-		})
+		nfiles, deleted, bytes = rf.GlobalSize()
 	}
 	return
 }
@@ -374,16 +391,7 @@ func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithHaveTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
-			if f.IsInvalid() {
-				return true
-			}
-			fs, de, by := sizeOfFile(f)
-			nfiles += fs
-			deleted += de
-			bytes += by
-			return true
-		})
+		nfiles, deleted, bytes = rf.LocalSize()
 	}
 	return
 }

lib/model/model_test.go

@@ -92,7 +92,7 @@ func init() {
 func TestRequest(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 
 	// device1 shares default, but device2 doesn't
 	m.AddFolder(defaultFolderConfig)
@@ -168,7 +168,7 @@ func BenchmarkIndex_100(b *testing.B) {
 
 func benchmarkIndex(b *testing.B, nfiles int) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.StartFolderRO("default")
 	m.ServeBackground()
@@ -197,7 +197,7 @@ func BenchmarkIndexUpdate_10000_1(b *testing.B) {
 
 func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.StartFolderRO("default")
 	m.ServeBackground()
@@ -262,7 +262,7 @@ func (FakeConnection) Statistics() protocol.Statistics {
 
 func BenchmarkRequest(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
 	m.ScanFolder("default")
@@ -318,7 +318,7 @@ func TestDeviceRename(t *testing.T) {
 	cfg := config.Wrap("tmpconfig.xml", rawCfg)
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 
 	fc := FakeConnection{
 		id:          device1,
@@ -393,7 +393,7 @@ func TestClusterConfig(t *testing.T) {
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 
-	m := NewModel(config.Wrap("/tmp/test", cfg), protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(config.Wrap("/tmp/test", cfg), protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(cfg.Folders[0])
 	m.AddFolder(cfg.Folders[1])
 	m.ServeBackground()
@@ -464,7 +464,7 @@ func TestIgnores(t *testing.T) {
 	ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.StartFolderRO("default")
 	m.ServeBackground()
@@ -539,7 +539,7 @@ func TestIgnores(t *testing.T) {
 
 func TestRefuseUnknownBits(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
 
@@ -598,7 +598,7 @@ func TestROScanRecovery(t *testing.T) {
 
 	os.RemoveAll(fcfg.RawPath)
 
-	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
 	m.StartFolderRO("default")
 	m.ServeBackground()
@@ -682,7 +682,7 @@ func TestRWScanRecovery(t *testing.T) {
 
 	os.RemoveAll(fcfg.RawPath)
 
-	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
 	m.StartFolderRW("default")
 	m.ServeBackground()
@@ -745,7 +745,7 @@ func TestRWScanRecovery(t *testing.T) {
 
 func TestGlobalDirectoryTree(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
 
@@ -995,7 +995,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 
 func TestGlobalDirectorySelfFixing(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
 
@@ -1169,7 +1169,7 @@ func BenchmarkTree_100_10(b *testing.B) {
 
 func benchmarkTree(b *testing.B, n1, n2 int) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
 
@@ -1187,7 +1187,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
 
 func TestIgnoreDelete(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 
 	// This folder should ignore external deletes
 	cfg := defaultFolderConfig

lib/model/rwfolder_test.go

@@ -70,7 +70,7 @@ func TestHandleFile(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocals("default", []protocol.FileInfo{existingFile})
@@ -126,7 +126,7 @@ func TestHandleFileWithTemp(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocals("default", []protocol.FileInfo{existingFile})
@@ -188,7 +188,7 @@ func TestCopierFinder(t *testing.T) {
 	requiredFile.Name = "file2"
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocals("default", []protocol.FileInfo{existingFile})
@@ -265,7 +265,7 @@ func TestCopierCleanup(t *testing.T) {
 	}
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 
 	// Create a file
@@ -314,7 +314,7 @@ func TestCopierCleanup(t *testing.T) {
 // if it fails to find the block.
 func TestLastResortPulling(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 
 	// Add a file to index (with the incorrect block representation, as content
@@ -389,7 +389,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 
 	emitter := NewProgressEmitter(defaultConfig)
@@ -481,7 +481,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 
 	emitter := NewProgressEmitter(defaultConfig)

lib/protocol/message.go

@@ -33,9 +33,13 @@ func (f FileInfo) Size() (bytes int64) {
 	if f.IsDeleted() || f.IsDirectory() {
 		return 128
 	}
+	if f.CachedSize > 0 {
+		return f.CachedSize
+	}
 	for _, b := range f.Blocks {
 		bytes += int64(b.Size)
 	}
+	f.CachedSize = bytes
 	return
 }
 

lib/protocol/protocol_test.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"encoding/hex"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,11 +20,21 @@ import (
 )
 
 var (
-	c0ID = NewDeviceID([]byte{1})
-	c1ID = NewDeviceID([]byte{2})
+	c0ID     = NewDeviceID([]byte{1})
+	c1ID     = NewDeviceID([]byte{2})
+	quickCfg = &quick.Config{}
 )
 
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if flag.Lookup("test.short").Value.String() != "false" {
+		quickCfg.MaxCount = 10
+	}
+	os.Exit(m.Run())
+}
+
 func TestHeaderFunctions(t *testing.T) {
+	t.Parallel()
 	f := func(ver, id, typ int) bool {
 		ver = int(uint(ver) % 16)
 		id = int(uint(id) % 4096)
@@ -38,6 +49,7 @@ func TestHeaderFunctions(t *testing.T) {
 }
 
 func TestHeaderLayout(t *testing.T) {
+	t.Parallel()
 	var e, a uint32
 
 	// Version are the first four bits
@@ -63,6 +75,7 @@ func TestHeaderLayout(t *testing.T) {
 }
 
 func TestPing(t *testing.T) {
+	t.Parallel()
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
@@ -82,6 +95,7 @@ func TestPing(t *testing.T) {
 }
 
 func TestVersionErr(t *testing.T) {
+	t.Parallel()
 	m0 := newTestModel()
 	m1 := newTestModel()
 
@@ -109,6 +123,7 @@ func TestVersionErr(t *testing.T) {
 }
 
 func TestTypeErr(t *testing.T) {
+	t.Parallel()
 	m0 := newTestModel()
 	m1 := newTestModel()
 
@@ -136,6 +151,7 @@ func TestTypeErr(t *testing.T) {
 }
 
 func TestClose(t *testing.T) {
+	t.Parallel()
 	m0 := newTestModel()
 	m1 := newTestModel()
 
@@ -171,6 +187,7 @@ func TestClose(t *testing.T) {
 }
 
 func TestElementSizeExceededNested(t *testing.T) {
+	t.Parallel()
 	m := ClusterConfigMessage{
 		ClientName: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon",
 	}
@@ -181,11 +198,7 @@ func TestElementSizeExceededNested(t *testing.T) {
 }
 
 func TestMarshalIndexMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
+	t.Parallel()
 	f := func(m1 IndexMessage) bool {
 		for i, f := range m1.Files {
 			m1.Files[i].CachedSize = 0
@@ -206,11 +219,7 @@ func TestMarshalIndexMessage(t *testing.T) {
 }
 
 func TestMarshalRequestMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
+	t.Parallel()
 	f := func(m1 RequestMessage) bool {
 		return testMarshal(t, "request", &m1, &RequestMessage{})
 	}
@@ -221,11 +230,7 @@ func TestMarshalRequestMessage(t *testing.T) {
 }
 
 func TestMarshalResponseMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
+	t.Parallel()
 	f := func(m1 ResponseMessage) bool {
 		if len(m1.Data) == 0 {
 			m1.Data = nil
@@ -239,11 +244,7 @@ func TestMarshalResponseMessage(t *testing.T) {
 }
 
 func TestMarshalClusterConfigMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
+	t.Parallel()
 	f := func(m1 ClusterConfigMessage) bool {
 		return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{})
 	}
@@ -254,11 +255,7 @@ func TestMarshalClusterConfigMessage(t *testing.T) {
 }
 
 func TestMarshalCloseMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
+	t.Parallel()
 	f := func(m1 CloseMessage) bool {
 		return testMarshal(t, "close", &m1, &CloseMessage{})
 	}

lib/relay/client/client.go

@@ -5,279 +5,37 @@ package client
 import (
 	"crypto/tls"
 	"fmt"
-	"net"
 	"net/url"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/dialer"
-	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
 )
 
-type ProtocolClient struct {
-	URI         *url.URL
-	Invitations chan protocol.SessionInvitation
+type relayClientFactory func(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) RelayClient
 
-	closeInvitationsOnFinish bool
+var (
+	supportedSchemes = map[string]relayClientFactory{
+		"relay":         newStaticClient,
+		"dynamic+http":  newDynamicClient,
+		"dynamic+https": newDynamicClient,
+	}
+)
 
-	config *tls.Config
-
-	timeout time.Duration
-
-	stop    chan struct{}
-	stopped chan struct{}
-
-	conn *tls.Conn
-
-	mut       sync.RWMutex
-	connected bool
-	latency   time.Duration
+type RelayClient interface {
+	Serve()
+	Stop()
+	StatusOK() bool
+	Latency() time.Duration
+	String() string
+	Invitations() chan protocol.SessionInvitation
+	URI() *url.URL
 }
 
-func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient {
-	closeInvitationsOnFinish := false
-	if invitations == nil {
-		closeInvitationsOnFinish = true
-		invitations = make(chan protocol.SessionInvitation)
+func NewClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) (RelayClient, error) {
+	factory, ok := supportedSchemes[uri.Scheme]
+	if !ok {
+		return nil, fmt.Errorf("Unsupported scheme: %s", uri.Scheme)
 	}
 
-	return &ProtocolClient{
-		URI:         uri,
-		Invitations: invitations,
-
-		closeInvitationsOnFinish: closeInvitationsOnFinish,
-
-		config: configForCerts(certs),
-
-		timeout: time.Minute * 2,
-
-		stop:    make(chan struct{}),
-		stopped: make(chan struct{}),
-
-		mut:       sync.NewRWMutex(),
-		connected: false,
-	}
-}
-
-func (c *ProtocolClient) Serve() {
-	c.stop = make(chan struct{})
-	c.stopped = make(chan struct{})
-	defer close(c.stopped)
-
-	if err := c.connect(); err != nil {
-		l.Debugln("Relay connect:", err)
-		return
-	}
-
-	l.Debugln(c, "connected", c.conn.RemoteAddr())
-
-	if err := c.join(); err != nil {
-		c.conn.Close()
-		l.Infoln("Relay join:", err)
-		return
-	}
-
-	if err := c.conn.SetDeadline(time.Time{}); err != nil {
-		c.conn.Close()
-		l.Infoln("Relay set deadline:", err)
-		return
-	}
-
-	l.Debugln(c, "joined", c.conn.RemoteAddr(), "via", c.conn.LocalAddr())
-
-	defer c.cleanup()
-	c.mut.Lock()
-	c.connected = true
-	c.mut.Unlock()
-
-	messages := make(chan interface{})
-	errors := make(chan error, 1)
-
-	go messageReader(c.conn, messages, errors)
-
-	timeout := time.NewTimer(c.timeout)
-
-	for {
-		select {
-		case message := <-messages:
-			timeout.Reset(c.timeout)
-			l.Debugf("%s received message %T", c, message)
-
-			switch msg := message.(type) {
-			case protocol.Ping:
-				if err := protocol.WriteMessage(c.conn, protocol.Pong{}); err != nil {
-					l.Infoln("Relay write:", err)
-					return
-
-				}
-				l.Debugln(c, "sent pong")
-
-			case protocol.SessionInvitation:
-				ip := net.IP(msg.Address)
-				if len(ip) == 0 || ip.IsUnspecified() {
-					msg.Address = c.conn.RemoteAddr().(*net.TCPAddr).IP[:]
-				}
-				c.Invitations <- msg
-
-			default:
-				l.Infoln("Relay: protocol error: unexpected message %v", msg)
-				return
-			}
-
-		case <-c.stop:
-			l.Debugln(c, "stopping")
-			return
-
-		case err := <-errors:
-			l.Infoln("Relay received:", err)
-			return
-
-		case <-timeout.C:
-			l.Debugln(c, "timed out")
-			return
-		}
-	}
-}
-
-func (c *ProtocolClient) Stop() {
-	if c.stop == nil {
-		return
-	}
-
-	close(c.stop)
-	<-c.stopped
-}
-
-func (c *ProtocolClient) StatusOK() bool {
-	c.mut.RLock()
-	con := c.connected
-	c.mut.RUnlock()
-	return con
-}
-
-func (c *ProtocolClient) Latency() time.Duration {
-	c.mut.RLock()
-	lat := c.latency
-	c.mut.RUnlock()
-	return lat
-}
-
-func (c *ProtocolClient) String() string {
-	return fmt.Sprintf("ProtocolClient@%p", c)
-}
-
-func (c *ProtocolClient) connect() error {
-	if c.URI.Scheme != "relay" {
-		return fmt.Errorf("Unsupported relay schema: %v", c.URI.Scheme)
-	}
-
-	t0 := time.Now()
-	tcpConn, err := dialer.Dial("tcp", c.URI.Host)
-	if err != nil {
-		return err
-	}
-
-	c.mut.Lock()
-	c.latency = time.Since(t0)
-	c.mut.Unlock()
-
-	conn := tls.Client(tcpConn, c.config)
-	if err = conn.Handshake(); err != nil {
-		return err
-	}
-
-	if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
-		conn.Close()
-		return err
-	}
-
-	if err := performHandshakeAndValidation(conn, c.URI); err != nil {
-		conn.Close()
-		return err
-	}
-
-	c.conn = conn
-	return nil
-}
-
-func (c *ProtocolClient) cleanup() {
-	if c.closeInvitationsOnFinish {
-		close(c.Invitations)
-		c.Invitations = make(chan protocol.SessionInvitation)
-	}
-
-	l.Debugln(c, "cleaning up")
-
-	c.mut.Lock()
-	c.connected = false
-	c.mut.Unlock()
-
-	c.conn.Close()
-}
-
-func (c *ProtocolClient) join() error {
-	if err := protocol.WriteMessage(c.conn, protocol.JoinRelayRequest{}); err != nil {
-		return err
-	}
-
-	message, err := protocol.ReadMessage(c.conn)
-	if err != nil {
-		return err
-	}
-
-	switch msg := message.(type) {
-	case protocol.Response:
-		if msg.Code != 0 {
-			return fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message)
-		}
-
-	default:
-		return fmt.Errorf("protocol error: expecting response got %v", msg)
-	}
-
-	return nil
-}
-
-func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error {
-	if err := conn.Handshake(); err != nil {
-		return err
-	}
-
-	cs := conn.ConnectionState()
-	if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != protocol.ProtocolName {
-		return fmt.Errorf("protocol negotiation error")
-	}
-
-	q := uri.Query()
-	relayIDs := q.Get("id")
-	if relayIDs != "" {
-		relayID, err := syncthingprotocol.DeviceIDFromString(relayIDs)
-		if err != nil {
-			return fmt.Errorf("relay address contains invalid verification id: %s", err)
-		}
-
-		certs := cs.PeerCertificates
-		if cl := len(certs); cl != 1 {
-			return fmt.Errorf("unexpected certificate count: %d", cl)
-		}
-
-		remoteID := syncthingprotocol.NewDeviceID(certs[0].Raw)
-		if remoteID != relayID {
-			return fmt.Errorf("relay id does not match. Expected %v got %v", relayID, remoteID)
-		}
-	}
-
-	return nil
-}
-
-func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
-	for {
-		msg, err := protocol.ReadMessage(conn)
-		if err != nil {
-			errors <- err
-			return
-		}
-		messages <- msg
-	}
+	return factory(uri, certs, invitations), nil
 }

lib/relay/client/dynamic.go

@@ -0,0 +1,217 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package client
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"sort"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type dynamicClient struct {
+	pooladdr                 *url.URL
+	certs                    []tls.Certificate
+	invitations              chan protocol.SessionInvitation
+	closeInvitationsOnFinish bool
+
+	mut    sync.RWMutex
+	client RelayClient
+	stop   chan struct{}
+}
+
+func newDynamicClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) RelayClient {
+	closeInvitationsOnFinish := false
+	if invitations == nil {
+		closeInvitationsOnFinish = true
+		invitations = make(chan protocol.SessionInvitation)
+	}
+	return &dynamicClient{
+		pooladdr:                 uri,
+		certs:                    certs,
+		invitations:              invitations,
+		closeInvitationsOnFinish: closeInvitationsOnFinish,
+
+		mut: sync.NewRWMutex(),
+	}
+}
+
+func (c *dynamicClient) Serve() {
+	c.mut.Lock()
+	c.stop = make(chan struct{})
+	c.mut.Unlock()
+
+	uri := *c.pooladdr
+
+	// Trim off the `dynamic+` prefix
+	uri.Scheme = uri.Scheme[8:]
+
+	l.Debugln(c, "looking up dynamic relays")
+
+	data, err := http.Get(uri.String())
+	if err != nil {
+		l.Debugln(c, "failed to lookup dynamic relays", err)
+		return
+	}
+
+	var ann dynamicAnnouncement
+	err = json.NewDecoder(data.Body).Decode(&ann)
+	data.Body.Close()
+	if err != nil {
+		l.Debugln(c, "failed to lookup dynamic relays", err)
+		return
+	}
+
+	defer c.cleanup()
+
+	var addrs []string
+	for _, relayAnn := range ann.Relays {
+		ruri, err := url.Parse(relayAnn.URL)
+		if err != nil {
+			l.Debugln(c, "failed to parse dynamic relay address", relayAnn.URL, err)
+			continue
+		}
+		l.Debugln(c, "found", ruri)
+		addrs = append(addrs, ruri.String())
+	}
+
+	for _, addr := range relayAddressesSortedByLatency(addrs) {
+		select {
+		case <-c.stop:
+			l.Debugln(c, "stopping")
+			return
+		default:
+			ruri, err := url.Parse(addr)
+			if err != nil {
+				l.Debugln(c, "skipping relay", addr, err)
+				continue
+			}
+			client, err := NewClient(ruri, c.certs, c.invitations)
+			if err != nil {
+				continue
+			}
+			c.mut.Lock()
+			c.client = client
+			c.mut.Unlock()
+
+			c.client.Serve()
+
+			c.mut.Lock()
+			c.client = nil
+			c.mut.Unlock()
+		}
+	}
+	l.Debugln(c, "could not find a connectable relay")
+}
+
+func (c *dynamicClient) Stop() {
+	c.mut.RLock()
+	defer c.mut.RUnlock()
+	close(c.stop)
+	if c.client == nil {
+		return
+	}
+	c.client.Stop()
+}
+
+func (c *dynamicClient) StatusOK() bool {
+	c.mut.RLock()
+	defer c.mut.RUnlock()
+	if c.client == nil {
+		return false
+	}
+	return c.client.StatusOK()
+}
+
+func (c *dynamicClient) Latency() time.Duration {
+	c.mut.RLock()
+	defer c.mut.RUnlock()
+	if c.client == nil {
+		return time.Hour
+	}
+	return c.client.Latency()
+}
+
+func (c *dynamicClient) String() string {
+	return fmt.Sprintf("DynamicClient:%p:%s@%s", c, c.URI(), c.pooladdr)
+}
+
+func (c *dynamicClient) URI() *url.URL {
+	c.mut.RLock()
+	defer c.mut.RUnlock()
+	if c.client == nil {
+		return c.pooladdr
+	}
+	return c.client.URI()
+}
+
+func (c *dynamicClient) Invitations() chan protocol.SessionInvitation {
+	c.mut.RLock()
+	inv := c.invitations
+	c.mut.RUnlock()
+	return inv
+}
+
+func (c *dynamicClient) cleanup() {
+	c.mut.Lock()
+	if c.closeInvitationsOnFinish {
+		close(c.invitations)
+		c.invitations = make(chan protocol.SessionInvitation)
+	}
+	c.mut.Unlock()
+}
+
+// This is the announcement recieved from the relay server;
+// {"relays": [{"url": "relay://10.20.30.40:5060"}, ...]}
+type dynamicAnnouncement struct {
+	Relays []struct {
+		URL string
+	}
+}
+
+// relayAddressesSortedByLatency adds local latency to the relay, and sorts them
+// by sum latency, and returns the addresses.
+func relayAddressesSortedByLatency(input []string) []string {
+	relays := make(relayList, len(input))
+	for i, relay := range input {
+		if latency, err := osutil.GetLatencyForURL(relay); err == nil {
+			relays[i] = relayWithLatency{relay, int(latency / time.Millisecond)}
+		} else {
+			relays[i] = relayWithLatency{relay, int(time.Hour / time.Millisecond)}
+		}
+	}
+
+	sort.Sort(relays)
+
+	addresses := make([]string, len(relays))
+	for i, relay := range relays {
+		addresses[i] = relay.relay
+	}
+	return addresses
+}
+
+type relayWithLatency struct {
+	relay   string
+	latency int
+}
+
+type relayList []relayWithLatency
+
+func (l relayList) Len() int {
+	return len(l)
+}
+
+func (l relayList) Less(a, b int) bool {
+	return l[a].latency < l[b].latency
+}
+
+func (l relayList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}

lib/relay/client/methods.go

@@ -102,7 +102,11 @@ func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) {
 func TestRelay(uri *url.URL, certs []tls.Certificate, sleep time.Duration, times int) bool {
 	id := syncthingprotocol.NewDeviceID(certs[0].Certificate[0])
 	invs := make(chan protocol.SessionInvitation, 1)
-	c := NewProtocolClient(uri, certs, invs)
+	c, err := NewClient(uri, certs, invs)
+	if err != nil {
+		close(invs)
+		return false
+	}
 	go c.Serve()
 	defer func() {
 		close(invs)

lib/relay/client/static.go

@@ -0,0 +1,291 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package client
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/url"
+	"time"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type staticClient struct {
+	uri         *url.URL
+	invitations chan protocol.SessionInvitation
+
+	closeInvitationsOnFinish bool
+
+	config *tls.Config
+
+	timeout time.Duration
+
+	stop    chan struct{}
+	stopped chan struct{}
+
+	conn *tls.Conn
+
+	mut       sync.RWMutex
+	connected bool
+	latency   time.Duration
+}
+
+func newStaticClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) RelayClient {
+	closeInvitationsOnFinish := false
+	if invitations == nil {
+		closeInvitationsOnFinish = true
+		invitations = make(chan protocol.SessionInvitation)
+	}
+
+	return &staticClient{
+		uri:         uri,
+		invitations: invitations,
+
+		closeInvitationsOnFinish: closeInvitationsOnFinish,
+
+		config: configForCerts(certs),
+
+		timeout: time.Minute * 2,
+
+		stop:    make(chan struct{}),
+		stopped: make(chan struct{}),
+
+		mut:       sync.NewRWMutex(),
+		connected: false,
+	}
+}
+
+func (c *staticClient) Serve() {
+	c.stop = make(chan struct{})
+	c.stopped = make(chan struct{})
+	defer close(c.stopped)
+
+	if err := c.connect(); err != nil {
+		l.Debugln("Relay connect:", err)
+		return
+	}
+
+	l.Debugln(c, "connected", c.conn.RemoteAddr())
+
+	if err := c.join(); err != nil {
+		c.conn.Close()
+		l.Infoln("Relay join:", err)
+		return
+	}
+
+	if err := c.conn.SetDeadline(time.Time{}); err != nil {
+		c.conn.Close()
+		l.Infoln("Relay set deadline:", err)
+		return
+	}
+
+	l.Debugln(c, "joined", c.conn.RemoteAddr(), "via", c.conn.LocalAddr())
+
+	defer c.cleanup()
+	c.mut.Lock()
+	c.connected = true
+	c.mut.Unlock()
+
+	messages := make(chan interface{})
+	errors := make(chan error, 1)
+
+	go messageReader(c.conn, messages, errors)
+
+	timeout := time.NewTimer(c.timeout)
+
+	for {
+		select {
+		case message := <-messages:
+			timeout.Reset(c.timeout)
+			l.Debugf("%s received message %T", c, message)
+
+			switch msg := message.(type) {
+			case protocol.Ping:
+				if err := protocol.WriteMessage(c.conn, protocol.Pong{}); err != nil {
+					l.Infoln("Relay write:", err)
+					return
+
+				}
+				l.Debugln(c, "sent pong")
+
+			case protocol.SessionInvitation:
+				ip := net.IP(msg.Address)
+				if len(ip) == 0 || ip.IsUnspecified() {
+					msg.Address = c.conn.RemoteAddr().(*net.TCPAddr).IP[:]
+				}
+				c.invitations <- msg
+
+			default:
+				l.Infoln("Relay: protocol error: unexpected message %v", msg)
+				return
+			}
+
+		case <-c.stop:
+			l.Debugln(c, "stopping")
+			return
+
+		case err := <-errors:
+			l.Infoln("Relay received:", err)
+			return
+
+		case <-timeout.C:
+			l.Debugln(c, "timed out")
+			return
+		}
+	}
+}
+
+func (c *staticClient) Stop() {
+	if c.stop == nil {
+		return
+	}
+
+	close(c.stop)
+	<-c.stopped
+}
+
+func (c *staticClient) StatusOK() bool {
+	c.mut.RLock()
+	con := c.connected
+	c.mut.RUnlock()
+	return con
+}
+
+func (c *staticClient) Latency() time.Duration {
+	c.mut.RLock()
+	lat := c.latency
+	c.mut.RUnlock()
+	return lat
+}
+
+func (c *staticClient) String() string {
+	return fmt.Sprintf("StaticClient:%p@%s", c, c.URI())
+}
+
+func (c *staticClient) URI() *url.URL {
+	return c.uri
+}
+
+func (c *staticClient) Invitations() chan protocol.SessionInvitation {
+	c.mut.RLock()
+	inv := c.invitations
+	c.mut.RUnlock()
+	return inv
+}
+
+func (c *staticClient) connect() error {
+	if c.uri.Scheme != "relay" {
+		return fmt.Errorf("Unsupported relay schema: %v", c.uri.Scheme)
+	}
+
+	t0 := time.Now()
+	tcpConn, err := net.Dial("tcp", c.uri.Host)
+	if err != nil {
+		return err
+	}
+
+	c.mut.Lock()
+	c.latency = time.Since(t0)
+	c.mut.Unlock()
+
+	conn := tls.Client(tcpConn, c.config)
+	if err = conn.Handshake(); err != nil {
+		return err
+	}
+
+	if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
+		conn.Close()
+		return err
+	}
+
+	if err := performHandshakeAndValidation(conn, c.uri); err != nil {
+		conn.Close()
+		return err
+	}
+
+	c.conn = conn
+	return nil
+}
+
+func (c *staticClient) cleanup() {
+	l.Debugln(c, "cleaning up")
+	c.mut.Lock()
+	if c.closeInvitationsOnFinish {
+		close(c.invitations)
+		c.invitations = make(chan protocol.SessionInvitation)
+	}
+	c.connected = false
+	c.mut.Unlock()
+
+	c.conn.Close()
+}
+
+func (c *staticClient) join() error {
+	if err := protocol.WriteMessage(c.conn, protocol.JoinRelayRequest{}); err != nil {
+		return err
+	}
+
+	message, err := protocol.ReadMessage(c.conn)
+	if err != nil {
+		return err
+	}
+
+	switch msg := message.(type) {
+	case protocol.Response:
+		if msg.Code != 0 {
+			return fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message)
+		}
+
+	default:
+		return fmt.Errorf("protocol error: expecting response got %v", msg)
+	}
+
+	return nil
+}
+
+func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error {
+	if err := conn.Handshake(); err != nil {
+		return err
+	}
+
+	cs := conn.ConnectionState()
+	if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != protocol.ProtocolName {
+		return fmt.Errorf("protocol negotiation error")
+	}
+
+	q := uri.Query()
+	relayIDs := q.Get("id")
+	if relayIDs != "" {
+		relayID, err := syncthingprotocol.DeviceIDFromString(relayIDs)
+		if err != nil {
+			return fmt.Errorf("relay address contains invalid verification id: %s", err)
+		}
+
+		certs := cs.PeerCertificates
+		if cl := len(certs); cl != 1 {
+			return fmt.Errorf("unexpected certificate count: %d", cl)
+		}
+
+		remoteID := syncthingprotocol.NewDeviceID(certs[0].Raw)
+		if remoteID != relayID {
+			return fmt.Errorf("relay id does not match. Expected %v got %v", relayID, remoteID)
+		}
+	}
+
+	return nil
+}
+
+func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
+	for {
+		msg, err := protocol.ReadMessage(conn)
+		if err != nil {
+			errors <- err
+			return
+		}
+		messages <- msg
+	}
+}

lib/relay/relay.go

@@ -8,15 +8,13 @@ package relay
 
 import (
 	"crypto/tls"
-	"encoding/json"
-	"net/http"
+
 	"net/url"
 	"sort"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -34,7 +32,7 @@ type Svc struct {
 	tlsCfg *tls.Config
 
 	tokens      map[string]suture.ServiceToken
-	clients     map[string]*client.ProtocolClient
+	clients     map[string]client.RelayClient
 	mut         sync.RWMutex
 	invitations chan protocol.SessionInvitation
 	conns       chan *tls.Conn
@@ -56,7 +54,7 @@ func NewSvc(cfg *config.Wrapper, tlsCfg *tls.Config) *Svc {
 		tlsCfg: tlsCfg,
 
 		tokens:      make(map[string]suture.ServiceToken),
-		clients:     make(map[string]*client.ProtocolClient),
+		clients:     make(map[string]client.RelayClient),
 		mut:         sync.NewRWMutex(),
 		invitations: make(chan protocol.SessionInvitation),
 		conns:       conns,
@@ -74,7 +72,8 @@ func NewSvc(cfg *config.Wrapper, tlsCfg *tls.Config) *Svc {
 	}
 
 	eventBc := &eventBroadcaster{
-		svc: svc,
+		svc:  svc,
+		stop: make(chan struct{}),
 	}
 
 	svc.Add(receiver)
@@ -105,61 +104,17 @@ func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
 		existing[uri.String()] = uri
 	}
 
-	// Query dynamic addresses, and pick the closest relay from the ones they provide.
-	for key, uri := range existing {
-		if uri.Scheme != "dynamic+http" && uri.Scheme != "dynamic+https" {
-			continue
-		}
-		delete(existing, key)
-
-		// Trim off the `dynamic+` prefix
-		uri.Scheme = uri.Scheme[8:]
-
-		l.Debugln("Looking up dynamic relays from", uri)
-
-		data, err := http.Get(uri.String())
-		if err != nil {
-			l.Debugln("Failed to lookup dynamic relays", err)
-			continue
-		}
-
-		var ann dynamicAnnouncement
-		err = json.NewDecoder(data.Body).Decode(&ann)
-		data.Body.Close()
-		if err != nil {
-			l.Debugln("Failed to lookup dynamic relays", err)
-			continue
-		}
-
-		var dynRelayAddrs []string
-		for _, relayAnn := range ann.Relays {
-			ruri, err := url.Parse(relayAnn.URL)
-			if err != nil {
-				l.Debugln("Failed to parse dynamic relay address", relayAnn.URL, err)
-				continue
-			}
-			l.Debugln("Found", ruri, "via", uri)
-			dynRelayAddrs = append(dynRelayAddrs, ruri.String())
-		}
-
-		if len(dynRelayAddrs) > 0 {
-			dynRelayAddrs = relayAddressesSortedByLatency(dynRelayAddrs)
-			closestRelay := dynRelayAddrs[0]
-			l.Debugln("Picking", closestRelay, "as closest dynamic relay from", uri)
-			ruri, _ := url.Parse(closestRelay)
-			existing[closestRelay] = ruri
-		} else {
-			l.Debugln("No dynamic relay found on", uri)
-		}
-	}
-
 	s.mut.Lock()
 
 	for key, uri := range existing {
 		_, ok := s.tokens[key]
 		if !ok {
 			l.Debugln("Connecting to relay", uri)
-			c := client.NewProtocolClient(uri, s.tlsCfg.Certificates, s.invitations)
+			c, err := client.NewClient(uri, s.tlsCfg.Certificates, s.invitations)
+			if err != nil {
+				l.Debugln("Failed to connect to relay", uri, err)
+				continue
+			}
 			s.tokens[key] = s.Add(c)
 			s.clients[key] = c
 		}
@@ -196,8 +151,8 @@ func (s *Svc) Relays() []string {
 
 	s.mut.RLock()
 	relays := make([]string, 0, len(s.clients))
-	for uri := range s.clients {
-		relays = append(relays, uri)
+	for _, client := range s.clients {
+		relays = append(relays, client.URI().String())
 	}
 	s.mut.RUnlock()
 
@@ -215,14 +170,14 @@ func (s *Svc) RelayStatus(uri string) (time.Duration, bool) {
 	}
 
 	s.mut.RLock()
-	client, ok := s.clients[uri]
+	for _, client := range s.clients {
+		if client.URI().String() == uri {
+			return client.Latency(), client.StatusOK()
+		}
+	}
 	s.mut.RUnlock()
 
-	if !ok || !client.StatusOK() {
-		return time.Hour, false
-	}
-
-	return client.Latency(), true
+	return time.Hour, false
 }
 
 // Accept returns a new *tls.Conn. The connection is already handshaken.
@@ -276,7 +231,7 @@ func (r *invitationReceiver) Stop() {
 // The eventBroadcaster sends a RelayStateChanged event when the relay status
 // changes. We need this somewhat ugly polling mechanism as there's currently
 // no way to get the event feed directly from the relay lib. This may be
-// somethign to revisit later, possibly.
+// something to revisit later, possibly.
 type eventBroadcaster struct {
 	svc  *Svc
 	stop chan struct{}
@@ -321,51 +276,3 @@ func (e *eventBroadcaster) Serve() {
 func (e *eventBroadcaster) Stop() {
 	close(e.stop)
 }
-
-// This is the announcement recieved from the relay server;
-// {"relays": [{"url": "relay://10.20.30.40:5060"}, ...]}
-type dynamicAnnouncement struct {
-	Relays []struct {
-		URL string
-	}
-}
-
-// relayAddressesSortedByLatency adds local latency to the relay, and sorts them
-// by sum latency, and returns the addresses.
-func relayAddressesSortedByLatency(input []string) []string {
-	relays := make(relayList, len(input))
-	for i, relay := range input {
-		if latency, err := osutil.GetLatencyForURL(relay); err == nil {
-			relays[i] = relayWithLatency{relay, int(latency / time.Millisecond)}
-		} else {
-			relays[i] = relayWithLatency{relay, int(time.Hour / time.Millisecond)}
-		}
-	}
-
-	sort.Sort(relays)
-
-	addresses := make([]string, len(relays))
-	for i, relay := range relays {
-		addresses[i] = relay.relay
-	}
-	return addresses
-}
-
-type relayWithLatency struct {
-	relay   string
-	latency int
-}
-
-type relayList []relayWithLatency
-
-func (l relayList) Len() int {
-	return len(l)
-}
-
-func (l relayList) Less(a, b int) bool {
-	return l[a].latency < l[b].latency
-}
-
-func (l relayList) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}

man/refresh.sh

@@ -14,6 +14,7 @@ pages=(
 	syncthing-versioning.7
 	syncthing-bep.7
 	syncthing-localdisco.7
+	syncthing-globaldisco.7
 )
 
 for page in "${pages[@]}" ; do

man/syncthing-bep.7

@@ -88,15 +88,16 @@ fingerprints (SHA\-256) referred to as "Device IDs".
 There is no required order or synchronization among BEP messages except
 as noted per message type \- any message type may be sent at any time and
 the sender need not await a response to one message before sending
-another. Responses MUST however be sent in the same order as the
-requests are received.
+another.
 .sp
 The underlying transport protocol MUST be TCP.
 .SH MESSAGES
 .sp
-Every message starts with one 32 bit word indicating the message
-version, type and ID, followed by the length of the message. The header
-is in network byte order, i.e. big endian.
+Every message starts with one 32 bit word indicating the message version, type
+and ID, followed by the length of the message. The header is in network byte
+order, i.e. big endian. In this document, in diagrams and text, "bit 0" refers
+to the \fImost significant\fP bit of a word; "bit 31" is thus the least
+significant bit of a 32 bit word.
 .INDENT 0.0
 .INDENT 3.5
 .sp
@@ -114,26 +115,33 @@ is in network byte order, i.e. big endian.
 .UNINDENT
 .UNINDENT
 .sp
-For BEP v1 the Version field is set to zero. Future versions with
+For BEP v1 the \fBVersion\fP field is set to zero. Future versions with
 incompatible message formats will increment the Version field. A message
 with an unknown version is a protocol error and MUST result in the
 connection being terminated. A client supporting multiple versions MAY
 retry with a different protocol version upon disconnection.
 .sp
-The Message ID is set to a unique value for each transmitted request
-message. In response messages it is set to the Message ID of the
-corresponding request message. The uniqueness requirement implies that
-no more than 4096 messages may be outstanding at any given moment. The
-ordering requirement implies that a response to a given message ID also
-means that all preceding messages have been received, specifically those
-which do not otherwise demand a response. Hence their message ID:s may
-be reused.
+The \fBMessage ID\fP is set to a unique value for each transmitted Request
+message. In Response messages it is set to the Message ID of the corresponding
+Request message. The uniqueness requirement implies that no more than 4096
+request messages may be outstanding at any given moment. For message types
+that do not have a corresponding response (Cluster Configuration, Index, etc.)
+the Message ID field is irrelevant and SHOULD be set to zero.
 .sp
-The Type field indicates the type of data following the message header
+The \fBType\fP field indicates the type of data following the message header
 and is one of the integers defined below. A message of an unknown type
 is a protocol error and MUST result in the connection being terminated.
 .sp
-The Compression bit "C" indicates the compression used for the message.
+The \fBCompression\fP bit "C" indicates the compression used for the message.
+.sp
+For C=0:
+.INDENT 0.0
+.IP \(bu 2
+The Length field contains the length, in bytes, of the uncompressed
+message data.
+.IP \(bu 2
+The message is not compressed.
+.UNINDENT
 .sp
 For C=1:
 .INDENT 0.0
@@ -148,15 +156,6 @@ The message data is compressed using the LZ4 format and algorithm
 described in \fI\%http://www.lz4.org/\fP\&.
 .UNINDENT
 .sp
-For C=0:
-.INDENT 0.0
-.IP \(bu 2
-The Length field contains the length, in bytes, of the uncompressed
-message data.
-.IP \(bu 2
-The message is not compressed.
-.UNINDENT
-.sp
 All data within the message (post decompression, if compression is in
 use) MUST be in XDR (RFC 1014) encoding. All fields shorter than 32 bits
 and all variable length data MUST be padded to a multiple of 32 bits.
@@ -164,10 +163,10 @@ The actual data types in use by BEP, in XDR naming convention, are the
 following:
 .INDENT 0.0
 .TP
-.B (unsigned) int
+.B (unsigned) int:
 (unsigned) 32 bit integer
 .TP
-.B (unsigned) hyper
+.B (unsigned) hyper:
 (unsigned) 64 bit integer
 .TP
 .B opaque<>
@@ -200,16 +199,22 @@ ClusterConfigMessage Structure:
  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
-|                     Length of ClientName                      |
+|                     Length of Device Name                     |
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 /                                                               /
-\e                 ClientName (variable length)                  \e
+\e                 Device Name (variable length)                 \e
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
-|                    Length of ClientVersion                    |
+|                     Length of Client Name                     |
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 /                                                               /
-\e                ClientVersion (variable length)                \e
+\e                 Client Name (variable length)                 \e
+/                                                               /
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+|                   Length of Client Version                    |
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+/                                                               /
+\e               Client Version (variable length)                \e
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 |                       Number of Folders                       |
@@ -225,7 +230,6 @@ ClusterConfigMessage Structure:
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 
-
 Folder Structure:
 
  0                   1                   2                   3
@@ -252,7 +256,6 @@ Folder Structure:
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 
-
 Device Structure:
 
  0                   1                   2                   3
@@ -264,6 +267,28 @@ Device Structure:
 \e                     ID (variable length)                      \e
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+|                        Length of Name                         |
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+/                                                               /
+\e                    Name (variable length)                     \e
+/                                                               /
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+|                      Number of Addresses                      |
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+|                       Length of Address                       |
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+/                                                               /
+\e                   Address (variable length)                   \e
+/                                                               /
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+|                          Compression                          |
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+|                      Length of Cert Name                      |
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+/                                                               /
+\e                  Cert Name (variable length)                  \e
+/                                                               /
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 |                                                               |
 +                  Max Local Version (64 bits)                  +
 |                                                               |
@@ -277,7 +302,6 @@ Device Structure:
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 
-
 Option Structure:
 
  0                   1                   2                   3
@@ -299,29 +323,110 @@ Option Structure:
 .fi
 .UNINDENT
 .UNINDENT
-.SS Fields
+.SS Fields (ClusterConfigMessage)
 .sp
-The ClientName and ClientVersion fields identify the implementation. The
-values SHOULD be simple strings identifying the implementation name, as
-a user would expect to see it, and the version string in the same
-manner. An example ClientName is "syncthing" and an example
-ClientVersion is "v0.7.2". The ClientVersion field SHOULD follow the
-patterns laid out in the \fI\%Semantic Versioning\fP <\fBhttp://semver.org/\fP>
-standard.
+The \fBDevice Name\fP is a human readable (configured or auto detected) device
+name or host name, for the sending device.
 .sp
-The Folders field lists all folders that will be synchronized over the
-current connection. Each folder has a list of participating Devices,
-Flags and Options. Currently no flags are defined so the field MUST be
-set to all zeroes. The Options field is implementation specific and
-described below.
+The \fBClient Name\fP and \fBClient Version\fP identifies the implementation. The
+values SHOULD  be simple strings identifying the implementation name, as a
+user would expect to see it, and the version string in the same manner. An
+example Client Name is "syncthing" and an example Client Version is "v0.7.2".
+The Client Version field SHOULD follow the patterns laid out in the \fI\%Semantic
+Versioning\fP <\fBhttp://semver.org/\fP> standard.
 .sp
-The Device ID is a 32 byte number that uniquely identifies the device.
-For instance, the reference implementation uses the SHA\-256 of the
+The \fBFolders\fP field contains the list of folders that will be synchronized
+over the current connection.
+.sp
+The \fBOptions\fP field is a list of options that apply to the current
+connection. The options are used in an implementation specific manner. The
+options list is conceptually a map of keys to values, although it is
+transmitted in the form of a list of key and value pairs, both of string type.
+Key ID:s are implementation specific. An implementation MUST ignore unknown
+keys. An implementation MAY impose limits on the length keys and values. The
+options list may be used to inform devices of relevant local configuration
+options such as rate limiting or make recommendations about request
+parallelism, device priorities, etc. An empty options list is valid for
+devices not having any such information to share. Devices MAY NOT make any
+assumptions about peers acting in a specific manner as a result of sent
+options.
+.SS Fields (Folder Structure)
+.sp
+The \fBID\fP field contains the folder ID, as a human readable string.
+.sp
+The \fBDevices\fP field is list of devices participating in sharing this folder.
+.sp
+The \fBFlags\fP field contains flags that affect the behavior of the folder. The
+folder Flags field contains the following single bit flags:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+|                       Reserved                          |D|P|R|
++\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.INDENT 0.0
+.TP
+.B Bit 31 ("R", Read Only)
+is set for folders that the device will accept no updates from the network
+for.
+.TP
+.B Bit 30 ("P", Ignore Permissions)
+is set for folders that the device will not accept or announce file
+permissions for.
+.TP
+.B Bit 29 ("D", Ignore Deletes)
+is set for folders that the device will ignore deletes for.
+.UNINDENT
+.sp
+The \fBOptions\fP field contains a list of options that apply to the folder.
+.SS Fields (Device Structure)
+.sp
+The device \fBID\fP field is a 32 byte number that uniquely identifies the
+device. For instance, the reference implementation uses the SHA\-256 of the
 device X.509 certificate.
 .sp
-Each device has an associated Flags field to indicate the sharing mode
-of that device for the folder in question. See the discussion on Sharing
-Modes. The Device Flags field contains the following single bit flags:
+The \fBName\fP field is a human readable name assigned to the described device
+by the sending device. It MAY be empty and it need not be unique.
+.sp
+The list of \fBAddressess\fP is that used by the sending device to connect to
+the described device.
+.sp
+The \fBCompression\fP field indicates the compression mode in use for this
+device and folder. The following values are valid:
+.INDENT 0.0
+.TP
+.B 0
+Compress metadata. This enables compression of metadata messages such as Index.
+.TP
+.B 1
+Compression disabled. No compression is used on any message.
+.TP
+.B 2
+Compress always. Metadata messages as well as Response messages are compressed.
+.UNINDENT
+.sp
+The \fBCert Name\fP field indicates the expected certificate name for this
+device. It is commonly blank, indicating to use the implementation default.
+.sp
+The \fBMax Local Version\fP field contains the highest local file
+version number of the files already known to be in the index sent by
+this device. If nothing is known about the index of a given device, this
+field MUST be set to zero. When receiving a Cluster Config message with
+a non\-zero Max Local Version for the local device ID, a device MAY elect
+to send an Index Update message containing only files with higher local
+version numbers in place of the initial Index message.
+.sp
+The \fBFlags\fP field indicates the sharing mode of the folder and other device
+& folder specific settings. See the discussion on Sharing Modes. The Device
+Flags field contains the following single bit flags:
 .INDENT 0.0
 .INDENT 3.5
 .sp
@@ -380,25 +485,7 @@ are reserved and MUST be set to zero.
 .sp
 Exactly one of the T and R bits MUST be set.
 .sp
-The per device Max Local Version field contains the highest local file
-version number of the files already known to be in the index sent by
-this device. If nothing is known about the index of a given device, this
-field MUST be set to zero. When receiving a Cluster Config message with
-a non\-zero Max Local Version for the local device ID, a device MAY elect
-to send an Index Update message containing only files with higher local
-version numbers in place of the initial Index message.
-.sp
-The Options field contain option values to be used in an implementation
-specific manner. The options list is conceptually a map of Key => Value
-items, although it is transmitted in the form of a list of (Key, Value)
-pairs, both of string type. Key ID:s are implementation specific. An
-implementation MUST ignore unknown keys. An implementation MAY impose
-limits on the length keys and values. The options list may be used to
-inform devices of relevant local configuration options such as rate
-limiting or make recommendations about request parallelism, device
-priorities, etc. An empty options list is valid for devices not having
-any such information to share. Devices MAY NOT make any assumptions
-about peers acting in a specific manner as a result of sent options.
+The \fBOptions\fP field contains a list of options that apply to the device.
 .SS XDR
 .INDENT 0.0
 .INDENT 3.5
@@ -406,29 +493,34 @@ about peers acting in a specific manner as a result of sent options.
 .nf
 .ft C
 struct ClusterConfigMessage {
-    string ClientName<>;
-    string ClientVersion<>;
-    Folder Folders<>;
-    Option Options<>;
+    string DeviceName<64>;
+    string ClientName<64>;
+    string ClientVersion<64>;
+    Folder Folders<1000000>;
+    Option Options<64>;
 }
 
 struct Folder {
-    string ID<64>;
-    Device Devices<>;
+    string ID<256>;
+    Device Devices<1000000>;
     unsigned int Flags;
     Option Options<64>;
 }
 
 struct Device {
     opaque ID<32>;
+    string Name<64>;
+    string Addresses<64>;
+    unsigned int Compression;
+    string CertName<64>;
     hyper MaxLocalVersion;
     unsigned int Flags;
     Option Options<64>;
 }
 
 struct Option {
-    string Key<>;
-    string Value<>;
+    string Key<64>;
+    string Value<1024>;
 }
 .ft P
 .fi
@@ -475,7 +567,6 @@ IndexMessage Structure:
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 
-
 FileInfo Structure:
 
  0                   1                   2                   3
@@ -494,7 +585,7 @@ FileInfo Structure:
 |                                                               |
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 /                                                               /
-\e                       Vector Structure                        \e
+\e                   Version (variable length)                   \e
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 |                                                               |
@@ -508,7 +599,6 @@ FileInfo Structure:
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 
-
 Vector Structure:
 
  0                   1                   2                   3
@@ -521,7 +611,6 @@ Vector Structure:
 /                                                               /
 +\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+\-+
 
-
 Counter Structure:
 
  0                   1                   2                   3
@@ -554,32 +643,27 @@ BlockInfo Structure:
 .fi
 .UNINDENT
 .UNINDENT
-.SS Fields
+.SS Fields (Index Message)
 .sp
-The Folder field identifies the folder that the index message pertains
-to. For single folder implementations the device MUST use the string
-"default".
+The \fBFolder\fP field identifies the folder that the index message pertains to.
 .sp
-The Name is the file name path relative to the folder root. Like all
+\fBFiles\fP
+.sp
+The \fBFlags\fP field is reserved for future use and MUST currently be set to
+zero.
+.sp
+The \fBOptions\fP list is implementation defined and as described in the
+ClusterConfig message section.
+.SS Fields (FileInfo Structure)
+.sp
+The \fBName\fP is the file name path relative to the folder root. Like all
 strings in BEP, the Name is always in UTF\-8 NFC regardless of operating
 system or file system specific conventions. The Name field uses the
 slash character ("/") as path separator, regardless of the
 implementation\(aqs operating system conventions. The combination of Folder
 and Name uniquely identifies each file in a cluster.
 .sp
-The Version field is a version vector describing the updates performed
-to file by all members in the cluster. Each counter in the version
-vector is an ID\-Value tuple. The ID is used the first 64 bits of the
-device ID. The Value is a simple incrementing counter, starting at zero.
-The combination of Folder, Name and Version uniquely identifies the
-contents of a file at a given point in time.
-.sp
-The Local Version field is the value of a device local monotonic clock
-at the time of last local database update to a file. The clock ticks on
-every local database update.
-.sp
-The Flags field (per FileInfo) is made up of the following single bit
-flags:
+The \fBFlags\fP field is made up of the following single bit flags:
 .INDENT 0.0
 .INDENT 3.5
 .sp
@@ -614,10 +698,10 @@ temporarily not serve data for the file.
 .TP
 .B Bit 17 ("P")
 is set when there is no permission information for the
-file. This is the case when it originates on a non\-permission\-
-supporting file system. Changes to only permission bits SHOULD be
-disregarded on files with this bit set. The permissions bits MUST be
-set to the octal value 0666.
+file. This is the case when it originates on a file system which
+does not support permissions. Changes to only permission bits SHOULD
+be disregarded on files with this bit set. The permissions bits MUST
+be set to the octal value 0666.
 .TP
 .B Bit 16 ("S")
 is set when the file is a symbolic link. The block list
@@ -635,25 +719,26 @@ are reserved for future use and SHALL be set to
 zero.
 .UNINDENT
 .sp
-The hash algorithm is implied by the Hash length. Currently, the hash
-MUST be 32 bytes long and computed by SHA256.
-.sp
-The Modified time is expressed as the number of seconds since the Unix
+The \fBModified\fP time is expressed as the number of seconds since the Unix
 Epoch (1970\-01\-01 00:00:00 UTC).
 .sp
-In the rare occasion that a file is simultaneously and independently
-modified by two devices in the same cluster and thus end up on the same
-Version number after modification, the Modified field is used as a tie
-breaker (higher being better), followed by the hash values of the file
-blocks (lower being better).
+The \fBVersion\fP field is a version vector describing the updates performed
+to a file by all members in the cluster. Each counter in the version
+vector is an ID\-Value tuple. The ID is used the first 64 bits of the
+device ID. The Value is a simple incrementing counter, starting at zero.
+The combination of Folder, Name and Version uniquely identifies the
+contents of a file at a given point in time.
 .sp
-The Blocks list contains the size and hash for each block in the file.
+The \fBLocal Version\fP field is the value of a device local monotonic clock
+at the time of last local database update to a file. The clock ticks on
+every local database update.
+.sp
+The \fBBlocks\fP list contains the size and hash for each block in the file.
 Each block represents a 128 KiB slice of the file, except for the last
 block which may represent a smaller amount of data.
 .sp
-The Flags field (in IndexMessage) is reserved for future use and MUST
-currently be set to zero. The Options list is implementation defined and
-as described in the ClusterConfig message section.
+The hash algorithm is implied by the \fBHash\fP length. Currently, the hash
+MUST be 32 bytes long and computed by SHA256.
 .SS XDR
 .INDENT 0.0
 .INDENT 3.5
@@ -661,8 +746,8 @@ as described in the ClusterConfig message section.
 .nf
 .ft C
 struct IndexMessage {
-    string Folder<>;
-    FileInfo Files<>;
+    string Folder<256>;
+    FileInfo Files<1000000>;
     unsigned int Flags;
     Option Options<64>;
 }
@@ -673,7 +758,7 @@ struct FileInfo {
     hyper Modified;
     Vector Version;
     hyper LocalVersion;
-    BlockInfo Blocks<>;
+    BlockInfo Blocks<1000000>;
 }
 
 struct Vector {
@@ -687,7 +772,7 @@ struct Counter {
 
 struct BlockInfo {
     unsigned int Size;
-    opaque Hash<>;
+    opaque Hash<64>;
 }
 .ft P
 .fi
@@ -806,11 +891,11 @@ ResponseMessage Structure:
 .UNINDENT
 .SS Fields
 .sp
-The Data field contains either a full 128 KiB block, a shorter block in
+The \fBData\fP field contains either a full 128 KiB block, a shorter block in
 the case of the last block in a file, or is empty (zero length) if the
 requested block is not available.
 .sp
-The Code field contains an error code describing the reason a Request
+The \fBCode\fP field contains an error code describing the reason a Request
 could not be fulfilled, in the case where a zero length Data was
 returned. The following values are defined:
 .INDENT 0.0
@@ -845,13 +930,11 @@ struct ResponseMessage {
 .UNINDENT
 .SS Ping (Type = 4)
 .sp
-The Ping message is used to determine that a connection is alive, and to
-keep connections alive through state tracking network elements such as
-firewalls and NAT gateways. The Ping message has no contents.
-.SS Pong (Type = 5)
-.sp
-The Pong message is sent in response to a Ping. The Pong message has no
-contents, but copies the Message ID from the Ping.
+The Ping message is used to determine that a connection is alive, and to keep
+connections alive through state tracking network elements such as firewalls
+and NAT gateways. The Ping message has no contents. A Ping message is sent
+every 90 seconds, if no other message has been sent in the preceding 90
+seconds.
 .SS Close (Type = 7)
 .sp
 The Close message MAY be sent to indicate that the connection will be
@@ -882,8 +965,8 @@ CloseMessage Structure:
 .UNINDENT
 .SS Fields
 .sp
-The Reason field contains a human description of the error condition,
-suitable for consumption by a human. The Code field is for a machine
+The \fBReason\fP field contains a human description of the error condition,
+suitable for consumption by a human. The \fBCode\fP field is for a machine
 readable error code. Codes are reserved for future use and MUST
 currently be set to zero.
 .INDENT 0.0
@@ -1260,17 +1343,16 @@ T}
 _
 .TE
 .sp
-The connection is established and at 1. both peers send
-ClusterConfiguration messages and then Index records. The Index records
-are received and both peers recompute their knowledge of the data in the
-cluster. In this example, peer A has four missing or outdated blocks. At
-2 through 5 peer A sends requests for these blocks. The requests are
-received by peer B, who retrieves the data from the folder and transmits
-Response records (6 through 9). Device A updates their folder contents
-and transmits an Index Update message (10). Both peers enter idle state
-after 10. At some later time 11, peer A determines that it has not seen
-data from B for some time and sends a Ping request. A response is sent
-at 12.
+The connection is established and at 1. both peers send ClusterConfiguration
+messages and then Index records. The Index records are received and both peers
+recompute their knowledge of the data in the cluster. In this example, peer A
+has four missing or outdated blocks. At 5 through 8 peer A sends requests for
+these blocks. The requests are received by peer B, who retrieves the data from
+the folder and transmits Response records (9 through 12). Device A updates
+their folder contents and transmits an Index Update message (13). Both peers
+enter idle state after 13. At some later time 14, the ping timer on device B
+expires and a Ping message is sent. The same process occurs for device A at
+15.
 .SH EXAMPLES OF STRONG CIPHER SUITES
 .TS
 center;

man/syncthing-globaldisco.7

@@ -0,0 +1,112 @@
+.\" Man page generated from reStructuredText.
+.
+.TH "SYNCTHING-GLOBALDISCO" "7" "October 20, 2015" "v0.11" "Syncthing"
+.SH NAME
+syncthing-globaldisco \- Global Discovery Protocol v3
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH ANNOUNCEMENTS
+.sp
+A device should announce itself at startup. It does this by an HTTPS POST to
+the announce server URL (with the path usually being "/", but this is of
+course up to the discovery server). The POST has a JSON payload listing direct
+connection addresses (if any) and relay addresses (if any):
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+{
+        direct: ["tcp://192.0.2.45:22000", "tcp://:22202"],
+        relays: [{"url": "relay://192.0.2.99:22028", "latency": 142}]
+}
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+It\(aqs OK for either of the "direct" or "relays" fields to be either the empty
+list (\fB[]\fP), \fBnull\fP, or missing entirely. An announcment with both fields missing
+or empty is however not useful...
+.sp
+Any empty or unspecified IP addresses (i.e. addresses like \fBtcp://:22000\fP,
+\fBtcp://0.0.0.0:22000\fP, \fBtcp://[::]:22000\fP) are interpreted as referring to
+the source IP address of the announcement.
+.sp
+The device ID of the announcing device is not part of the announcement.
+Instead, the server requires that the client perform certificate
+authentication. The device ID is deduced from the presented certificate.
+.sp
+The server response is empty, with code \fB204\fP (No Content) on success. If no
+certificate was presented, status \fB403\fP (Forbidden) is returned. If the
+posted data doesn\(aqt conform to the expected format, \fB400\fP (Bad Request) is
+returned.
+.sp
+In successfull responses, the server may return a 
+.nf
+\(ga\(ga
+.fi
+Reannounce\-After"
+.nf
+\(ga\(ga
+.fi
+header
+containing the number of seconds after which the client should perform a new
+announcement.
+.sp
+In error responses, the server may return a \fBRetry\-After\fP header containing
+the number of seconds after which the client should retry.
+.sp
+Performing announcements significantly more often than indicated by the
+\fBReannounce\-After\fP or \fBRetry\-After\fP headers may result in the client being
+throttled. In such cases the server may respond with status code \fB429\fP (Too
+Many Requests).
+.SH QUERIES
+.sp
+Queries are performed as HTTPS GET requests to the announce server URL. The
+requested device ID is passed as the query parameter "device", in canonical
+string form, i.e. \fBhttps://announce.syncthing.net/?device=ABC12345\-....\fP
+.sp
+Successfull responses will have status code \fB200\fP (OK) and carry a JSON payload
+of the same format as the announcement above. The response will not contain
+empty or unspecified addresses.
+.sp
+If the "device" query parameter is missing or malformed, the status code 400
+(Bad Request) is returned.
+.sp
+If the device ID is of a valid format but not found in the registry, 404 (Not
+Found) is returned.
+.sp
+If the client has exceeded a rate limit, the server may respond with 429 (Too
+Many Requests).
+.SH AUTHOR
+The Syncthing Authors
+.SH COPYRIGHT
+2015, The Syncthing Authors
+.\" Generated by docutils manpage writer.
+.

test/h1/config.xml

@@ -1,5 +1,5 @@
 <configuration version="12">
-    <folder id="default" path="s1" ro="false" rescanIntervalS="10" ignorePerms="false" autoNormalize="true">
+    <folder id="default" path="s1/" ro="false" rescanIntervalS="10" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
@@ -11,9 +11,12 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
-    <folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" path="s12-1" ro="false" rescanIntervalS="10" ignorePerms="false" autoNormalize="true">
+    <folder id="s12" path="s12-1/" ro="false" rescanIntervalS="10" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <minDiskFreePct>1</minDiskFreePct>
@@ -23,6 +26,9 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
     <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4" compression="metadata" introducer="false">
@@ -51,9 +57,9 @@
         <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>true</localAnnounceEnabled>
-        <localAnnouncePort>21025</localAnnouncePort>
+        <localAnnouncePort>21027</localAnnouncePort>
         <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
-        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
+        <relayServer>dynamic+https://relays.syncthing.net/endpoint</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
@@ -67,6 +73,9 @@
         <upnpTimeoutSeconds>10</upnpTimeoutSeconds>
         <urAccepted>-1</urAccepted>
         <urUniqueID></urUniqueID>
+        <urURL>https://data.syncthing.net/newdata</urURL>
+        <urPostInsecurely>false</urPostInsecurely>
+        <urInitialDelayS>1800</urInitialDelayS>
         <restartOnWakeup>true</restartOnWakeup>
         <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
         <keepTemporariesH>24</keepTemporariesH>
@@ -75,8 +84,7 @@
         <symlinksEnabled>true</symlinksEnabled>
         <limitBandwidthInLan>false</limitBandwidthInLan>
         <databaseBlockCacheMiB>0</databaseBlockCacheMiB>
-        <pingTimeoutS>30</pingTimeoutS>
-        <pingIdleTimeS>60</pingIdleTimeS>
         <minHomeDiskFreePct>1</minHomeDiskFreePct>
+        <releasesURL>https://api.github.com/repos/syncthing/syncthing/releases?per_page=30</releasesURL>
     </options>
 </configuration>

test/h2/config.xml

@@ -12,9 +12,12 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
-    <folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" path="s12-2/" ro="false" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
+    <folder id="s12" path="s12-2/" ro="false" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <minDiskFreePct>1</minDiskFreePct>
@@ -24,6 +27,9 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
     <folder id="s23" path="s23-2/" ro="false" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
@@ -36,6 +42,9 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
@@ -54,11 +63,11 @@
     <options>
         <listenAddress>tcp://127.0.0.1:22002</listenAddress>
         <globalAnnounceServer>default</globalAnnounceServer>
-        <globalAnnounceEnabled>true</globalAnnounceEnabled>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>true</localAnnounceEnabled>
-        <localAnnouncePort>21025</localAnnouncePort>
+        <localAnnouncePort>21027</localAnnouncePort>
         <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
-        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
+        <relayServer>dynamic+https://relays.syncthing.net/endpoint</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
@@ -70,11 +79,11 @@
         <upnpLeaseMinutes>0</upnpLeaseMinutes>
         <upnpRenewalMinutes>1</upnpRenewalMinutes>
         <upnpTimeoutSeconds>10</upnpTimeoutSeconds>
-        <urAccepted>2</urAccepted>
-        <urUniqueID>621mlbJP</urUniqueID>
-        <urURL>https://localhost:8443/newdata</urURL>
-        <urPostInsecurely>true</urPostInsecurely>
-        <urInitialDelayS>10</urInitialDelayS>
+        <urAccepted>-1</urAccepted>
+        <urUniqueID></urUniqueID>
+        <urURL>https://data.syncthing.net/newdata</urURL>
+        <urPostInsecurely>false</urPostInsecurely>
+        <urInitialDelayS>1800</urInitialDelayS>
         <restartOnWakeup>true</restartOnWakeup>
         <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
         <keepTemporariesH>24</keepTemporariesH>

test/h3/config.xml

@@ -1,5 +1,5 @@
 <configuration version="12">
-    <folder id="s23" path="s23-3" ro="false" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
+    <folder id="s23" path="s23-3/" ro="false" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <minDiskFreePct>1</minDiskFreePct>
@@ -9,9 +9,12 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
-    <folder id="default" path="s3" ro="false" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
+    <folder id="default" path="s3/" ro="false" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
@@ -24,6 +27,9 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
@@ -44,9 +50,9 @@
         <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
-        <localAnnouncePort>21025</localAnnouncePort>
+        <localAnnouncePort>21027</localAnnouncePort>
         <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
-        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
+        <relayServer>dynamic+https://relays.syncthing.net/endpoint</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
@@ -60,6 +66,9 @@
         <upnpTimeoutSeconds>10</upnpTimeoutSeconds>
         <urAccepted>-1</urAccepted>
         <urUniqueID></urUniqueID>
+        <urURL>https://data.syncthing.net/newdata</urURL>
+        <urPostInsecurely>false</urPostInsecurely>
+        <urInitialDelayS>1800</urInitialDelayS>
         <restartOnWakeup>true</restartOnWakeup>
         <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
         <keepTemporariesH>24</keepTemporariesH>
@@ -68,8 +77,7 @@
         <symlinksEnabled>true</symlinksEnabled>
         <limitBandwidthInLan>false</limitBandwidthInLan>
         <databaseBlockCacheMiB>0</databaseBlockCacheMiB>
-        <pingTimeoutS>30</pingTimeoutS>
-        <pingIdleTimeS>60</pingIdleTimeS>
         <minHomeDiskFreePct>1</minHomeDiskFreePct>
+        <releasesURL>https://api.github.com/repos/syncthing/syncthing/releases?per_page=30</releasesURL>
     </options>
 </configuration>

test/h4/config.xml

@@ -1,5 +1,5 @@
 <configuration version="12">
-    <folder id="default" path="s4" ro="false" rescanIntervalS="60" ignorePerms="false" autoNormalize="false">
+    <folder id="default" path="s4/" ro="false" rescanIntervalS="60" ignorePerms="false" autoNormalize="false">
         <device id="7PBCTLL-JJRYBSA-MOWZRKL-MSDMN4N-4US4OMX-SYEXUS4-HSBGNRY-CZXRXAT"></device>
         <minDiskFreePct>1</minDiskFreePct>
         <versioning></versioning>
@@ -8,6 +8,9 @@
         <hashers>0</hashers>
         <order>random</order>
         <ignoreDelete>false</ignoreDelete>
+        <scanProgressIntervalS>0</scanProgressIntervalS>
+        <pullerSleepS>0</pullerSleepS>
+        <pullerPauseS>0</pullerPauseS>
         <maxConflicts>-1</maxConflicts>
     </folder>
     <device id="7PBCTLL-JJRYBSA-MOWZRKL-MSDMN4N-4US4OMX-SYEXUS4-HSBGNRY-CZXRXAT" name="s4" compression="metadata" introducer="false">
@@ -22,9 +25,9 @@
         <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
-        <localAnnouncePort>21025</localAnnouncePort>
+        <localAnnouncePort>21027</localAnnouncePort>
         <localAnnounceMCAddr>[ff12::8384]:21027</localAnnounceMCAddr>
-        <relayServer>dynamic+https://relays.syncthing.net</relayServer>
+        <relayServer>dynamic+https://relays.syncthing.net/endpoint</relayServer>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>60</reconnectionIntervalS>
@@ -38,6 +41,9 @@
         <upnpTimeoutSeconds>10</upnpTimeoutSeconds>
         <urAccepted>-1</urAccepted>
         <urUniqueID></urUniqueID>
+        <urURL>https://data.syncthing.net/newdata</urURL>
+        <urPostInsecurely>false</urPostInsecurely>
+        <urInitialDelayS>1800</urInitialDelayS>
         <restartOnWakeup>true</restartOnWakeup>
         <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
         <keepTemporariesH>24</keepTemporariesH>
@@ -46,8 +52,7 @@
         <symlinksEnabled>true</symlinksEnabled>
         <limitBandwidthInLan>false</limitBandwidthInLan>
         <databaseBlockCacheMiB>0</databaseBlockCacheMiB>
-        <pingTimeoutS>30</pingTimeoutS>
-        <pingIdleTimeS>60</pingIdleTimeS>
         <minHomeDiskFreePct>1</minHomeDiskFreePct>
+        <releasesURL>https://api.github.com/repos/syncthing/syncthing/releases?per_page=30</releasesURL>
     </options>
 </configuration>