Log/error fields changed name

Make puller pause configurable

.gitignore

@@ -1,4 +1,6 @@
-./syncthing
+syncthing
+!gui/syncthing
+!Godeps/_workspace/src/github.com/syncthing
 syncthing.exe
 *.tar.gz
 *.zip
@@ -9,8 +11,6 @@ files/pidx
 bin
 perfstats*.csv
 coverage.xml
-!gui/scripts/syncthing
-syncthing.md5
-syncthing.exe.md5
+syncthing.sig
 RELEASE
 deb

AUTHORS

@@ -48,6 +48,7 @@ Lord Landon Agahnim <lordlandon@gmail.com>
 Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol <kilburn@la3.org>
 Marcin Dziadus <dziadus.marcin@gmail.com>
+Mateon1 <matin1111@wp.pl>
 Matt Burke <mburke@amplify.com>
 Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Tilli <pyfisch@gmail.com>

Godeps/Godeps.json

@@ -1,22 +1,18 @@
 {
 	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.5",
+	"GoVersion": "go1.5.1",
 	"Packages": [
 		"./cmd/..."
 	],
 	"Deps": [
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "d47913b1412890a261b9fefae99d72d2bf5aebd8"
+			"Rev": "74ddf82598bc4745b965729e9c6a463bedd33049"
 		},
 		{
 			"ImportPath": "github.com/calmh/du",
 			"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
 		},
-		{
-			"ImportPath": "github.com/calmh/logger",
-			"Rev": "c96f6a1a8c7b6bf2f4860c667867d90174799eb2"
-		},
 		{
 			"ImportPath": "github.com/calmh/luhn",
 			"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
@@ -37,13 +33,9 @@
 			"ImportPath": "github.com/kardianos/osext",
 			"Rev": "6e7f843663477789fac7c02def0d0909e969b4e5"
 		},
-		{
-			"ImportPath": "github.com/syncthing/protocol",
-			"Rev": "388a29bbe21d8772ee4c29f4520aa8040309607d"
-		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "b743d92d3215f11c9b5ce8830fafe1f16786adf4"
+			"Rev": "1a9d62f03ea92815b46fcaab357cfd4df264b1a0"
 		},
 		{
 			"ImportPath": "github.com/thejerf/suture",
@@ -64,19 +56,19 @@
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
+			"Rev": "81bf7719a6b7ce9b665598222362b50122dfc13b"
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
+			"Rev": "81bf7719a6b7ce9b665598222362b50122dfc13b"
 		},
 		{
 			"ImportPath": "golang.org/x/net/internal/iana",
-			"Rev": "66f0418ca41253f8d1a024eb9754e9441a8e79b9"
+			"Rev": "db8e4de5b2d6653f66aea53094624468caad15d2"
 		},
 		{
 			"ImportPath": "golang.org/x/net/ipv6",
-			"Rev": "66f0418ca41253f8d1a024eb9754e9441a8e79b9"
+			"Rev": "db8e4de5b2d6653f66aea53094624468caad15d2"
 		},
 		{
 			"ImportPath": "golang.org/x/text/transform",

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -4,5 +4,6 @@ go:
   - 1.1
   - 1.2
   - 1.3
-  - 1.4
+  - 1.4
+  - 1.5
   - tip

Godeps/_workspace/src/github.com/calmh/logger/README.md

@@ -1,15 +0,0 @@
-logger
-======
-
-A small wrapper around `log` to provide log levels.
-
-Documentation
--------------
-
-http://godoc.org/github.com/calmh/logger
-
-License
--------
-
-MIT
-

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -1,187 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// Package logger implements a standardized logger with callback functionality
-package logger
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"strings"
-	"sync"
-)
-
-type LogLevel int
-
-const (
-	LevelDebug LogLevel = iota
-	LevelVerbose
-	LevelInfo
-	LevelOK
-	LevelWarn
-	LevelFatal
-	NumLevels
-)
-
-// A MessageHandler is called with the log level and message text.
-type MessageHandler func(l LogLevel, msg string)
-
-type Logger struct {
-	logger   *log.Logger
-	handlers [NumLevels][]MessageHandler
-	mut      sync.Mutex
-}
-
-// The default logger logs to standard output with a time prefix.
-var DefaultLogger = New()
-
-func New() *Logger {
-	if os.Getenv("LOGGER_DISCARD") != "" {
-		// Hack to completely disable logging, for example when running benchmarks.
-		return &Logger{
-			logger: log.New(ioutil.Discard, "", 0),
-		}
-	}
-
-	return &Logger{
-		logger: log.New(os.Stdout, "", log.Ltime),
-	}
-}
-
-// AddHandler registers a new MessageHandler to receive messages with the
-// specified log level or above.
-func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	l.handlers[level] = append(l.handlers[level], h)
-}
-
-// See log.SetFlags
-func (l *Logger) SetFlags(flag int) {
-	l.logger.SetFlags(flag)
-}
-
-// See log.SetPrefix
-func (l *Logger) SetPrefix(prefix string) {
-	l.logger.SetPrefix(prefix)
-}
-
-func (l *Logger) callHandlers(level LogLevel, s string) {
-	for _, h := range l.handlers[level] {
-		h(level, strings.TrimSpace(s))
-	}
-}
-
-// Debugln logs a line with a DEBUG prefix.
-func (l *Logger) Debugln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "DEBUG: "+s)
-	l.callHandlers(LevelDebug, s)
-}
-
-// Debugf logs a formatted line with a DEBUG prefix.
-func (l *Logger) Debugf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "DEBUG: "+s)
-	l.callHandlers(LevelDebug, s)
-}
-
-// Infoln logs a line with a VERBOSE prefix.
-func (l *Logger) Verboseln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "VERBOSE: "+s)
-	l.callHandlers(LevelVerbose, s)
-}
-
-// Infof logs a formatted line with a VERBOSE prefix.
-func (l *Logger) Verbosef(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "VERBOSE: "+s)
-	l.callHandlers(LevelVerbose, s)
-}
-
-// Infoln logs a line with an INFO prefix.
-func (l *Logger) Infoln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "INFO: "+s)
-	l.callHandlers(LevelInfo, s)
-}
-
-// Infof logs a formatted line with an INFO prefix.
-func (l *Logger) Infof(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "INFO: "+s)
-	l.callHandlers(LevelInfo, s)
-}
-
-// Okln logs a line with an OK prefix.
-func (l *Logger) Okln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "OK: "+s)
-	l.callHandlers(LevelOK, s)
-}
-
-// Okf logs a formatted line with an OK prefix.
-func (l *Logger) Okf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "OK: "+s)
-	l.callHandlers(LevelOK, s)
-}
-
-// Warnln logs a formatted line with a WARNING prefix.
-func (l *Logger) Warnln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "WARNING: "+s)
-	l.callHandlers(LevelWarn, s)
-}
-
-// Warnf logs a formatted line with a WARNING prefix.
-func (l *Logger) Warnf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "WARNING: "+s)
-	l.callHandlers(LevelWarn, s)
-}
-
-// Fatalln logs a line with a FATAL prefix and exits the process with exit
-// code 1.
-func (l *Logger) Fatalln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "FATAL: "+s)
-	l.callHandlers(LevelFatal, s)
-	os.Exit(1)
-}
-
-// Fatalf logs a formatted line with a FATAL prefix and exits the process with
-// exit code 1.
-func (l *Logger) Fatalf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "FATAL: "+s)
-	l.callHandlers(LevelFatal, s)
-	os.Exit(1)
-}

Godeps/_workspace/src/github.com/calmh/logger/logger_test.go

@@ -1,58 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package logger
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestAPI(t *testing.T) {
-	l := New()
-	l.SetFlags(0)
-	l.SetPrefix("testing")
-
-	debug := 0
-	l.AddHandler(LevelDebug, checkFunc(t, LevelDebug, "test 0", &debug))
-	info := 0
-	l.AddHandler(LevelInfo, checkFunc(t, LevelInfo, "test 1", &info))
-	warn := 0
-	l.AddHandler(LevelWarn, checkFunc(t, LevelWarn, "test 2", &warn))
-	ok := 0
-	l.AddHandler(LevelOK, checkFunc(t, LevelOK, "test 3", &ok))
-
-	l.Debugf("test %d", 0)
-	l.Debugln("test", 0)
-	l.Infof("test %d", 1)
-	l.Infoln("test", 1)
-	l.Warnf("test %d", 2)
-	l.Warnln("test", 2)
-	l.Okf("test %d", 3)
-	l.Okln("test", 3)
-
-	if debug != 2 {
-		t.Errorf("Debug handler called %d != 2 times", debug)
-	}
-	if info != 2 {
-		t.Errorf("Info handler called %d != 2 times", info)
-	}
-	if warn != 2 {
-		t.Errorf("Warn handler called %d != 2 times", warn)
-	}
-	if ok != 2 {
-		t.Errorf("Ok handler called %d != 2 times", ok)
-	}
-}
-
-func checkFunc(t *testing.T, expectl LogLevel, expectmsg string, counter *int) func(LogLevel, string) {
-	return func(l LogLevel, msg string) {
-		*counter++
-		if l != expectl {
-			t.Errorf("Incorrect message level %d != %d", l, expectl)
-		}
-		if !strings.HasSuffix(msg, expectmsg) {
-			t.Errorf("%q does not end with %q", msg, expectmsg)
-		}
-	}
-}

Godeps/_workspace/src/github.com/syncthing/protocol/debug.go

@@ -1,15 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-package protocol
-
-import (
-	"os"
-	"strings"
-
-	"github.com/calmh/logger"
-)
-
-var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
-)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go

@@ -201,6 +201,7 @@ func (p *BufferPool) String() string {
 
 func (p *BufferPool) drain() {
 	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
 	for {
 		select {
 		case <-ticker.C:

Godeps/_workspace/src/github.com/thejerf/suture/README.md

@@ -6,10 +6,8 @@ Suture
 Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" ->
 "sutree" -> "suture" -> holds your code together when it's trying to die.
 
-This is intended to be a production-quality library going into code that I
-will be very early on the phone tree to support when it goes down. However,
-it has not been deployed into something quite that serious yet. (I will
-update this statement when that changes.)
+This library has hit maturity, and isn't expected to be changed
+radically. This can also be imported via gopkg.in/thejerf/suture.v1 .
 
 It is intended to deal gracefully with the real failure cases that can
 occur with supervision trees (such as burning all your CPU time endlessly
@@ -24,10 +22,6 @@ This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/su
 including an example, usage, and everything else you might expect from a
 README.md on GitHub. (DRY.)
 
-This is not currently tagged with particular git tags for Go as this is
-currently considered to be alpha code. As I move this into production and
-feel more confident about it, I'll give it relevant tags.
-
 Code Signing
 ------------
 

NICKS

@@ -42,6 +42,7 @@ kozec		<kozec@kozec.com>
 krozycki	<rozycki.karol@gmail.com>
 marcindziadus	<dziadus.marcin@gmail.com>
 marclaporte <marc@marclaporte.com>
+mateon1	<matin1111@wp.pl>
 mogwa1	<devriesb@gmail.com>
 moshen		<moshen.colin@gmail.com>
 mvdan		<mvdan@mvdan.cc>

README.md

@@ -1,17 +1,16 @@
-Syncthing
-=========
+# Syncthing
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
-[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
-[![MPLv2 License](http://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
+[![Latest Build (Official)](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
+[![AppVeyor Build](https://img.shields.io/appveyor/ci/calmh/syncthing/master.svg?style=flat-square)](https://ci.appveyor.com/project/calmh/syncthing)
+[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
+[![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 
 This is the Syncthing project which pursues the following goals:
 
  1. Define a protocol for synchronization of a folder between a number of
     collaborating devices. This protocol should be well defined, unambiguous,
     easily understood, free to use, efficient, secure and language neutral.
-    This is called the [Block Exchange
-    Protocol](https://github.com/syncthing/specs/blob/master/BEPv1.md).
+    This is called the [Block Exchange Protocol][1].
 
  2. Provide the reference implementation to demonstrate the usability of
     said protocol. This is the `syncthing` utility. We hope that
@@ -21,38 +20,38 @@ The two are evolving together; the protocol is not to be considered
 stable until Syncthing 1.0 is released, at which point it is locked down
 for incompatible changes.
 
-Getting Started
----------------
+## Getting Started
 
-Take a look at the [getting started
-guide](http://docs.syncthing.net/intro/getting-started.html).
+Take a look at the [getting started guide][2].
 
 There are a few examples for keeping Syncthing running in the background
-on your system in [the etc directory](https://github.com/syncthing/syncthing/blob/master/etc).
+on your system in [the etc directory][3].
 
-There is an IRC channel, `#syncthing` on Freenode, for talking directly
+There is an IRC channel, `#syncthing` on [Freenode][4], for talking directly
 to developers and users.
 
-Building
---------
+## Building
 
-Building Syncthing from source is easy, and there's a
-[guide](http://docs.syncthing.net/dev/building.html).
+Building Syncthing from source is easy, and there's a [guide][5].
 that describes it for both Unix and Windows systems.
 
-Signed Releases
----------------
+## Signed Releases
 
 As of v0.10.15 and onwards, git tags and release binaries are GPG signed
 with the key D26E6ED000654A3E (see https://syncthing.net/security.html).
 For release binaries, MD5 and SHA1 checksums are calculated and signed,
 available in the md5sum.txt.asc and sha1sum.txt.asc files.
 
-Documentation
-=============
+## Documentation
 
-Please see the [Syncthing
-documentation site](http://docs.syncthing.net/).
+Please see the [Syncthing documentation site][6].
 
-All code is licensed under the
-[MPLv2 License](https://github.com/syncthing/syncthing/blob/master/LICENSE).
+All code is licensed under the [MPLv2 License][7].
+
+[1]: https://github.com/syncthing/specs/blob/master/BEPv1.md
+[2]: http://docs.syncthing.net/intro/getting-started.html
+[3]: https://github.com/syncthing/syncthing/blob/master/etc
+[4]: https://webchat.freenode.net/
+[5]: http://docs.syncthing.net/dev/building.html
+[6]: http://docs.syncthing.net/
+[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE

appveyor.yaml

@@ -0,0 +1,12 @@
+version: '{branch}-{build}'
+clone_folder: C:\src\github.com\syncthing\syncthing
+init:
+- go version
+environment:
+  GOPATH: C:\
+build_script:
+- go run build.go zip
+test_script:
+- go run build.go test
+artifacts:
+- path: '*.zip'

build.go

@@ -13,7 +13,6 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
-	"crypto/md5"
 	"flag"
 	"fmt"
 	"io"
@@ -215,7 +214,7 @@ func build(pkg string, tags []string) {
 		binary += ".exe"
 	}
 
-	rmr(binary, binary+".md5")
+	rmr(binary)
 	args := []string{"build", "-ldflags", ldflags()}
 	if len(tags) > 0 {
 		args = append(args, "-tags", strings.Join(tags, ","))
@@ -226,13 +225,6 @@ func build(pkg string, tags []string) {
 	args = append(args, pkg)
 	setBuildEnv()
 	runPrint("go", args...)
-
-	// Create an md5 checksum of the binary, to be included in the archive for
-	// automatic upgrades.
-	err := md5File(binary)
-	if err != nil {
-		log.Fatal(err)
-	}
 }
 
 func buildTar() {
@@ -249,7 +241,6 @@ func buildTar() {
 		{src: "LICENSE", dst: name + "/LICENSE.txt"},
 		{src: "AUTHORS", dst: name + "/AUTHORS.txt"},
 		{src: "syncthing", dst: name + "/syncthing"},
-		{src: "syncthing.md5", dst: name + "/syncthing.md5"},
 	}
 
 	for _, file := range listFiles("etc") {
@@ -277,7 +268,6 @@ func buildZip() {
 		{src: "LICENSE", dst: name + "/LICENSE.txt"},
 		{src: "AUTHORS", dst: name + "/AUTHORS.txt"},
 		{src: "syncthing.exe", dst: name + "/syncthing.exe"},
-		{src: "syncthing.exe.md5", dst: name + "/syncthing.exe.md5"},
 	}
 
 	for _, file := range listFiles("extra") {
@@ -411,7 +401,7 @@ func assets() {
 }
 
 func xdr() {
-	runPrint("go", "generate", "./lib/discover", "./lib/db")
+	runPrint("go", "generate", "./lib/discover", "./lib/db", "./lib/protocol")
 }
 
 func translate() {
@@ -712,32 +702,6 @@ func zipFile(out string, files []archiveFile) {
 	}
 }
 
-func md5File(file string) error {
-	fd, err := os.Open(file)
-	if err != nil {
-		return err
-	}
-	defer fd.Close()
-
-	h := md5.New()
-	_, err = io.Copy(h, fd)
-	if err != nil {
-		return err
-	}
-
-	out, err := os.Create(file + ".md5")
-	if err != nil {
-		return err
-	}
-
-	_, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
-	if err != nil {
-		return err
-	}
-
-	return out.Close()
-}
-
 func vet(pkg string) {
 	bs, err := runError("go", "vet", pkg)
 	if err != nil && err.Error() == "exit status 3" || bytes.Contains(bs, []byte("no such tool \"vet\"")) {

cmd/stfileinfo/main.go

@@ -12,7 +12,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 )
 
@@ -68,7 +68,7 @@ func main() {
 		if *standardBlocks || blockSize < protocol.BlockSize {
 			blockSize = protocol.BlockSize
 		}
-		bs, err := scanner.Blocks(fd, blockSize, fi.Size())
+		bs, err := scanner.Blocks(fd, blockSize, fi.Size(), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -7,37 +7,105 @@
 package main
 
 import (
+	"crypto/tls"
+	"errors"
 	"flag"
-	"log"
+	"fmt"
+	"net/url"
 	"os"
+	"time"
 
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func main() {
-	log.SetFlags(0)
-	log.SetOutput(os.Stdout)
+var timeout = 5 * time.Second
 
+func main() {
 	var server string
 
-	flag.StringVar(&server, "server", "udp4://announce.syncthing.net:22026", "Announce server")
+	flag.StringVar(&server, "server", "", "Announce server (blank for default set)")
+	flag.DurationVar(&timeout, "timeout", timeout, "Query timeout")
+	flag.Usage = usage
 	flag.Parse()
 
-	if len(flag.Args()) != 1 || server == "" {
-		log.Printf("Usage: %s [-server=\"udp4://announce.syncthing.net:22026\"] <device>", os.Args[0])
+	if flag.NArg() != 1 {
+		flag.Usage()
 		os.Exit(64)
 	}
 
 	id, err := protocol.DeviceIDFromString(flag.Args()[0])
 	if err != nil {
-		log.Println(err)
+		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	discoverer := discover.NewDiscoverer(protocol.LocalDeviceID, nil)
-	discoverer.StartGlobal([]string{server}, 1)
-	for _, addr := range discoverer.Lookup(id) {
-		log.Println(addr)
+	if server != "" {
+		checkServers(id, server)
+	} else {
+		checkServers(id, config.DefaultDiscoveryServers...)
 	}
 }
+
+type checkResult struct {
+	server string
+	direct []string
+	relays []discover.Relay
+	error
+}
+
+func checkServers(deviceID protocol.DeviceID, servers ...string) {
+	t0 := time.Now()
+	resc := make(chan checkResult)
+	for _, srv := range servers {
+		srv := srv
+		go func() {
+			res := checkServer(deviceID, srv)
+			res.server = srv
+			resc <- res
+		}()
+	}
+
+	for _ = range servers {
+		res := <-resc
+
+		u, _ := url.Parse(res.server)
+		fmt.Printf("%s (%v):\n", u.Host, time.Since(t0))
+
+		if res.error != nil {
+			fmt.Println("  " + res.error.Error())
+		}
+		for _, addr := range res.direct {
+			fmt.Println("  address:", addr)
+		}
+		for _, rel := range res.relays {
+			fmt.Printf("  relay: %s (%d ms)\n", rel.URL, rel.Latency)
+		}
+	}
+}
+
+func checkServer(deviceID protocol.DeviceID, server string) checkResult {
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, nil)
+	if err != nil {
+		return checkResult{error: err}
+	}
+
+	res := make(chan checkResult, 1)
+
+	time.AfterFunc(timeout, func() {
+		res <- checkResult{error: errors.New("timeout")}
+	})
+
+	go func() {
+		direct, relays, err := disco.Lookup(deviceID)
+		res <- checkResult{direct: direct, relays: relays, error: err}
+	}()
+
+	return <-res
+}
+
+func usage() {
+	fmt.Printf("Usage:\n\t%s [options] <device ID>\n\nOptions:\n", os.Args[0])
+	flag.PrintDefaults()
+}

cmd/stindex/main.go

@@ -13,8 +13,8 @@ import (
 	"log"
 	"os"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 )

cmd/stsigtool/main.go

@@ -0,0 +1,115 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/signature"
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
+
+	flag.Parse()
+
+	if flag.NArg() < 1 {
+		log.Println(`Usage:
+	stsigtool <command>
+
+Where command is one of:
+
+	gen
+		- generate a new key pair
+
+	sign <privkeyfile> <datafile>
+		- sign a file
+
+	verify <signaturefile> <datafile>
+		- verify a signature, using the built in public key
+
+	verify <signaturefile> <datafile> <pubkeyfile>
+		- verify a signature, using the specified public key file
+`)
+	}
+
+	switch flag.Arg(0) {
+	case "gen":
+		gen()
+	case "sign":
+		sign(flag.Arg(1), flag.Arg(2))
+	case "verify":
+		if flag.NArg() == 4 {
+			verifyWithFile(flag.Arg(1), flag.Arg(2), flag.Arg(3))
+		} else {
+			verifyWithKey(flag.Arg(1), flag.Arg(2), upgrade.SigningKey)
+		}
+	}
+}
+
+func gen() {
+	priv, pub, err := signature.GenerateKeys()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Stdout.Write(priv)
+	os.Stdout.Write(pub)
+}
+
+func sign(keyname, dataname string) {
+	privkey, err := ioutil.ReadFile(keyname)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fd, err := os.Open(dataname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fd.Close()
+
+	sig, err := signature.Sign(privkey, fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Stdout.Write(sig)
+}
+
+func verifyWithFile(signame, dataname, keyname string) {
+	pubkey, err := ioutil.ReadFile(keyname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	verifyWithKey(signame, dataname, pubkey)
+}
+
+func verifyWithKey(signame, dataname string, pubkey []byte) {
+	sig, err := ioutil.ReadFile(signame)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fd, err := os.Open(dataname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fd.Close()
+
+	err = signature.Verify(pubkey, sig, fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("correct signature")
+}

cmd/syncthing/addresslister.go

@@ -0,0 +1,119 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+type addressLister struct {
+	upnpSvc *upnpSvc
+	cfg     *config.Wrapper
+}
+
+func newAddressLister(upnpSvc *upnpSvc, cfg *config.Wrapper) *addressLister {
+	return &addressLister{
+		upnpSvc: upnpSvc,
+		cfg:     cfg,
+	}
+}
+
+// ExternalAddresses returns a list of addresses that are our best guess for
+// where we are reachable from the outside. As a special case, we may return
+// one or more addresses with an empty IP address (0.0.0.0 or ::) and just
+// port number - this means that the outside address of a NAT gateway should
+// be substituted.
+func (e *addressLister) ExternalAddresses() []string {
+	return e.addresses(false)
+}
+
+// AllAddresses returns a list of addresses that are our best guess for where
+// we are reachable from the local network. Same conditions as
+// ExternalAddresses, but private IPv4 addresses are included.
+func (e *addressLister) AllAddresses() []string {
+	return e.addresses(true)
+}
+
+func (e *addressLister) addresses(includePrivateIPV4 bool) []string {
+	var addrs []string
+
+	// Grab our listen addresses from the config. Unspecified ones are passed
+	// on verbatim (to be interpreted by a global discovery server or local
+	// discovery peer). Public addresses are passed on verbatim. Private
+	// addresses are filtered.
+	for _, addrStr := range e.cfg.Options().ListenAddress {
+		addrURL, err := url.Parse(addrStr)
+		if err != nil {
+			l.Infoln("Listen address", addrStr, "is invalid:", err)
+			continue
+		}
+		addr, err := net.ResolveTCPAddr("tcp", addrURL.Host)
+		if err != nil {
+			l.Infoln("Listen address", addrStr, "is invalid:", err)
+			continue
+		}
+
+		if addr.IP == nil || addr.IP.IsUnspecified() {
+			// Address like 0.0.0.0:22000 or [::]:22000 or :22000; include as is.
+			addrs = append(addrs, "tcp://"+addr.String())
+		} else if isPublicIPv4(addr.IP) || isPublicIPv6(addr.IP) {
+			// A public address; include as is.
+			addrs = append(addrs, "tcp://"+addr.String())
+		} else if includePrivateIPV4 && addr.IP.To4().IsGlobalUnicast() {
+			// A private IPv4 address.
+			addrs = append(addrs, "tcp://"+addr.String())
+		}
+	}
+
+	// Get an external port mapping from the upnpSvc, if it has one. If so,
+	// add it as another unspecified address.
+	if e.upnpSvc != nil {
+		if port := e.upnpSvc.ExternalPort(); port != 0 {
+			addrs = append(addrs, fmt.Sprintf("tcp://:%d", port))
+		}
+	}
+
+	return addrs
+}
+
+func isPublicIPv4(ip net.IP) bool {
+	ip = ip.To4()
+	if ip == nil {
+		// Not an IPv4 address (IPv6)
+		return false
+	}
+
+	// IsGlobalUnicast below only checks that it's not link local or
+	// multicast, and we want to exclude private (NAT:ed) addresses as well.
+	rfc1918 := []net.IPNet{
+		{IP: net.IP{10, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{172, 16, 0, 0}, Mask: net.IPMask{255, 240, 0, 0}},
+		{IP: net.IP{192, 168, 0, 0}, Mask: net.IPMask{255, 255, 0, 0}},
+	}
+	for _, n := range rfc1918 {
+		if n.Contains(ip) {
+			return false
+		}
+	}
+
+	return ip.IsGlobalUnicast()
+}
+
+func isPublicIPv6(ip net.IP) bool {
+	if ip.To4() != nil {
+		// Not an IPv6 address (IPv4)
+		// (To16() returns a v6 mapped v4 address so can't be used to check
+		// that it's an actual v6 address)
+		return false
+	}
+
+	return ip.IsGlobalUnicast()
+}

cmd/syncthing/connections.go

@@ -1,372 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"crypto/tls"
-	"fmt"
-	"io"
-	"net"
-	"strings"
-	"time"
-
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/model"
-	"github.com/thejerf/suture"
-)
-
-// The connection service listens on TLS and dials configured unconnected
-// devices. Successful connections are handed to the model.
-type connectionSvc struct {
-	*suture.Supervisor
-	cfg    *config.Wrapper
-	myID   protocol.DeviceID
-	model  *model.Model
-	tlsCfg *tls.Config
-	conns  chan *tls.Conn
-}
-
-func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, model *model.Model, tlsCfg *tls.Config) *connectionSvc {
-	svc := &connectionSvc{
-		Supervisor: suture.NewSimple("connectionSvc"),
-		cfg:        cfg,
-		myID:       myID,
-		model:      model,
-		tlsCfg:     tlsCfg,
-		conns:      make(chan *tls.Conn),
-	}
-
-	// There are several moving parts here; one routine per listening address
-	// to handle incoming connections, one routine to periodically attempt
-	// outgoing connections, and lastly one routine to the the common handling
-	// regardless of whether the connection was incoming or outgoing. It ends
-	// up as in the diagram below. We embed a Supervisor to manage the
-	// routines (i.e. log and restart if they crash or exit, etc).
-	//
-	//                +-----------------+
-	//    Incoming    | +---------------+-+      +-----------------+
-	//   Connections  | |                 |      |                 |   Outgoing
-	// -------------->| |   svc.listen    |      |                 |  Connections
-	//                | |  (1 per listen  |      |   svc.connect   |-------------->
-	//                | |    address)     |      |                 |
-	//                +-+                 |      |                 |
-	//                  +-----------------+      +-----------------+
-	//                           v                        v
-	//                           |                        |
-	//                           |                        |
-	//                           +------------+-----------+
-	//                                        |
-	//                                        | svc.conns
-	//                                        v
-	//                               +-----------------+
-	//                               |                 |
-	//                               |                 |
-	//                               |   svc.handle    |------> model.AddConnection()
-	//                               |                 |
-	//                               |                 |
-	//                               +-----------------+
-	//
-	// TODO: Clean shutdown, and/or handling config changes on the fly. We
-	// partly do this now - new devices and addresses will be picked up, but
-	// not new listen addresses and we don't support disconnecting devices
-	// that are removed and so on...
-
-	svc.Add(serviceFunc(svc.connect))
-	for _, addr := range svc.cfg.Options().ListenAddress {
-		addr := addr
-		listener := serviceFunc(func() {
-			svc.listen(addr)
-		})
-		svc.Add(listener)
-	}
-	svc.Add(serviceFunc(svc.handle))
-
-	return svc
-}
-
-func (s *connectionSvc) handle() {
-next:
-	for conn := range s.conns {
-		cs := conn.ConnectionState()
-
-		// We should have negotiated the next level protocol "bep/1.0" as part
-		// of the TLS handshake. Unfortunately this can't be a hard error,
-		// because there are implementations out there that don't support
-		// protocol negotiation (iOS for one...).
-		if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != bepProtocolName {
-			l.Infof("Peer %s did not negotiate bep/1.0", conn.RemoteAddr())
-		}
-
-		// We should have received exactly one certificate from the other
-		// side. If we didn't, they don't have a device ID and we drop the
-		// connection.
-		certs := cs.PeerCertificates
-		if cl := len(certs); cl != 1 {
-			l.Infof("Got peer certificate list of length %d != 1 from %s; protocol error", cl, conn.RemoteAddr())
-			conn.Close()
-			continue
-		}
-		remoteCert := certs[0]
-		remoteID := protocol.NewDeviceID(remoteCert.Raw)
-
-		// The device ID should not be that of ourselves. It can happen
-		// though, especially in the presence of NAT hairpinning, multiple
-		// clients between the same NAT gateway, and global discovery.
-		if remoteID == myID {
-			l.Infof("Connected to myself (%s) - should not happen", remoteID)
-			conn.Close()
-			continue
-		}
-
-		// We should not already be connected to the other party. TODO: This
-		// could use some better handling. If the old connection is dead but
-		// hasn't timed out yet we may want to drop *that* connection and keep
-		// this one. But in case we are two devices connecting to each other
-		// in parallel we don't want to do that or we end up with no
-		// connections still established...
-		if s.model.ConnectedTo(remoteID) {
-			l.Infof("Connected to already connected device (%s)", remoteID)
-			conn.Close()
-			continue
-		}
-
-		for deviceID, deviceCfg := range s.cfg.Devices() {
-			if deviceID == remoteID {
-				// Verify the name on the certificate. By default we set it to
-				// "syncthing" when generating, but the user may have replaced
-				// the certificate and used another name.
-				certName := deviceCfg.CertName
-				if certName == "" {
-					certName = tlsDefaultCommonName
-				}
-				err := remoteCert.VerifyHostname(certName)
-				if err != nil {
-					// Incorrect certificate name is something the user most
-					// likely wants to know about, since it's an advanced
-					// config. Warn instead of Info.
-					l.Warnf("Bad certificate from %s (%v): %v", remoteID, conn.RemoteAddr(), err)
-					conn.Close()
-					continue next
-				}
-
-				// If rate limiting is set, and based on the address we should
-				// limit the connection, then we wrap it in a limiter.
-
-				limit := s.shouldLimit(conn.RemoteAddr())
-
-				wr := io.Writer(conn)
-				if limit && writeRateLimit != nil {
-					wr = &limitedWriter{conn, writeRateLimit}
-				}
-
-				rd := io.Reader(conn)
-				if limit && readRateLimit != nil {
-					rd = &limitedReader{conn, readRateLimit}
-				}
-
-				name := fmt.Sprintf("%s-%s", conn.LocalAddr(), conn.RemoteAddr())
-				protoConn := protocol.NewConnection(remoteID, rd, wr, s.model, name, deviceCfg.Compression)
-
-				l.Infof("Established secure connection to %s at %s", remoteID, name)
-				if debugNet {
-					l.Debugf("cipher suite: %04X in lan: %t", conn.ConnectionState().CipherSuite, !limit)
-				}
-
-				s.model.AddConnection(conn, protoConn)
-				continue next
-			}
-		}
-
-		if !s.cfg.IgnoredDevice(remoteID) {
-			events.Default.Log(events.DeviceRejected, map[string]string{
-				"device":  remoteID.String(),
-				"address": conn.RemoteAddr().String(),
-			})
-			l.Infof("Connection from %s with unknown device ID %s", conn.RemoteAddr(), remoteID)
-		} else {
-			l.Infof("Connection from %s with ignored device ID %s", conn.RemoteAddr(), remoteID)
-		}
-
-		conn.Close()
-	}
-}
-
-func (s *connectionSvc) listen(addr string) {
-	if debugNet {
-		l.Debugln("listening on", addr)
-	}
-
-	tcaddr, err := net.ResolveTCPAddr("tcp", addr)
-	if err != nil {
-		l.Fatalln("listen (BEP):", err)
-	}
-	listener, err := net.ListenTCP("tcp", tcaddr)
-	if err != nil {
-		l.Fatalln("listen (BEP):", err)
-	}
-
-	for {
-		conn, err := listener.Accept()
-		if err != nil {
-			l.Warnln("Accepting connection:", err)
-			continue
-		}
-
-		if debugNet {
-			l.Debugln("connect from", conn.RemoteAddr())
-		}
-
-		tcpConn := conn.(*net.TCPConn)
-		s.setTCPOptions(tcpConn)
-
-		tc := tls.Server(conn, s.tlsCfg)
-		err = tc.Handshake()
-		if err != nil {
-			l.Infoln("TLS handshake:", err)
-			tc.Close()
-			continue
-		}
-
-		s.conns <- tc
-	}
-}
-
-func (s *connectionSvc) connect() {
-	delay := time.Second
-	for {
-	nextDevice:
-		for deviceID, deviceCfg := range s.cfg.Devices() {
-			if deviceID == myID {
-				continue
-			}
-
-			if s.model.ConnectedTo(deviceID) {
-				continue
-			}
-
-			var addrs []string
-			for _, addr := range deviceCfg.Addresses {
-				if addr == "dynamic" {
-					if discoverer != nil {
-						t := discoverer.Lookup(deviceID)
-						if len(t) == 0 {
-							continue
-						}
-						addrs = append(addrs, t...)
-					}
-				} else {
-					addrs = append(addrs, addr)
-				}
-			}
-
-			for _, addr := range addrs {
-				host, port, err := net.SplitHostPort(addr)
-				if err != nil && strings.HasPrefix(err.Error(), "missing port") {
-					// addr is on the form "1.2.3.4"
-					addr = net.JoinHostPort(addr, "22000")
-				} else if err == nil && port == "" {
-					// addr is on the form "1.2.3.4:"
-					addr = net.JoinHostPort(host, "22000")
-				}
-				if debugNet {
-					l.Debugln("dial", deviceCfg.DeviceID, addr)
-				}
-
-				raddr, err := net.ResolveTCPAddr("tcp", addr)
-				if err != nil {
-					if debugNet {
-						l.Debugln(err)
-					}
-					continue
-				}
-
-				conn, err := net.DialTCP("tcp", nil, raddr)
-				if err != nil {
-					if debugNet {
-						l.Debugln(err)
-					}
-					continue
-				}
-
-				s.setTCPOptions(conn)
-
-				tc := tls.Client(conn, s.tlsCfg)
-				err = tc.Handshake()
-				if err != nil {
-					l.Infoln("TLS handshake:", err)
-					tc.Close()
-					continue
-				}
-
-				s.conns <- tc
-				continue nextDevice
-			}
-		}
-
-		time.Sleep(delay)
-		delay *= 2
-		if maxD := time.Duration(s.cfg.Options().ReconnectIntervalS) * time.Second; delay > maxD {
-			delay = maxD
-		}
-	}
-}
-
-func (*connectionSvc) setTCPOptions(conn *net.TCPConn) {
-	var err error
-	if err = conn.SetLinger(0); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetNoDelay(false); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetKeepAlive(true); err != nil {
-		l.Infoln(err)
-	}
-}
-
-func (s *connectionSvc) shouldLimit(addr net.Addr) bool {
-	if s.cfg.Options().LimitBandwidthInLan {
-		return true
-	}
-
-	tcpaddr, ok := addr.(*net.TCPAddr)
-	if !ok {
-		return true
-	}
-	for _, lan := range lans {
-		if lan.Contains(tcpaddr.IP) {
-			return false
-		}
-	}
-	return !tcpaddr.IP.IsLoopback()
-}
-
-func (s *connectionSvc) VerifyConfiguration(from, to config.Configuration) error {
-	return nil
-}
-
-func (s *connectionSvc) CommitConfiguration(from, to config.Configuration) bool {
-	// We require a restart if a device as been removed.
-
-	newDevices := make(map[protocol.DeviceID]bool, len(to.Devices))
-	for _, dev := range to.Devices {
-		newDevices[dev.DeviceID] = true
-	}
-
-	for _, dev := range from.Devices {
-		if !newDevices[dev.DeviceID] {
-			return false
-		}
-	}
-
-	return true
-}

cmd/syncthing/debug.go

@@ -9,10 +9,24 @@ package main
 import (
 	"os"
 	"strings"
+
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debugNet    = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
-	debugHTTP   = strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all"
-	debugSuture = strings.Contains(os.Getenv("STTRACE"), "suture") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger.NewFacility("main", "Main package")
+	httpl = logger.DefaultLogger.NewFacility("http", "REST API")
 )
+
+func init() {
+	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
+	l.SetDebug("http", strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
+}
+
+func shouldDebugMain() bool {
+	return l.ShouldDebug("main")
+}
+
+func shouldDebugHTTP() bool {
+	return l.ShouldDebug("http")
+}

cmd/syncthing/gui.go

@@ -20,65 +20,75 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/calmh/logger"
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/auto"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/relay"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/vitrun/qart/qr"
 	"golang.org/x/crypto/bcrypt"
 )
 
-type guiError struct {
-	Time  time.Time `json:"time"`
-	Error string    `json:"error"`
-}
-
 var (
 	configInSync = true
-	guiErrors    = []guiError{}
-	guiErrorsMut = sync.NewMutex()
 	startTime    = time.Now()
 )
 
 type apiSvc struct {
 	id              protocol.DeviceID
-	cfg             config.GUIConfiguration
+	cfg             *config.Wrapper
 	assetDir        string
 	model           *model.Model
+	eventSub        *events.BufferedSubscription
+	discoverer      *discover.CachingMux
+	relaySvc        *relay.Svc
 	listener        net.Listener
 	fss             *folderSummarySvc
 	stop            chan struct{}
 	systemConfigMut sync.Mutex
-	eventSub        *events.BufferedSubscription
+
+	guiErrors *logger.Recorder
+	systemLog *logger.Recorder
 }
 
-func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription) (*apiSvc, error) {
+func newAPISvc(id protocol.DeviceID, cfg *config.Wrapper, assetDir string, m *model.Model, eventSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc, errors, systemLog *logger.Recorder) (*apiSvc, error) {
 	svc := &apiSvc{
 		id:              id,
 		cfg:             cfg,
 		assetDir:        assetDir,
 		model:           m,
-		systemConfigMut: sync.NewMutex(),
 		eventSub:        eventSub,
+		discoverer:      discoverer,
+		relaySvc:        relaySvc,
+		systemConfigMut: sync.NewMutex(),
+		guiErrors:       errors,
+		systemLog:       systemLog,
 	}
 
 	var err error
-	svc.listener, err = svc.getListener(cfg)
+	svc.listener, err = svc.getListener(cfg.GUI())
 	return svc, err
 }
 
 func (s *apiSvc) getListener(cfg config.GUIConfiguration) (net.Listener, error) {
+	if guiAddress != "" {
+		// Override from the environment
+		cfg.Address = guiAddress
+	}
+
 	cert, err := tls.LoadX509KeyPair(locations[locHTTPSCertFile], locations[locHTTPSKeyFile])
 	if err != nil {
 		l.Infoln("Loading HTTPS certificate:", err)
@@ -92,7 +102,7 @@ func (s *apiSvc) getListener(cfg config.GUIConfiguration) (net.Listener, error)
 			name = tlsDefaultCommonName
 		}
 
-		cert, err = newCertificate(locations[locHTTPSCertFile], locations[locHTTPSKeyFile], name)
+		cert, err = tlsutil.NewCertificate(locations[locHTTPSCertFile], locations[locHTTPSKeyFile], name, tlsRSABits)
 	}
 	if err != nil {
 		return nil, err
@@ -120,15 +130,13 @@ func (s *apiSvc) getListener(cfg config.GUIConfiguration) (net.Listener, error)
 		return nil, err
 	}
 
-	listener := &DowngradingListener{rawListener, tlsCfg}
+	listener := &tlsutil.DowngradingListener{rawListener, tlsCfg}
 	return listener, nil
 }
 
 func (s *apiSvc) Serve() {
 	s.stop = make(chan struct{})
 
-	l.AddHandler(logger.LevelWarn, s.showGuiError)
-
 	// The GET handlers
 	getRestMux := http.NewServeMux()
 	getRestMux.HandleFunc("/rest/db/completion", s.getDBCompletion)              // device folder
@@ -153,6 +161,9 @@ func (s *apiSvc) Serve() {
 	getRestMux.HandleFunc("/rest/system/status", s.getSystemStatus)              // -
 	getRestMux.HandleFunc("/rest/system/upgrade", s.getSystemUpgrade)            // -
 	getRestMux.HandleFunc("/rest/system/version", s.getSystemVersion)            // -
+	getRestMux.HandleFunc("/rest/system/debug", s.getSystemDebug)                // -
+	getRestMux.HandleFunc("/rest/system/log", s.getSystemLog)                    // [since]
+	getRestMux.HandleFunc("/rest/system/log.txt", s.getSystemLogTxt)             // [since]
 
 	// The POST handlers
 	postRestMux := http.NewServeMux()
@@ -161,7 +172,6 @@ func (s *apiSvc) Serve() {
 	postRestMux.HandleFunc("/rest/db/override", s.postDBOverride)              // folder
 	postRestMux.HandleFunc("/rest/db/scan", s.postDBScan)                      // folder [sub...] [delay]
 	postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig)          // <body>
-	postRestMux.HandleFunc("/rest/system/discovery", s.postSystemDiscovery)    // device addr
 	postRestMux.HandleFunc("/rest/system/error", s.postSystemError)            // <body>
 	postRestMux.HandleFunc("/rest/system/error/clear", s.postSystemErrorClear) // -
 	postRestMux.HandleFunc("/rest/system/ping", s.restPing)                    // -
@@ -169,6 +179,9 @@ func (s *apiSvc) Serve() {
 	postRestMux.HandleFunc("/rest/system/restart", s.postSystemRestart)        // -
 	postRestMux.HandleFunc("/rest/system/shutdown", s.postSystemShutdown)      // -
 	postRestMux.HandleFunc("/rest/system/upgrade", s.postSystemUpgrade)        // -
+	postRestMux.HandleFunc("/rest/system/pause", s.postSystemPause)            // device
+	postRestMux.HandleFunc("/rest/system/resume", s.postSystemResume)          // device
+	postRestMux.HandleFunc("/rest/system/debug", s.postSystemDebug)            // [enable] [disable]
 
 	// Debug endpoints, not for general use
 	getRestMux.HandleFunc("/rest/debug/peerCompletion", s.getPeerCompletion)
@@ -188,33 +201,37 @@ func (s *apiSvc) Serve() {
 		assets:   auto.Assets(),
 	})
 
+	guiCfg := s.cfg.GUI()
+	if guiAPIKey != "" {
+		// Override from the environment
+		guiCfg.APIKey = guiAPIKey
+	}
+
 	// Wrap everything in CSRF protection. The /rest prefix should be
 	// protected, other requests will grant cookies.
-	handler := csrfMiddleware(s.id.String()[:5], "/rest", s.cfg.APIKey, mux)
+	handler := csrfMiddleware(s.id.String()[:5], "/rest", guiCfg.APIKey, mux)
 
 	// Add our version and ID as a header to responses
 	handler = withDetailsMiddleware(s.id, handler)
 
 	// Wrap everything in basic auth, if user/password is set.
-	if len(s.cfg.User) > 0 && len(s.cfg.Password) > 0 {
-		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], s.cfg, handler)
+	if len(guiCfg.User) > 0 && len(guiCfg.Password) > 0 {
+		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, handler)
 	}
 
 	// Redirect to HTTPS if we are supposed to
-	if s.cfg.UseTLS {
+	if guiCfg.UseTLS {
 		handler = redirectToHTTPSMiddleware(handler)
 	}
 
-	if debugHTTP {
-		handler = debugMiddleware(handler)
-	}
+	handler = debugMiddleware(handler)
 
 	srv := http.Server{
 		Handler:     handler,
 		ReadTimeout: 10 * time.Second,
 	}
 
-	s.fss = newFolderSummarySvc(s.model)
+	s.fss = newFolderSummarySvc(s.cfg, s.model)
 	defer s.fss.Stop()
 	s.fss.ServeBackground()
 
@@ -266,7 +283,6 @@ func (s *apiSvc) CommitConfiguration(from, to config.Configuration) bool {
 		// method.
 		return false
 	}
-	s.cfg = to.GUI
 
 	close(s.stop)
 
@@ -362,6 +378,36 @@ func (s *apiSvc) getSystemVersion(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+func (s *apiSvc) getSystemDebug(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	names := l.Facilities()
+	enabled := l.FacilityDebugging()
+	sort.Strings(enabled)
+	json.NewEncoder(w).Encode(map[string]interface{}{
+		"facilities": names,
+		"enabled":    enabled,
+	})
+}
+
+func (s *apiSvc) postSystemDebug(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	q := r.URL.Query()
+	for _, f := range strings.Split(q.Get("enable"), ",") {
+		if f == "" {
+			continue
+		}
+		l.SetDebug(f, true)
+		l.Infof("Enabled debug data for %q", f)
+	}
+	for _, f := range strings.Split(q.Get("disable"), ",") {
+		if f == "" {
+			continue
+		}
+		l.SetDebug(f, false)
+		l.Infof("Disabled debug data for %q", f)
+	}
+}
+
 func (s *apiSvc) getDBBrowse(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")
@@ -402,12 +448,12 @@ func (s *apiSvc) getDBCompletion(w http.ResponseWriter, r *http.Request) {
 func (s *apiSvc) getDBStatus(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")
-	res := folderSummary(s.model, folder)
+	res := folderSummary(s.cfg, s.model, folder)
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(res)
 }
 
-func folderSummary(m *model.Model, folder string) map[string]interface{} {
+func folderSummary(cfg *config.Wrapper, m *model.Model, folder string) map[string]interface{} {
 	var res = make(map[string]interface{})
 
 	res["invalid"] = cfg.Folders()[folder].Invalid
@@ -517,7 +563,7 @@ func (s *apiSvc) getDBFile(w http.ResponseWriter, r *http.Request) {
 
 func (s *apiSvc) getSystemConfig(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	json.NewEncoder(w).Encode(cfg.Raw())
+	json.NewEncoder(w).Encode(s.cfg.Raw())
 }
 
 func (s *apiSvc) postSystemConfig(w http.ResponseWriter, r *http.Request) {
@@ -532,7 +578,7 @@ func (s *apiSvc) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if to.GUI.Password != cfg.GUI().Password {
+	if to.GUI.Password != s.cfg.GUI().Password {
 		if to.GUI.Password != "" {
 			hash, err := bcrypt.GenerateFromPassword([]byte(to.GUI.Password), 0)
 			if err != nil {
@@ -547,7 +593,7 @@ func (s *apiSvc) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 
 	// Fixup usage reporting settings
 
-	if curAcc := cfg.Options().URAccepted; to.Options.URAccepted > curAcc {
+	if curAcc := s.cfg.Options().URAccepted; to.Options.URAccepted > curAcc {
 		// UR was enabled
 		to.Options.URAccepted = usageReportVersion
 		to.Options.URUniqueID = randomString(8)
@@ -559,9 +605,9 @@ func (s *apiSvc) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 
 	// Activate and save
 
-	resp := cfg.Replace(to)
+	resp := s.cfg.Replace(to)
 	configInSync = !resp.RequiresRestart
-	cfg.Save()
+	s.cfg.Save()
 }
 
 func (s *apiSvc) getSystemConfigInsync(w http.ResponseWriter, r *http.Request) {
@@ -579,7 +625,7 @@ func (s *apiSvc) postSystemReset(w http.ResponseWriter, r *http.Request) {
 	folder := qs.Get("folder")
 
 	if len(folder) > 0 {
-		if _, ok := cfg.Folders()[folder]; !ok {
+		if _, ok := s.cfg.Folders()[folder]; !ok {
 			http.Error(w, "Invalid folder ID", 500)
 			return
 		}
@@ -587,7 +633,7 @@ func (s *apiSvc) postSystemReset(w http.ResponseWriter, r *http.Request) {
 
 	if len(folder) == 0 {
 		// Reset all folders.
-		for folder := range cfg.Folders() {
+		for folder := range s.cfg.Folders() {
 			s.model.ResetFolder(folder)
 		}
 		s.flushResponse(`{"ok": "resetting database"}`, w)
@@ -625,8 +671,30 @@ func (s *apiSvc) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["alloc"] = m.Alloc
 	res["sys"] = m.Sys - m.HeapReleased
 	res["tilde"] = tilde
-	if cfg.Options().GlobalAnnEnabled && discoverer != nil {
-		res["extAnnounceOK"] = discoverer.ExtAnnounceOK()
+	if s.cfg.Options().LocalAnnEnabled || s.cfg.Options().GlobalAnnEnabled {
+		res["discoveryEnabled"] = true
+		discoErrors := make(map[string]string)
+		discoMethods := 0
+		for disco, err := range s.discoverer.ChildErrors() {
+			discoMethods++
+			if err != nil {
+				discoErrors[disco] = err.Error()
+			}
+		}
+		res["discoveryMethods"] = discoMethods
+		res["discoveryErrors"] = discoErrors
+	}
+	if s.relaySvc != nil {
+		res["relaysEnabled"] = true
+		relayClientStatus := make(map[string]bool)
+		relayClientLatency := make(map[string]int)
+		for _, relay := range s.relaySvc.Relays() {
+			latency, ok := s.relaySvc.RelayStatus(relay)
+			relayClientStatus[relay] = ok
+			relayClientLatency[relay] = int(latency / time.Millisecond)
+		}
+		res["relayClientStatus"] = relayClientStatus
+		res["relayClientLatency"] = relayClientLatency
 	}
 	cpuUsageLock.RLock()
 	var cpusum float64
@@ -637,6 +705,7 @@ func (s *apiSvc) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["cpuPercent"] = cpusum / float64(len(cpuUsagePercent)) / float64(runtime.NumCPU())
 	res["pathSeparator"] = string(filepath.Separator)
 	res["uptime"] = int(time.Since(startTime).Seconds())
+	res["startTime"] = startTime
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(res)
@@ -644,51 +713,53 @@ func (s *apiSvc) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 
 func (s *apiSvc) getSystemError(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	guiErrorsMut.Lock()
-	json.NewEncoder(w).Encode(map[string][]guiError{"errors": guiErrors})
-	guiErrorsMut.Unlock()
+	json.NewEncoder(w).Encode(map[string][]logger.Line{
+		"errors": s.guiErrors.Since(time.Time{}),
+	})
 }
 
 func (s *apiSvc) postSystemError(w http.ResponseWriter, r *http.Request) {
 	bs, _ := ioutil.ReadAll(r.Body)
 	r.Body.Close()
-	s.showGuiError(0, string(bs))
+	l.Warnln(string(bs))
 }
 
 func (s *apiSvc) postSystemErrorClear(w http.ResponseWriter, r *http.Request) {
-	guiErrorsMut.Lock()
-	guiErrors = []guiError{}
-	guiErrorsMut.Unlock()
+	s.guiErrors.Clear()
 }
 
-func (s *apiSvc) showGuiError(l logger.LogLevel, err string) {
-	guiErrorsMut.Lock()
-	guiErrors = append(guiErrors, guiError{time.Now(), err})
-	if len(guiErrors) > 5 {
-		guiErrors = guiErrors[len(guiErrors)-5:]
-	}
-	guiErrorsMut.Unlock()
+func (s *apiSvc) getSystemLog(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	since, err := time.Parse(time.RFC3339, q.Get("since"))
+	l.Debugln(err)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+	json.NewEncoder(w).Encode(map[string][]logger.Line{
+		"messages": s.systemLog.Since(since),
+	})
 }
 
-func (s *apiSvc) postSystemDiscovery(w http.ResponseWriter, r *http.Request) {
-	var qs = r.URL.Query()
-	var device = qs.Get("device")
-	var addr = qs.Get("addr")
-	if len(device) != 0 && len(addr) != 0 && discoverer != nil {
-		discoverer.Hint(device, []string{addr})
+func (s *apiSvc) getSystemLogTxt(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	since, err := time.Parse(time.RFC3339, q.Get("since"))
+	l.Debugln(err)
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+
+	for _, line := range s.systemLog.Since(since) {
+		fmt.Fprintf(w, "%s: %s\n", line.When.Format(time.RFC3339), line.Message)
 	}
 }
 
 func (s *apiSvc) getSystemDiscovery(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	devices := map[string][]discover.CacheEntry{}
+	devices := make(map[string]discover.CacheEntry)
 
-	if discoverer != nil {
+	if s.discoverer != nil {
 		// Device ids can't be marshalled as keys so we need to manually
 		// rebuild this map using strings. Discoverer may be nil if discovery
 		// has not started yet.
-		for device, entries := range discoverer.All() {
-			devices[device.String()] = entries
+		for device, entry := range s.discoverer.Cache() {
+			devices[device.String()] = entry
 		}
 	}
 
@@ -697,7 +768,7 @@ func (s *apiSvc) getSystemDiscovery(w http.ResponseWriter, r *http.Request) {
 
 func (s *apiSvc) getReport(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	json.NewEncoder(w).Encode(reportData(s.model))
+	json.NewEncoder(w).Encode(reportData(s.cfg, s.model))
 }
 
 func (s *apiSvc) getDBIgnores(w http.ResponseWriter, r *http.Request) {
@@ -766,7 +837,7 @@ func (s *apiSvc) getSystemUpgrade(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, upgrade.ErrUpgradeUnsupported.Error(), 500)
 		return
 	}
-	rel, err := upgrade.LatestRelease(Version)
+	rel, err := upgrade.LatestRelease(s.cfg.Options().ReleasesURL, Version)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return
@@ -809,7 +880,7 @@ func (s *apiSvc) getLang(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *apiSvc) postSystemUpgrade(w http.ResponseWriter, r *http.Request) {
-	rel, err := upgrade.LatestRelease(Version)
+	rel, err := upgrade.LatestRelease(s.cfg.Options().ReleasesURL, Version)
 	if err != nil {
 		l.Warnln("getting latest release:", err)
 		http.Error(w, err.Error(), 500)
@@ -830,6 +901,32 @@ func (s *apiSvc) postSystemUpgrade(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *apiSvc) postSystemPause(w http.ResponseWriter, r *http.Request) {
+	var qs = r.URL.Query()
+	var deviceStr = qs.Get("device")
+
+	device, err := protocol.DeviceIDFromString(deviceStr)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	s.model.PauseDevice(device)
+}
+
+func (s *apiSvc) postSystemResume(w http.ResponseWriter, r *http.Request) {
+	var qs = r.URL.Query()
+	var deviceStr = qs.Get("device")
+
+	device, err := protocol.DeviceIDFromString(deviceStr)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	s.model.ResumeDevice(device)
+}
+
 func (s *apiSvc) postDBScan(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")
@@ -881,7 +978,7 @@ func (s *apiSvc) getPeerCompletion(w http.ResponseWriter, r *http.Request) {
 	tot := map[string]float64{}
 	count := map[string]float64{}
 
-	for _, folder := range cfg.Folders() {
+	for _, folder := range s.cfg.Folders() {
 		for _, device := range folder.DeviceIDs() {
 			deviceStr := device.String()
 			if s.model.ConnectedTo(device) {

cmd/syncthing/gui_auth.go

@@ -25,7 +25,6 @@ var (
 )
 
 func basicAuthAndSessionMiddleware(cookieName string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
-
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if cfg.APIKey != "" && r.Header.Get("X-API-Key") == cfg.APIKey {
 			next.ServeHTTP(w, r)
@@ -43,9 +42,7 @@ func basicAuthAndSessionMiddleware(cookieName string, cfg config.GUIConfiguratio
 			}
 		}
 
-		if debugHTTP {
-			l.Debugln("Sessionless HTTP request with authentication; this is expensive.")
-		}
+		httpl.Debugln("Sessionless HTTP request with authentication; this is expensive.")
 
 		error := func() {
 			time.Sleep(time.Duration(rand.Intn(100)+100) * time.Millisecond)

cmd/syncthing/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"bytes"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -21,31 +22,34 @@ import (
 	"regexp"
 	"runtime"
 	"runtime/pprof"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/calmh/logger"
-	"github.com/juju/ratelimit"
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/connections"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/relay"
 	"github.com/syncthing/syncthing/lib/symlinks"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
+
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/thejerf/suture"
-	"golang.org/x/crypto/bcrypt"
 )
 
 var (
 	Version     = "unknown-dev"
-	Codename    = "Aluminium Ant"
+	Codename    = "Beryllium Bedbug"
 	BuildEnv    = "default"
 	BuildStamp  = "0"
 	BuildDate   time.Time
@@ -65,11 +69,21 @@ const (
 )
 
 const (
-	bepProtocolName   = "bep/1.0"
-	pingEventInterval = time.Minute
+	bepProtocolName      = "bep/1.0"
+	tlsDefaultCommonName = "syncthing"
+	tlsRSABits           = 3072
+	pingEventInterval    = time.Minute
+	maxSystemErrors      = 5
+	initialSystemLog     = 10
+	maxSystemLog         = 250
 )
 
-var l = logger.DefaultLogger
+// The discovery results are sorted by their source priority.
+const (
+	ipv6LocalDiscoveryPriority = iota
+	ipv4LocalDiscoveryPriority
+	globalDiscoveryPriority
+)
 
 func init() {
 	if Version != "unknown-dev" {
@@ -102,16 +116,12 @@ func init() {
 }
 
 var (
-	cfg            *config.Wrapper
-	myID           protocol.DeviceID
-	confDir        string
-	logFlags       = log.Ltime
-	writeRateLimit *ratelimit.Bucket
-	readRateLimit  *ratelimit.Bucket
-	stop           = make(chan int)
-	discoverer     *discover.Discoverer
-	cert           tls.Certificate
-	lans           []*net.IPNet
+	myID     protocol.DeviceID
+	confDir  string
+	logFlags = log.Ltime
+	stop     = make(chan int)
+	cert     tls.Certificate
+	lans     []*net.IPNet
 )
 
 const (
@@ -141,25 +151,11 @@ Development Settings
 The following environment variables modify syncthing's behavior in ways that
 are mostly useful for developers. Use with care.
 
- STGUIASSETS     Directory to load GUI assets from. Overrides compiled in assets.
+ STGUIASSETS     Directory to load GUI assets from. Overrides compiled in
+                 assets.
 
  STTRACE         A comma separated string of facilities to trace. The valid
-                 facility strings are:
-
-                 - "beacon"   (the beacon package)
-                 - "discover" (the discover package)
-                 - "events"   (the events package)
-                 - "files"    (the files package)
-                 - "http"     (the main package; HTTP requests)
-                 - "locks"    (the sync package; trace long held locks)
-                 - "net"      (the main package; connections & network messages)
-                 - "model"    (the model package)
-                 - "scanner"  (the scanner package)
-                 - "stats"    (the stats package)
-                 - "suture"   (the suture package; service management)
-                 - "upnp"     (the upnp package)
-                 - "xdr"      (the xdr package)
-                 - "all"      (all of the above)
+                 facility strings listed below.
 
  STPROFILER      Set to a listen address such as "127.0.0.1:9090" to start the
                  profiler with HTTP access.
@@ -182,32 +178,40 @@ are mostly useful for developers. Use with care.
 
  GOGC            Percentage of heap growth at which to trigger GC. Default is
                  100. Lower numbers keep peak memory usage down, at the price
-                 of CPU usage (ie. performance).`
+                 of CPU usage (ie. performance).
+
+
+Debugging Facilities
+--------------------
+
+The following are valid values for the STTRACE variable:
+
+%s`
 )
 
 // Command line and environment options
 var (
-	reset             bool
-	showVersion       bool
-	doUpgrade         bool
-	doUpgradeCheck    bool
-	upgradeTo         string
-	noBrowser         bool
-	noConsole         bool
-	generateDir       string
-	logFile           string
-	auditEnabled      bool
-	verbose           bool
-	noRestart         = os.Getenv("STNORESTART") != ""
-	noUpgrade         = os.Getenv("STNOUPGRADE") != ""
-	guiAddress        = os.Getenv("STGUIADDRESS") // legacy
-	guiAuthentication = os.Getenv("STGUIAUTH")    // legacy
-	guiAPIKey         = os.Getenv("STGUIAPIKEY")  // legacy
-	profiler          = os.Getenv("STPROFILER")
-	guiAssets         = os.Getenv("STGUIASSETS")
-	cpuProfile        = os.Getenv("STCPUPROFILE") != ""
-	stRestarting      = os.Getenv("STRESTART") != ""
-	innerProcess      = os.Getenv("STNORESTART") != "" || os.Getenv("STMONITORED") != ""
+	reset          bool
+	showVersion    bool
+	doUpgrade      bool
+	doUpgradeCheck bool
+	upgradeTo      string
+	noBrowser      bool
+	noConsole      bool
+	generateDir    string
+	logFile        string
+	auditEnabled   bool
+	verbose        bool
+	paused         bool
+	noRestart      = os.Getenv("STNORESTART") != ""
+	noUpgrade      = os.Getenv("STNOUPGRADE") != ""
+	guiAddress     = os.Getenv("STGUIADDRESS") // legacy
+	guiAPIKey      = os.Getenv("STGUIAPIKEY")  // legacy
+	profiler       = os.Getenv("STPROFILER")
+	guiAssets      = os.Getenv("STGUIASSETS")
+	cpuProfile     = os.Getenv("STCPUPROFILE") != ""
+	stRestarting   = os.Getenv("STRESTART") != ""
+	innerProcess   = os.Getenv("STNORESTART") != "" || os.Getenv("STMONITORED") != ""
 )
 
 func main() {
@@ -224,7 +228,6 @@ func main() {
 
 	flag.StringVar(&generateDir, "generate", "", "Generate key and config in specified dir, then exit")
 	flag.StringVar(&guiAddress, "gui-address", guiAddress, "Override GUI address")
-	flag.StringVar(&guiAuthentication, "gui-authentication", guiAuthentication, "Override GUI authentication; username:password")
 	flag.StringVar(&guiAPIKey, "gui-apikey", guiAPIKey, "Override GUI API key")
 	flag.StringVar(&confDir, "home", "", "Set configuration directory")
 	flag.IntVar(&logFlags, "logflags", logFlags, "Select information in log line prefix")
@@ -237,8 +240,10 @@ func main() {
 	flag.StringVar(&upgradeTo, "upgrade-to", upgradeTo, "Force upgrade directly from specified URL")
 	flag.BoolVar(&auditEnabled, "audit", false, "Write events to audit file")
 	flag.BoolVar(&verbose, "verbose", false, "Print verbose log output")
+	flag.BoolVar(&paused, "paused", false, "Start with all devices paused")
 
-	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, baseDirs["config"]))
+	longUsage := fmt.Sprintf(extraUsage, baseDirs["config"], debugFacilities())
+	flag.Usage = usageFor(flag.CommandLine, usage, longUsage)
 	flag.Parse()
 
 	if noConsole {
@@ -293,10 +298,13 @@ func main() {
 			l.Warnln("Key exists; will not overwrite.")
 			l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
 		} else {
-			cert, err = newCertificate(certFile, keyFile, tlsDefaultCommonName)
+			cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, tlsRSABits)
+			if err != nil {
+				l.Fatalln("Create certificate:", err)
+			}
 			myID = protocol.NewDeviceID(cert.Certificate[0])
 			if err != nil {
-				l.Fatalln("load cert:", err)
+				l.Fatalln("Load certificate:", err)
 			}
 			if err == nil {
 				l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
@@ -336,7 +344,11 @@ func main() {
 	}
 
 	if doUpgrade || doUpgradeCheck {
-		rel, err := upgrade.LatestRelease(Version)
+		releasesURL := "https://api.github.com/repos/syncthing/syncthing/releases?per_page=30"
+		if cfg, _, err := loadConfig(locations[locConfigFile]); err == nil {
+			releasesURL = cfg.Options().ReleasesURL
+		}
+		rel, err := upgrade.LatestRelease(releasesURL, Version)
 		if err != nil {
 			l.Fatalln("Upgrade:", err) // exits 1
 		}
@@ -383,6 +395,28 @@ func main() {
 	}
 }
 
+func debugFacilities() string {
+	facilities := l.Facilities()
+
+	// Get a sorted list of names
+	var names []string
+	maxLen := 0
+	for name := range facilities {
+		names = append(names, name)
+		if len(name) > maxLen {
+			maxLen = len(name)
+		}
+	}
+	sort.Strings(names)
+
+	// Format the choices
+	b := new(bytes.Buffer)
+	for _, name := range names {
+		fmt.Fprintf(b, " %-*s - %s\n", maxLen, name, facilities[name])
+	}
+	return b.String()
+}
+
 func upgradeViaRest() error {
 	cfg, err := config.Load(locations[locConfigFile], protocol.LocalDeviceID)
 	if err != nil {
@@ -425,9 +459,7 @@ func syncthingMain() {
 	// We want any logging it does to go through our log system.
 	mainSvc := suture.New("main", suture.Spec{
 		Log: func(line string) {
-			if debugSuture {
-				l.Debugln(line)
-			}
+			l.Debugln(line)
 		},
 	})
 	mainSvc.ServeBackground()
@@ -444,6 +476,9 @@ func syncthingMain() {
 		mainSvc.Add(newVerboseSvc())
 	}
 
+	errors := logger.NewRecorder(l, logger.LevelWarn, maxSystemErrors, 0)
+	systemLog := logger.NewRecorder(l, logger.LevelDebug, maxSystemLog, initialSystemLog)
+
 	// Event subscription for the API; must start early to catch the early events.
 	apiSub := events.NewBufferedSubscription(events.Default.Subscribe(events.AllEvents), 1000)
 
@@ -459,9 +494,10 @@ func syncthingMain() {
 	// Ensure that that we have a certificate and key.
 	cert, err := tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
 	if err != nil {
-		cert, err = newCertificate(locations[locCertFile], locations[locKeyFile], tlsDefaultCommonName)
+		l.Infof("Generating RSA key and certificate for %s...", tlsDefaultCommonName)
+		cert, err = tlsutil.NewCertificate(locations[locCertFile], locations[locKeyFile], tlsDefaultCommonName, tlsRSABits)
 		if err != nil {
-			l.Fatalln("load cert:", err)
+			l.Fatalln(err)
 		}
 	}
 
@@ -486,33 +522,21 @@ func syncthingMain() {
 
 	cfgFile := locations[locConfigFile]
 
-	var myName string
-
 	// Load the configuration file, if it exists.
 	// If it does not, create a template.
 
-	if info, err := os.Stat(cfgFile); err == nil {
-		if !info.Mode().IsRegular() {
-			l.Fatalln("Config file is not a file?")
-		}
-		cfg, err = config.Load(cfgFile, myID)
-		if err == nil {
-			myCfg := cfg.Devices()[myID]
-			if myCfg.Name == "" {
-				myName, _ = os.Hostname()
-			} else {
-				myName = myCfg.Name
-			}
+	cfg, myName, err := loadConfig(cfgFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			l.Infoln("No config file; starting with empty defaults")
+			myName, _ = os.Hostname()
+			newCfg := defaultConfig(myName)
+			cfg = config.Wrap(cfgFile, newCfg)
+			cfg.Save()
+			l.Infof("Edit %s to taste or use the GUI\n", cfgFile)
 		} else {
-			l.Fatalln("Configuration:", err)
+			l.Fatalln("Loading config:", err)
 		}
-	} else {
-		l.Infoln("No config file; starting with empty defaults")
-		myName, _ = os.Hostname()
-		newCfg := defaultConfig(myName)
-		cfg = config.Wrap(cfgFile, newCfg)
-		cfg.Save()
-		l.Infof("Edit %s to taste or use the GUI\n", cfgFile)
 	}
 
 	if cfg.Raw().OriginalVersion != config.CurrentVersion {
@@ -567,29 +591,28 @@ func syncthingMain() {
 		symlinks.Supported = false
 	}
 
-	protocol.PingTimeout = time.Duration(opts.PingTimeoutS) * time.Second
-	protocol.PingIdleTime = time.Duration(opts.PingIdleTimeS) * time.Second
-
-	if opts.MaxSendKbps > 0 {
-		writeRateLimit = ratelimit.NewBucketWithRate(float64(1000*opts.MaxSendKbps), int64(5*1000*opts.MaxSendKbps))
-	}
-	if opts.MaxRecvKbps > 0 {
-		readRateLimit = ratelimit.NewBucketWithRate(float64(1000*opts.MaxRecvKbps), int64(5*1000*opts.MaxRecvKbps))
-	}
-
 	if (opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0) && !opts.LimitBandwidthInLan {
 		lans, _ = osutil.GetLans()
 		networks := make([]string, 0, len(lans))
 		for _, lan := range lans {
 			networks = append(networks, lan.String())
 		}
+		for _, lan := range opts.AlwaysLocalNets {
+			_, ipnet, err := net.ParseCIDR(lan)
+			if err != nil {
+				l.Infoln("Network", lan, "is malformed:", err)
+				continue
+			}
+			networks = append(networks, ipnet.String())
+		}
 		l.Infoln("Local networks:", strings.Join(networks, ", "))
 	}
 
 	dbFile := locations[locDatabase]
-	ldb, err := leveldb.OpenFile(dbFile, dbOpts())
+	dbOpts := dbOpts(cfg)
+	ldb, err := leveldb.OpenFile(dbFile, dbOpts)
 	if leveldbIsCorrupted(err) {
-		ldb, err = leveldb.RecoverFile(dbFile, dbOpts())
+		ldb, err = leveldb.RecoverFile(dbFile, dbOpts)
 	}
 	if leveldbIsCorrupted(err) {
 		// The database is corrupted, and we've tried to recover it but it
@@ -599,7 +622,7 @@ func syncthingMain() {
 		if err := resetDB(); err != nil {
 			l.Fatalln("Remove database:", err)
 		}
-		ldb, err = leveldb.OpenFile(dbFile, dbOpts())
+		ldb, err = leveldb.OpenFile(dbFile, dbOpts)
 	}
 	if err != nil {
 		l.Fatalln("Cannot open database:", err, "- Is another copy of Syncthing already running?")
@@ -623,7 +646,13 @@ func syncthingMain() {
 			m.StartDeadlockDetector(time.Duration(it) * time.Second)
 		}
 	} else if !IsRelease || IsBeta {
-		m.StartDeadlockDetector(20 * 60 * time.Second)
+		m.StartDeadlockDetector(20 * time.Minute)
+	}
+
+	if paused {
+		for device := range cfg.Devices() {
+			m.PauseDevice(device)
+		}
 	}
 
 	// Clear out old indexes for other devices. Otherwise we'll start up and
@@ -648,32 +677,88 @@ func syncthingMain() {
 
 	mainSvc.Add(m)
 
-	// GUI
-
-	setupGUI(mainSvc, cfg, m, apiSub)
-
 	// The default port we announce, possibly modified by setupUPnP next.
 
-	addr, err := net.ResolveTCPAddr("tcp", opts.ListenAddress[0])
+	uri, err := url.Parse(opts.ListenAddress[0])
+	if err != nil {
+		l.Fatalf("Failed to parse listen address %s: %v", opts.ListenAddress[0], err)
+	}
+
+	addr, err := net.ResolveTCPAddr("tcp", uri.Host)
 	if err != nil {
 		l.Fatalln("Bad listen address:", err)
 	}
 
-	// Start discovery
+	// The externalAddr tracks our external addresses for discovery purposes.
 
-	localPort := addr.Port
-	discoverer = discovery(localPort)
+	var addrList *addressLister
 
-	// Start UPnP. The UPnP service will restart global discovery if the
-	// external port changes.
+	// Start UPnP
 
 	if opts.UPnPEnabled {
-		upnpSvc := newUPnPSvc(cfg, localPort)
+		upnpSvc := newUPnPSvc(cfg, addr.Port)
 		mainSvc.Add(upnpSvc)
+
+		// The external address tracker needs to know about the UPnP service
+		// so it can check for an external mapped port.
+		addrList = newAddressLister(upnpSvc, cfg)
+	} else {
+		addrList = newAddressLister(nil, cfg)
 	}
 
-	connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg)
-	cfg.Subscribe(connectionSvc)
+	// Start relay management
+
+	var relaySvc *relay.Svc
+	if opts.RelaysEnabled && (opts.GlobalAnnEnabled || opts.RelayWithoutGlobalAnn) {
+		relaySvc = relay.NewSvc(cfg, tlsCfg)
+		mainSvc.Add(relaySvc)
+	}
+
+	// Start discovery
+
+	cachedDiscovery := discover.NewCachingMux()
+	mainSvc.Add(cachedDiscovery)
+
+	if cfg.Options().GlobalAnnEnabled {
+		for _, srv := range cfg.GlobalDiscoveryServers() {
+			l.Infoln("Using discovery server", srv)
+			gd, err := discover.NewGlobal(srv, cert, addrList, relaySvc)
+			if err != nil {
+				l.Warnln("Global discovery:", err)
+				continue
+			}
+
+			// Each global discovery server gets its results cached for five
+			// minutes, and is not asked again for a minute when it's returned
+			// unsuccessfully.
+			cachedDiscovery.Add(gd, 5*time.Minute, time.Minute, globalDiscoveryPriority)
+		}
+	}
+
+	if cfg.Options().LocalAnnEnabled {
+		// v4 broadcasts
+		bcd, err := discover.NewLocal(myID, fmt.Sprintf(":%d", cfg.Options().LocalAnnPort), addrList, relaySvc)
+		if err != nil {
+			l.Warnln("IPv4 local discovery:", err)
+		} else {
+			cachedDiscovery.Add(bcd, 0, 0, ipv4LocalDiscoveryPriority)
+		}
+		// v6 multicasts
+		mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, addrList, relaySvc)
+		if err != nil {
+			l.Warnln("IPv6 local discovery:", err)
+		} else {
+			cachedDiscovery.Add(mcd, 0, 0, ipv6LocalDiscoveryPriority)
+		}
+	}
+
+	// GUI
+
+	setupGUI(mainSvc, cfg, m, apiSub, cachedDiscovery, relaySvc, errors, systemLog)
+
+	// Start connection management
+
+	connectionSvc := connections.NewConnectionSvc(cfg, myID, m, tlsCfg, cachedDiscovery, relaySvc, bepProtocolName, tlsDefaultCommonName, lans)
 	mainSvc.Add(connectionSvc)
 
 	if cpuProfile {
@@ -709,7 +794,7 @@ func syncthingMain() {
 	// The usageReportingManager registers itself to listen to configuration
 	// changes, and there's nothing more we need to tell it from the outside.
 	// Hence we don't keep the returned pointer.
-	newUsageReportingManager(m, cfg)
+	newUsageReportingManager(cfg, m)
 
 	if opts.RestartOnWakeup {
 		go standbyMonitor()
@@ -719,7 +804,7 @@ func syncthingMain() {
 		if noUpgrade {
 			l.Infof("No automatic upgrades; STNOUPGRADE environment variable defined.")
 		} else if IsRelease {
-			go autoUpgrade()
+			go autoUpgrade(cfg)
 		} else {
 			l.Infof("No automatic upgrades; %s is not a release version.", Version)
 		}
@@ -745,7 +830,30 @@ func syncthingMain() {
 	os.Exit(code)
 }
 
-func dbOpts() *opt.Options {
+func loadConfig(cfgFile string) (*config.Wrapper, string, error) {
+	info, err := os.Stat(cfgFile)
+	if err != nil {
+		return nil, "", err
+	}
+	if !info.Mode().IsRegular() {
+		return nil, "", errors.New("configuration is not a file")
+	}
+
+	cfg, err := config.Load(cfgFile, myID)
+	if err != nil {
+		return nil, "", err
+	}
+
+	myCfg := cfg.Devices()[myID]
+	myName := myCfg.Name
+	if myName == "" {
+		myName, _ = os.Hostname()
+	}
+
+	return cfg, myName, nil
+}
+
+func dbOpts(cfg *config.Wrapper) *opt.Options {
 	// Calculate a suitable database block cache capacity.
 
 	// Default is 8 MiB.
@@ -796,48 +904,53 @@ func startAuditing(mainSvc *suture.Supervisor) {
 	l.Infoln("Audit log in", auditFile)
 }
 
-func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription) {
-	opts := cfg.Options()
-	guiCfg := overrideGUIConfig(cfg.GUI(), guiAddress, guiAuthentication, guiAPIKey)
+func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc, errors, systemLog *logger.Recorder) {
+	guiCfg := cfg.GUI()
 
-	if guiCfg.Enabled && guiCfg.Address != "" {
-		addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address)
+	if !guiCfg.Enabled {
+		return
+	}
+	if guiCfg.Address == "" {
+		return
+	}
+
+	addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address)
+	if err != nil {
+		l.Fatalf("Cannot start GUI on %q: %v", guiCfg.Address, err)
+	} else {
+		var hostOpen, hostShow string
+		switch {
+		case addr.IP == nil:
+			hostOpen = "localhost"
+			hostShow = "0.0.0.0"
+		case addr.IP.IsUnspecified():
+			hostOpen = "localhost"
+			hostShow = addr.IP.String()
+		default:
+			hostOpen = addr.IP.String()
+			hostShow = hostOpen
+		}
+
+		var proto = "http"
+		if guiCfg.UseTLS {
+			proto = "https"
+		}
+
+		urlShow := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostShow, strconv.Itoa(addr.Port)))
+		l.Infoln("Starting web GUI on", urlShow)
+
+		api, err := newAPISvc(myID, cfg, guiAssets, m, apiSub, discoverer, relaySvc, errors, systemLog)
 		if err != nil {
-			l.Fatalf("Cannot start GUI on %q: %v", guiCfg.Address, err)
-		} else {
-			var hostOpen, hostShow string
-			switch {
-			case addr.IP == nil:
-				hostOpen = "localhost"
-				hostShow = "0.0.0.0"
-			case addr.IP.IsUnspecified():
-				hostOpen = "localhost"
-				hostShow = addr.IP.String()
-			default:
-				hostOpen = addr.IP.String()
-				hostShow = hostOpen
-			}
+			l.Fatalln("Cannot start GUI:", err)
+		}
+		cfg.Subscribe(api)
+		mainSvc.Add(api)
 
-			var proto = "http"
-			if guiCfg.UseTLS {
-				proto = "https"
-			}
-
-			urlShow := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostShow, strconv.Itoa(addr.Port)))
-			l.Infoln("Starting web GUI on", urlShow)
-			api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub)
-			if err != nil {
-				l.Fatalln("Cannot start GUI:", err)
-			}
-			cfg.Subscribe(api)
-			mainSvc.Add(api)
-
-			if opts.StartBrowser && !noBrowser && !stRestarting {
-				urlOpen := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostOpen, strconv.Itoa(addr.Port)))
-				// Can potentially block if the utility we are invoking doesn't
-				// fork, and just execs, hence keep it in it's own routine.
-				go openURL(urlOpen)
-			}
+		if cfg.Options().StartBrowser && !noBrowser && !stRestarting {
+			urlOpen := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostOpen, strconv.Itoa(addr.Port)))
+			// Can potentially block if the utility we are invoking doesn't
+			// fork, and just execs, hence keep it in it's own routine.
+			go openURL(urlOpen)
 		}
 	}
 }
@@ -871,7 +984,7 @@ func defaultConfig(myName string) config.Configuration {
 	if err != nil {
 		l.Fatalln("get free port (BEP):", err)
 	}
-	newCfg.Options.ListenAddress = []string{fmt.Sprintf("0.0.0.0:%d", port)}
+	newCfg.Options.ListenAddress = []string{fmt.Sprintf("tcp://0.0.0.0:%d", port)}
 	return newCfg
 }
 
@@ -896,23 +1009,6 @@ func shutdown() {
 	stop <- exitSuccess
 }
 
-func discovery(extPort int) *discover.Discoverer {
-	opts := cfg.Options()
-	disc := discover.NewDiscoverer(myID, opts.ListenAddress)
-
-	if opts.LocalAnnEnabled {
-		l.Infoln("Starting local discovery announcements")
-		disc.StartLocal(opts.LocalAnnPort, opts.LocalAnnMCAddr)
-	}
-
-	if opts.GlobalAnnEnabled {
-		l.Infoln("Starting global discovery announcements")
-		disc.StartGlobal(opts.GlobalAnnServers, uint16(extPort))
-	}
-
-	return disc
-}
-
 func ensureDir(dir string, mode int) {
 	fi, err := os.Stat(dir)
 	if os.IsNotExist(err) {
@@ -950,48 +1046,6 @@ func getFreePort(host string, ports ...int) (int, error) {
 	return addr.Port, nil
 }
 
-func overrideGUIConfig(cfg config.GUIConfiguration, address, authentication, apikey string) config.GUIConfiguration {
-	if address != "" {
-		cfg.Enabled = true
-
-		if !strings.Contains(address, "//") {
-			// Assume just an IP was given. Don't touch he TLS setting.
-			cfg.Address = address
-		} else {
-			parsed, err := url.Parse(address)
-			if err != nil {
-				l.Fatalln(err)
-			}
-			cfg.Address = parsed.Host
-			switch parsed.Scheme {
-			case "http":
-				cfg.UseTLS = false
-			case "https":
-				cfg.UseTLS = true
-			default:
-				l.Fatalln("Unknown scheme:", parsed.Scheme)
-			}
-		}
-	}
-
-	if authentication != "" {
-		authenticationParts := strings.SplitN(authentication, ":", 2)
-
-		hash, err := bcrypt.GenerateFromPassword([]byte(authenticationParts[1]), 0)
-		if err != nil {
-			l.Fatalln("Invalid GUI password:", err)
-		}
-
-		cfg.User = authenticationParts[0]
-		cfg.Password = string(hash)
-	}
-
-	if apikey != "" {
-		cfg.APIKey = apikey
-	}
-	return cfg
-}
-
 func standbyMonitor() {
 	restartDelay := time.Duration(60 * time.Second)
 	now := time.Now()
@@ -1012,7 +1066,7 @@ func standbyMonitor() {
 	}
 }
 
-func autoUpgrade() {
+func autoUpgrade(cfg *config.Wrapper) {
 	timer := time.NewTimer(0)
 	sub := events.Default.Subscribe(events.DeviceConnected)
 	for {
@@ -1026,7 +1080,7 @@ func autoUpgrade() {
 		case <-timer.C:
 		}
 
-		rel, err := upgrade.LatestRelease(Version)
+		rel, err := upgrade.LatestRelease(cfg.Options().ReleasesURL, Version)
 		if err == upgrade.ErrUpgradeUnsupported {
 			events.Default.Unsubscribe(sub)
 			return

cmd/syncthing/main_test.go

@@ -10,10 +10,10 @@ import (
 	"os"
 	"testing"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/protocol"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"

cmd/syncthing/perfstats_unix.go

@@ -15,7 +15,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 func init() {

cmd/syncthing/summarysvc.go

@@ -9,6 +9,7 @@ package main
 import (
 	"time"
 
+	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -20,6 +21,7 @@ import (
 type folderSummarySvc struct {
 	*suture.Supervisor
 
+	cfg       *config.Wrapper
 	model     *model.Model
 	stop      chan struct{}
 	immediate chan string
@@ -33,9 +35,10 @@ type folderSummarySvc struct {
 	lastEventReqMut sync.Mutex
 }
 
-func newFolderSummarySvc(m *model.Model) *folderSummarySvc {
+func newFolderSummarySvc(cfg *config.Wrapper, m *model.Model) *folderSummarySvc {
 	svc := &folderSummarySvc{
 		Supervisor:      suture.NewSimple("folderSummarySvc"),
+		cfg:             cfg,
 		model:           m,
 		stop:            make(chan struct{}),
 		immediate:       make(chan string),
@@ -162,13 +165,13 @@ func (c *folderSummarySvc) foldersToHandle() []string {
 func (c *folderSummarySvc) sendSummary(folder string) {
 	// The folder summary contains how many bytes, files etc
 	// are in the folder and how in sync we are.
-	data := folderSummary(c.model, folder)
+	data := folderSummary(c.cfg, c.model, folder)
 	events.Default.Log(events.FolderSummary, map[string]interface{}{
 		"folder":  folder,
 		"summary": data,
 	})
 
-	for _, devCfg := range cfg.Folders()[folder].Devices {
+	for _, devCfg := range c.cfg.Folders()[folder].Devices {
 		if devCfg.DeviceID.Equals(myID) {
 			// We already know about ourselves.
 			continue

cmd/syncthing/upnpsvc.go

@@ -11,26 +11,30 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/upnp"
 )
 
 // The UPnP service runs a loop for discovery of IGDs (Internet Gateway
 // Devices) and setup/renewal of a port mapping.
 type upnpSvc struct {
-	cfg       *config.Wrapper
-	localPort int
-	stop      chan struct{}
+	cfg        *config.Wrapper
+	localPort  int
+	extPort    int
+	extPortMut sync.Mutex
+	stop       chan struct{}
 }
 
 func newUPnPSvc(cfg *config.Wrapper, localPort int) *upnpSvc {
 	return &upnpSvc{
-		cfg:       cfg,
-		localPort: localPort,
+		cfg:        cfg,
+		localPort:  localPort,
+		extPortMut: sync.NewMutex(),
 	}
 }
 
 func (s *upnpSvc) Serve() {
-	extPort := 0
 	foundIGD := true
 	s.stop = make(chan struct{})
 
@@ -38,7 +42,15 @@ func (s *upnpSvc) Serve() {
 		igds := upnp.Discover(time.Duration(s.cfg.Options().UPnPTimeoutS) * time.Second)
 		if len(igds) > 0 {
 			foundIGD = true
-			extPort = s.tryIGDs(igds, extPort)
+			s.extPortMut.Lock()
+			oldExtPort := s.extPort
+			s.extPortMut.Unlock()
+
+			newExtPort := s.tryIGDs(igds, oldExtPort)
+
+			s.extPortMut.Lock()
+			s.extPort = newExtPort
+			s.extPortMut.Unlock()
 		} else if foundIGD {
 			// Only print a notice if we've previously found an IGD or this is
 			// the first time around.
@@ -64,6 +76,13 @@ func (s *upnpSvc) Stop() {
 	close(s.stop)
 }
 
+func (s *upnpSvc) ExternalPort() int {
+	s.extPortMut.Lock()
+	port := s.extPort
+	s.extPortMut.Unlock()
+	return port
+}
+
 func (s *upnpSvc) tryIGDs(igds []upnp.IGD, prevExtPort int) int {
 	// Lets try all the IGDs we found and use the first one that works.
 	// TODO: Use all of them, and sort out the resulting mess to the
@@ -76,17 +95,10 @@ func (s *upnpSvc) tryIGDs(igds []upnp.IGD, prevExtPort int) int {
 		}
 
 		if extPort != prevExtPort {
-			// External port changed; refresh the discovery announcement.
-			// TODO: Don't reach out to some magic global here?
 			l.Infof("New UPnP port mapping: external port %d to local port %d.", extPort, s.localPort)
-			if s.cfg.Options().GlobalAnnEnabled {
-				discoverer.StopGlobal()
-				discoverer.StartGlobal(s.cfg.Options().GlobalAnnServers, uint16(extPort))
-			}
-		}
-		if debugNet {
-			l.Debugf("Created/updated UPnP port mapping for external port %d on device %s.", extPort, igd.FriendlyIdentifier())
+			events.Default.Log(events.ExternalPortMappingChanged, map[string]int{"port": extPort})
 		}
+		l.Debugf("Created/updated UPnP port mapping for external port %d on device %s.", extPort, igd.FriendlyIdentifier())
 		return extPort
 	}
 

cmd/syncthing/usage_report.go

@@ -10,30 +10,36 @@ import (
 	"bytes"
 	"crypto/rand"
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
 	"runtime"
+	"sort"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/thejerf/suture"
 )
 
 // Current version number of the usage report, for acceptance purposes. If
 // fields are added or changed this integer must be incremented so that users
 // are prompted for acceptance of the new report.
-const usageReportVersion = 1
+const usageReportVersion = 2
 
 type usageReportingManager struct {
+	cfg   *config.Wrapper
 	model *model.Model
 	sup   *suture.Supervisor
 }
 
-func newUsageReportingManager(m *model.Model, cfg *config.Wrapper) *usageReportingManager {
+func newUsageReportingManager(cfg *config.Wrapper, m *model.Model) *usageReportingManager {
 	mgr := &usageReportingManager{
+		cfg:   cfg,
 		model: m,
 	}
 
@@ -54,9 +60,7 @@ func (m *usageReportingManager) VerifyConfiguration(from, to config.Configuratio
 func (m *usageReportingManager) CommitConfiguration(from, to config.Configuration) bool {
 	if to.Options.URAccepted >= usageReportVersion && m.sup == nil {
 		// Usage reporting was turned on; lets start it.
-		svc := &usageReportingService{
-			model: m.model,
-		}
+		svc := newUsageReportingService(m.cfg, m.model)
 		m.sup = suture.NewSimple("usageReporting")
 		m.sup.Add(svc)
 		m.sup.ServeBackground()
@@ -75,8 +79,9 @@ func (m *usageReportingManager) String() string {
 
 // reportData returns the data to be sent in a usage report. It's used in
 // various places, so not part of the usageReportingSvc object.
-func reportData(m *model.Model) map[string]interface{} {
+func reportData(cfg *config.Wrapper, m *model.Model) map[string]interface{} {
 	res := make(map[string]interface{})
+	res["urVersion"] = usageReportVersion
 	res["uniqueID"] = cfg.Options().URUniqueID
 	res["version"] = Version
 	res["longVersion"] = LongVersion
@@ -120,31 +125,152 @@ func reportData(m *model.Model) map[string]interface{} {
 	if err == nil {
 		res["memorySize"] = bytes / 1024 / 1024
 	}
+	res["numCPU"] = runtime.NumCPU()
+
+	var rescanIntvs []int
+	folderUses := map[string]int{
+		"readonly":      0,
+		"ignorePerms":   0,
+		"ignoreDelete":  0,
+		"autoNormalize": 0,
+	}
+	for _, cfg := range cfg.Folders() {
+		rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS)
+
+		if cfg.ReadOnly {
+			folderUses["readonly"]++
+		}
+		if cfg.IgnorePerms {
+			folderUses["ignorePerms"]++
+		}
+		if cfg.IgnoreDelete {
+			folderUses["ignoreDelete"]++
+		}
+		if cfg.AutoNormalize {
+			folderUses["autoNormalize"]++
+		}
+	}
+	sort.Ints(rescanIntvs)
+	res["rescanIntvs"] = rescanIntvs
+	res["folderUses"] = folderUses
+
+	deviceUses := map[string]int{
+		"introducer":       0,
+		"customCertName":   0,
+		"compressAlways":   0,
+		"compressMetadata": 0,
+		"compressNever":    0,
+		"dynamicAddr":      0,
+		"staticAddr":       0,
+	}
+	for _, cfg := range cfg.Devices() {
+		if cfg.Introducer {
+			deviceUses["introducer"]++
+		}
+		if cfg.CertName != "" && cfg.CertName != "syncthing" {
+			deviceUses["customCertName"]++
+		}
+		if cfg.Compression == protocol.CompressAlways {
+			deviceUses["compressAlways"]++
+		} else if cfg.Compression == protocol.CompressMetadata {
+			deviceUses["compressMetadata"]++
+		} else if cfg.Compression == protocol.CompressNever {
+			deviceUses["compressNever"]++
+		}
+		for _, addr := range cfg.Addresses {
+			if addr == "dynamic" {
+				deviceUses["dynamicAddr"]++
+			} else {
+				deviceUses["staticAddr"]++
+			}
+		}
+	}
+	res["deviceUses"] = deviceUses
+
+	defaultAnnounceServersDNS, defaultAnnounceServersIP, otherAnnounceServers := 0, 0, 0
+	for _, addr := range cfg.Options().GlobalAnnServers {
+		if addr == "default" {
+			defaultAnnounceServersDNS++
+		} else if stringIn(addr, config.DefaultDiscoveryServersIP) {
+			defaultAnnounceServersIP++
+		} else {
+			otherAnnounceServers++
+		}
+	}
+	res["announce"] = map[string]interface{}{
+		"globalEnabled":     cfg.Options().GlobalAnnEnabled,
+		"localEnabled":      cfg.Options().LocalAnnEnabled,
+		"defaultServersDNS": defaultAnnounceServersDNS,
+		"defaultServersIP":  defaultAnnounceServersIP,
+		"otherServers":      otherAnnounceServers,
+	}
+
+	defaultRelayServers, otherRelayServers := 0, 0
+	for _, addr := range cfg.Options().RelayServers {
+		switch addr {
+		case "dynamic+https://relays.syncthing.net":
+			defaultRelayServers++
+		default:
+			otherRelayServers++
+		}
+	}
+	res["relays"] = map[string]interface{}{
+		"enabled":        cfg.Options().RelaysEnabled,
+		"defaultServers": defaultRelayServers,
+		"otherServers":   otherRelayServers,
+	}
+
+	res["usesRateLimit"] = cfg.Options().MaxRecvKbps > 0 || cfg.Options().MaxSendKbps > 0
+
+	res["upgradeAllowedManual"] = !(upgrade.DisabledByCompilation || noUpgrade)
+	res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgrade) && cfg.Options().AutoUpgradeIntervalH > 0
 
 	return res
 }
 
+func stringIn(needle string, haystack []string) bool {
+	for _, s := range haystack {
+		if needle == s {
+			return true
+		}
+	}
+	return false
+}
+
 type usageReportingService struct {
+	cfg   *config.Wrapper
 	model *model.Model
 	stop  chan struct{}
 }
 
+func newUsageReportingService(cfg *config.Wrapper, model *model.Model) *usageReportingService {
+	return &usageReportingService{
+		cfg:   cfg,
+		model: model,
+		stop:  make(chan struct{}),
+	}
+}
+
 func (s *usageReportingService) sendUsageReport() error {
-	d := reportData(s.model)
+	d := reportData(s.cfg, s.model)
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(d)
 
-	var client = http.DefaultClient
+	transp := &http.Transport{}
+	client := &http.Client{Transport: transp}
 	if BuildEnv == "android" {
 		// This works around the lack of DNS resolution on Android... :(
-		tr := &http.Transport{
-			Dial: func(network, addr string) (net.Conn, error) {
-				return net.Dial(network, "194.126.249.13:443")
-			},
+		transp.Dial = func(network, addr string) (net.Conn, error) {
+			return net.Dial(network, "194.126.249.13:443")
 		}
-		client = &http.Client{Transport: tr}
 	}
-	_, err := client.Post("https://data.syncthing.net/newdata", "application/json", &b)
+
+	if s.cfg.Options().URPostInsecurely {
+		transp.TLSClientConfig = &tls.Config{
+			InsecureSkipVerify: true,
+		}
+	}
+	_, err := client.Post(s.cfg.Options().URURL, "application/json", &b)
 	return err
 }
 
@@ -154,7 +280,7 @@ func (s *usageReportingService) Serve() {
 	l.Infoln("Starting usage reporting")
 	defer l.Infoln("Stopping usage reporting")
 
-	t := time.NewTimer(10 * time.Minute) // time to initial report at start
+	t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second) // time to initial report at start
 	for {
 		select {
 		case <-s.stop:

cmd/syncthing/verbose.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/syncthing/syncthing/lib/events"
 )
@@ -60,7 +61,7 @@ func (s *verboseSvc) WaitForStart() {
 
 func (s *verboseSvc) formatEvent(ev events.Event) string {
 	switch ev.Type {
-	case events.Ping, events.DownloadProgress:
+	case events.Ping, events.DownloadProgress, events.LocalIndexUpdated:
 		// Skip
 		return ""
 
@@ -74,7 +75,7 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
 		return fmt.Sprintf("Discovered device %v at %v", data["device"], data["addrs"])
 	case events.DeviceConnected:
 		data := ev.Data.(map[string]string)
-		return fmt.Sprintf("Connected to device %v at %v", data["id"], data["addr"])
+		return fmt.Sprintf("Connected to device %v at %v (type %s)", data["id"], data["addr"], data["type"])
 	case events.DeviceDisconnected:
 		data := ev.Data.(map[string]string)
 		return fmt.Sprintf("Disconnected from device %v", data["id"])
@@ -86,9 +87,6 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
 	case events.RemoteIndexUpdated:
 		data := ev.Data.(map[string]interface{})
 		return fmt.Sprintf("Device %v sent an index update for %q with %d items", data["device"], data["folder"], data["items"])
-	case events.LocalIndexUpdated:
-		data := ev.Data.(map[string]interface{})
-		return fmt.Sprintf("Updated index for folder %q with %v items", data["folder"], data["items"])
 
 	case events.DeviceRejected:
 		data := ev.Data.(map[string]interface{})
@@ -123,6 +121,35 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
 		delete(sum, "ignorePatterns")
 		delete(sum, "stateChanged")
 		return fmt.Sprintf("Summary for folder %q is %v", data["folder"], data["summary"])
+	case events.FolderScanProgress:
+		data := ev.Data.(map[string]interface{})
+		folder := data["folder"].(string)
+		current := data["current"].(int64)
+		total := data["total"].(int64)
+		var pct int64
+		if total > 0 {
+			pct = 100 * current / total
+		}
+		return fmt.Sprintf("Scanning folder %q, %d%% done", folder, pct)
+
+	case events.DevicePaused:
+		data := ev.Data.(map[string]string)
+		device := data["device"]
+		return fmt.Sprintf("Device %v was paused", device)
+	case events.DeviceResumed:
+		data := ev.Data.(map[string]string)
+		device := data["device"]
+		return fmt.Sprintf("Device %v was resumed", device)
+
+	case events.ExternalPortMappingChanged:
+		data := ev.Data.(map[string]int)
+		port := data["port"]
+		return fmt.Sprintf("External port mapping changed; new port is %d.", port)
+	case events.RelayStateChanged:
+		data := ev.Data.(map[string][]string)
+		newRelays := data["new"]
+		return fmt.Sprintf("Relay state changed; connected relay(s) are %s.", strings.Join(newRelays, ", "))
+
 	}
 
 	return fmt.Sprintf("%s %#v", ev.Type, ev)

etc/linux-systemd/README.md

@@ -1,8 +1,8 @@
 # Systemd Configuration
 
-This directory contains configuration files for running syncthing under the
+This directory contains configuration files for running Syncthing under the
 "systemd" service manager on Linux both under either a systemd system service or
 systemd user service. For further documentation take a look at the [systemd
-section][1] on the Github Wiki.
+section][1] on http://docs.syncthing.net.
 
-[1]: http://docs.syncthing.net/users/autostart.html#systemd 
+[1]: http://docs.syncthing.net/users/autostart.html#systemd

gui/assets/css/overrides.css

@@ -21,6 +21,10 @@ ul+h5 {
     margin-top: 1.5em;
 }
 
+#content {
+    margin-bottom: 50px;
+}
+
 .panel-progress {
     background: #3498db;
     height: 3px;
@@ -41,8 +45,17 @@ identicon {
     position: relative;
     width: 1em;
     height: 1em;
-    line-height: 1em;
-    overflow: visible;
+    line-height: 1;
+    margin-right: 5px;
+}
+
+.identicon {
+    width: 1em;
+    height: 1em;
+}
+
+.identicon rect {
+    fill: #333;
 }
 
 .checkbox {
@@ -55,17 +68,7 @@ identicon {
 
 .popover {
     max-width: none;
-}
-
-.identicon {
-    width: 20px;
-    height: 20px;
-}
-
-.panel-heading .identicon {
-    display: block;
-    position: absolute;
-    top: 1px;
+    min-width: 250px;
 }
 
 .panel-heading .fa, .modal-header .fa {
@@ -77,10 +80,6 @@ identicon {
     overflow: hidden;
 }
 
-.identicon rect {
-    fill: #666;
-}
-
 .text-monospace {
     font-family: Menlo, Monaco, Consolas, "Courier New", monospace;
 }

gui/assets/lang/lang-en.json

@@ -49,7 +49,7 @@
    "Edit Folder": "Edit Folder",
    "Editing": "Editing",
    "Enable UPnP": "Enable UPnP",
-   "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.",
+   "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.",
    "Enter ignore patterns, one per line.": "Enter ignore patterns, one per line.",
    "Error": "Error",
    "External File Versioning": "External File Versioning",
@@ -113,6 +113,8 @@
    "Override Changes": "Override Changes",
    "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for",
    "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Path where versions should be stored (leave empty for the default .stversions folder in the folder).",
+   "Pause": "Pause",
+   "Paused": "Paused",
    "Please consult the release notes before performing a major upgrade.": "Please consult the release notes before performing a major upgrade.",
    "Please wait": "Please wait",
    "Preview": "Preview",
@@ -120,6 +122,8 @@
    "Quick guide to supported patterns": "Quick guide to supported patterns",
    "RAM Utilization": "RAM Utilization",
    "Random": "Random",
+   "Relayed via": "Relayed via",
+   "Relays": "Relays",
    "Release Notes": "Release Notes",
    "Remove": "Remove",
    "Rescan": "Rescan",
@@ -128,6 +132,7 @@
    "Restart": "Restart",
    "Restart Needed": "Restart Needed",
    "Restarting": "Restarting",
+   "Resume": "Resume",
    "Reused": "Reused",
    "Save": "Save",
    "Scanning": "Scanning",
@@ -207,6 +212,7 @@
    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.",
    "Yes": "Yes",
    "You must keep at least one version.": "You must keep at least one version.",
+   "days": "days",
    "full documentation": "full documentation",
    "items": "items",
    "{%device%} wants to share folder \"{%folder%}\".": "{{device}} wants to share folder \"{{folder}}\"."

gui/assets/lang/lang-it.json

@@ -10,8 +10,8 @@
     "Add new folder?": "Aggiungere una nuova cartella?",
     "Address": "Indirizzo",
     "Addresses": "Indirizzi",
-    "Advanced": "Avanzato",
-    "Advanced Configuration": "Configurazione avanzata",
+    "Advanced": "Advanced",
+    "Advanced Configuration": "Advanced Configuration",
     "All Data": "Tutti i Dati",
     "Allow Anonymous Usage Reporting?": "Abilitare Statistiche Anonime di Utilizzo?",
     "Alphabetic": "Alfabetico",
@@ -19,7 +19,7 @@
     "Anonymous Usage Reporting": "Statistiche Anonime di Utilizzo",
     "Any devices configured on an introducer device will be added to this device as well.": "Qualsiasi dispositivo configurato in un introduttore verrà aggiunto anche a questo dispositivo.",
     "Automatic upgrades": "Aggiornamenti automatici",
-    "Be careful!": "Fai attenzione!",
+    "Be careful!": "Be careful!",
     "Bugs": "Bug",
     "CPU Utilization": "Utilizzo CPU",
     "Changelog": "Changelog",
@@ -49,23 +49,23 @@
     "Edit Folder": "Modifica Cartella",
     "Editing": "Modifica di",
     "Enable UPnP": "Attiva UPnP",
-    "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Inserisci indirizzi separati da virgola (\"tcp://ip:porta\", \"tcp://host:porta\")  oppure \"dynamic\" per effettuare il rilevamento automatico dell'indirizzo.",
+    "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.",
     "Enter ignore patterns, one per line.": "Inserisci gli schemi di esclusione, uno per riga.",
     "Error": "Errore",
     "External File Versioning": "Controllo Versione Esterno",
-    "Failed Items": "Elementi errati",
+    "Failed Items": "Failed Items",
     "File Pull Order": "Ordine di prelievo dei file",
     "File Versioning": "Controllo Versione dei File",
     "File permission bits are ignored when looking for changes. Use on FAT file systems.": "Il software evita i bit dei permessi dei file durante il controllo delle modifiche. Utilizzato nei filesystem FAT.",
     "Files are moved to .stversions folder when replaced or deleted by Syncthing.": "I file sono spostati nella certella .stversions quando vengono sostituiti o cancellati da Syncthing.",
     "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.": "I file sostituiti o eliminati da Syncthing vengono datati e spostati in una cartella .stversions.",
     "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "I file sono protetti dalle modifiche effettuate negli altri dispositivi, ma le modifiche effettuate in questo dispositivo verranno inviate anche al resto del cluster.",
-    "Folder": "Cartella",
+    "Folder": "Folder",
     "Folder ID": "ID Cartella",
     "Folder Master": "Cartella Principale",
     "Folder Path": "Percorso Cartella",
     "Folders": "Cartelle",
-    "GUI": "Interfaccia grafica utente",
+    "GUI": "GUI",
     "GUI Authentication Password": "Password di Autenticazione dell'Utente",
     "GUI Authentication User": "Utente dell'Interfaccia Grafica",
     "GUI Listen Addresses": "Indirizzi dell'Interfaccia Grafica",
@@ -74,12 +74,12 @@
     "Global Discovery Server": "Server di Ricerca Globale",
     "Global State": "Stato Globale",
     "Help": "Aiuto",
-    "Home page": "Pagina home",
+    "Home page": "Home page",
     "Ignore": "Ignora",
     "Ignore Patterns": "Schemi Esclusione File",
     "Ignore Permissions": "Ignora Permessi",
     "Incoming Rate Limit (KiB/s)": "Limite Velocità in Ingresso (KiB/s)",
-    "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "Una configurazione incorretta potrebbe danneggiare il contenuto delle cartelle e rendere Syncthing inoperativo.",
+    "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "Incorrect configuration may damage your folder contents and render Syncthing inoperable.",
     "Introducer": "Introduttore",
     "Inversion of the given condition (i.e. do not exclude)": "Inversione della condizione indicata (ad es. non escludere)",
     "Keep Versions": "Versioni Mantenute",
@@ -93,7 +93,7 @@
     "Major Upgrade": "Aggiornamento principale",
     "Maximum Age": "Durata Massima",
     "Metadata Only": "Solo i Metadati",
-    "Minimum Free Disk Space": "Minimo spazio libero su disco",
+    "Minimum Free Disk Space": "Minimum Free Disk Space",
     "Move to top of queue": "Posiziona in cima alla coda",
     "Multi level wildcard (matches multiple directory levels)": "Metacarattere multi-livello (corrisponde alle cartelle e alle sotto-cartelle)",
     "Never": "Mai",
@@ -106,15 +106,15 @@
     "OK": "OK",
     "Off": "Disattiva",
     "Oldest First": "Prima il meno recente",
-    "Options": "Opzioni",
-    "Out of Sync": "Non sincronizzato",
+    "Options": "Options",
+    "Out of Sync": "Out of Sync",
     "Out of Sync Items": "Elementi Non Sincronizzati",
     "Outgoing Rate Limit (KiB/s)": "Limite Velocità in Uscita (KiB/s)",
     "Override Changes": "Ignora Modifiche",
     "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Percorso della cartella nel computer locale. Verrà creata se non esiste già. Il carattere tilde (~) può essere utilizzato come scorciatoia per",
     "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Percorso di salvataggio delle versioni (lasciare vuoto per utilizzare la cartella predefinita .stversions in questa cartella).",
-    "Pause": "Pausa",
-    "Paused": "In pausa",
+    "Pause": "Pause",
+    "Paused": "Paused",
     "Please consult the release notes before performing a major upgrade.": "Si prega di consultare le note di rilascio prima di eseguire un aggiornamento principale.",
     "Please wait": "Attendere prego",
     "Preview": "Anteprima",
@@ -122,17 +122,17 @@
     "Quick guide to supported patterns": "Guida veloce agli schemi supportati",
     "RAM Utilization": "Utilizzo RAM",
     "Random": "Casuale",
-    "Relayed via": "Reindirizzato tramite",
-    "Relays": "Servers di reindirizzamento",
+    "Relayed via": "Relayed via",
+    "Relays": "Relays",
     "Release Notes": "Note di rilascio",
-    "Remove": "Rimuovi",
+    "Remove": "Remove",
     "Rescan": "Riscansiona",
     "Rescan All": "Riscansiona Tutto",
     "Rescan Interval": "Intervallo Scansione",
     "Restart": "Riavvia",
     "Restart Needed": "Riavvio Necessario",
     "Restarting": "Riavvio",
-    "Resume": "Riprendi",
+    "Resume": "Resume",
     "Reused": "Riutilizzato",
     "Save": "Salva",
     "Scanning": "Scansione in corso",
@@ -157,7 +157,7 @@
     "Source Code": "Codice Sorgente",
     "Staggered File Versioning": "Controllo Versione Cadenzato",
     "Start Browser": "Avvia Browser",
-    "Statistics": "Statistiche",
+    "Statistics": "Statistics",
     "Stopped": "Fermato",
     "Support": "Supporto",
     "Sync Protocol Listen Addresses": "Indirizzi del Protocollo di Sincronizzazione",
@@ -180,18 +180,18 @@
     "The folder ID must be unique.": "L'ID della cartella dev'essere unico.",
     "The folder path cannot be blank.": "Il percorso della cartella non può essere vuoto.",
     "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "Vengono utilizzati i seguenti intervalli temporali: per la prima ora viene mantenuta una versione ogni 30 secondi, per il primo giorno viene mantenuta una versione ogni ora, per i primi 30 giorni viene mantenuta una versione al giorno, successivamente viene mantenuta una versione ogni settimana fino al periodo massimo impostato.",
-    "The following items could not be synchronized.": "Non è stato possibile sincronizzare i seguenti elementi",
+    "The following items could not be synchronized.": "The following items could not be synchronized.",
     "The maximum age must be a number and cannot be blank.": "La durata massima dev'essere un numero e non può essere vuoto.",
     "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "La durata massima di una versione (in giorni, imposta a 0 per mantenere le versioni per sempre).",
-    "The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).": "Lo spazio libero minimo su disco deve essere un numero non negativo tra 0 e 100 (inclusi)",
+    "The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).": "The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).",
     "The number of days must be a number and cannot be blank.": "Il numero di giorni deve essere un numero e non può essere vuoto.",
     "The number of days to keep files in the trash can. Zero means forever.": "Il numero di giorni per conservare i file nel cestino. Zero significa per sempre.",
     "The number of old versions to keep, per file.": "Il numero di vecchie versioni da mantenere, per file.",
     "The number of versions must be a number and cannot be blank.": "Il numero di versioni dev'essere un numero e non può essere vuoto.",
     "The path cannot be blank.": "Il percorso non può essere vuoto.",
-    "The rate limit must be a non-negative number (0: no limit)": "Il limite di banda deve essere un numero non negativo (da 0 a infinito)",
+    "The rate limit must be a non-negative number (0: no limit)": "The rate limit must be a non-negative number (0: no limit)",
     "The rescan interval must be a non-negative number of seconds.": "L'intervallo di scansione deve essere un numero superiore a zero secondi.",
-    "They are retried automatically and will be synced when the error is resolved.": "Verranno effettuati tentativi in automatico e verranno sincronizzati quando l'errore sarà risolto.",
+    "They are retried automatically and will be synced when the error is resolved.": "They are retried automatically and will be synced when the error is resolved.",
     "This is a major version upgrade.": "Questo è un aggiornamento di versione principale",
     "Trash Can File Versioning": "Controllo Versione con Cestino",
     "Unknown": "Sconosciuto",
@@ -212,7 +212,7 @@
     "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Quando aggiungi una nuova cartella, ricordati che gli ID vengono utilizzati per collegare le cartelle nei dispositivi. Distinguono maiuscole e minuscole e devono corrispondere esattamente su tutti i dispositivi.",
     "Yes": "Sì",
     "You must keep at least one version.": "È necessario mantenere almeno una versione.",
-    "days": "giorni",
+    "days": "days",
     "full documentation": "documentazione completa",
     "items": "elementi",
     "{%device%} wants to share folder \"{%folder%}\".": "{{device}} vuole condividere la cartella \"{{folder}}\"."

gui/assets/lang/lang-pt-BR.json

@@ -197,7 +197,7 @@
     "Unknown": "Desconhecida",
     "Unshared": "Não compartilhada",
     "Unused": "Não utilizado",
-    "Up to Date": "Em sincronia",
+    "Up to Date": "Sincronizada",
     "Updated": "Atualizado",
     "Upgrade": "Atualização",
     "Upgrade To {%version%}": "Atualizar para {{version}}",

gui/assets/lang/lang-uk.json

@@ -212,7 +212,7 @@
     "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Коли додаєте нову директорію, пам’ятайте, що ID цієї директорії використовується для того, щоб зв’язувати директорії разом між вузлами. Назви є чутливими до регістра та повинні співпадати точно між усіма вузлами.",
     "Yes": "Так",
     "You must keep at least one version.": "Ви повинні зберігати щонайменше одну версію.",
-    "days": "днів",
+    "days": "days",
     "full documentation": "повна документація",
     "items": "елементи",
     "{%device%} wants to share folder \"{%folder%}\".": "{{device}} хоче поділитися директорією \"{{folder}}\"."

gui/index.html

@@ -70,7 +70,7 @@
     </div>
   </nav>
 
-  <div class="container">
+  <div class="container" id="content">
 
     <!-- Panel: Restart Needed -->
 
@@ -171,7 +171,7 @@
         <div class="panel panel-warning">
           <div class="panel-heading"><h3 class="panel-title"><span class="fa fa-exclamation-circle"></span><span translate>Notice</span></h3></div>
           <div class="panel-body">
-            <p ng-repeat="err in errorList()"><small>{{err.time | date:"yyyy-MM-dd HH:mm:ss"}}:</small> {{friendlyDevices(err.error)}}</p>
+            <p ng-repeat="err in errorList()"><small>{{err.when | date:"yyyy-MM-dd HH:mm:ss"}}:</small> {{friendlyDevices(err.message)}}</p>
           </div>
           <div class="panel-footer">
             <button type="button" class="btn btn-sm btn-default pull-right" ng-click="clearErrors()">
@@ -190,18 +190,25 @@
       <!-- Folder list (top left) -->
 
       <div class="col-md-6">
+        <h3 translate>Folders</h3>
         <div class="panel-group" id="folders">
-        <div class="visible-xs"><h3><span translate>Folders</span></h3><hr></div>
           <div class="panel panel-default" ng-repeat="folder in folderList()">
             <div class="panel-heading" data-toggle="collapse" data-parent="#folders" href="#folder-{{$index}}" style="cursor: pointer">
               <div class="panel-progress" ng-show="folderStatus(folder) == 'syncing'" ng-attr-style="width: {{syncPercentage(folder.id)}}%"></div>
+              <div class="panel-progress" ng-show="folderStatus(folder) == 'scanning' && scanProgress[folder.id] != undefined" ng-attr-style="width: {{scanPercentage(folder.id)}}%"></div>
               <h3 class="panel-title">
                 <span class="fa fa-folder hidden-xs"></span>{{folder.id}}
                 <span class="pull-right text-{{folderClass(folder)}}" ng-switch="folderStatus(folder)">
                   <span ng-switch-when="unknown"><span class="hidden-xs" translate>Unknown</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="unshared"><span class="hidden-xs" translate>Unshared</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="stopped"><span class="hidden-xs" translate>Stopped</span><span class="visible-xs">&#9724;</span></span>
-                  <span ng-switch-when="scanning"><span class="hidden-xs" translate>Scanning</span><span class="visible-xs">&#9724;</span></span>
+                  <span ng-switch-when="scanning">
+                    <span class="hidden-xs" translate>Scanning</span>
+                    <span class="hidden-xs" ng-if="scanPercentage(folder.id) != undefined">
+                        ({{scanPercentage(folder.id)}}%)
+                    </span>
+                    <span class="visible-xs">&#9724;</span>
+                  </span>
                   <span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="syncing">
                     <span class="hidden-xs" translate>Syncing</span>
@@ -232,7 +239,7 @@
                       <td class="text-right">{{model[folder.id].localFiles | alwaysNumber}} <span translate>items</span>, ~{{model[folder.id].localBytes | binary}}B</td>
                     </tr>
                     <tr ng-if="model[folder.id].needFiles > 0">
-                      <th><span class="fa fa-fw fa-cloud-download"></span>&nbsp;<span translate>Out of Sync</span></th>
+                      <th><span class="fa fa-fw fa-cloud-download"></span>&nbsp;<span translate>Out of Sync Items</span></th>
                       <td class="text-right">
                         <a href="" ng-click="showNeed(folder.id)">{{model[folder.id].needFiles | alwaysNumber}} <span translate>items</span>, ~{{model[folder.id].needBytes | binary}}B</a>
                       </td>
@@ -341,7 +348,7 @@
       <!-- This device -->
 
       <div class="col-md-6">
-      <div class="visible-xs"><h3><span translate>Devices</span></h3><hr></div>
+        <h3 translate>Devices</h3>
         <div class="panel panel-default" ng-repeat="deviceCfg in [thisDevice()]">
           <div class="panel-heading" data-toggle="collapse" href="#device-this" style="cursor: pointer">
             <h3 class="panel-title">
@@ -372,15 +379,28 @@
                     <th><span class="fa fa-fw fa-tachometer"></span>&nbsp;<span translate>CPU Utilization</span></th>
                     <td class="text-right">{{system.cpuPercent | alwaysNumber | natural:1}}%</td>
                   </tr>
-                  <tr ng-if="system.extAnnounceOK != undefined && announceServersTotal > 0">
-                    <th><span class="fa fa-fw fa-bullhorn"></span>&nbsp;<span translate>Global Discovery</span></th>
+                  <tr ng-if="system.discoveryEnabled">
+                    <th><span class="fa fa-fw fa-map-signs"></span>&nbsp;<span translate>Discovery</span></th>
                     <td class="text-right">
-                      <span ng-if="announceServersFailed.length == 0" class="data text-success">
-                        <span>OK</span>
+                      <span ng-if="discoveryFailed.length == 0" class="data text-success">
+                        <span>{{discoveryTotal}}/{{discoveryTotal}}</span>
                       </span>
-                      <span ng-if="announceServersFailed.length != 0" class="data" ng-class="{'text-danger': announceServersFailed.length == announceServersTotal}">
-                        <span popover data-trigger="hover" data-placement="bottom" data-content="{{announceServersFailed.join('\n')}}">
-                          {{announceServersTotal-announceServersFailed.length}}/{{announceServersTotal}}
+                      <span ng-if="discoveryFailed.length != 0" class="data" ng-class="{'text-danger': discoveryFailed.length == discoveryTotal}">
+                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{discoveryFailed.join('<br>\n')}}">
+                          {{discoveryTotal-discoveryFailed.length}}/{{discoveryTotal}}
+                        </span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr ng-if="system.relaysEnabled">
+                    <th><span class="fa fa-fw fa-sitemap"></span>&nbsp;<span translate>Relays</span></th>
+                    <td class="text-right">
+                      <span ng-if="relaysFailed.length == 0" class="data text-success">
+                        <span>{{relaysTotal}}/{{relaysTotal}}</span>
+                      </span>
+                      <span ng-if="relaysFailed.length != 0" class="data" ng-class="{'text-danger': relaysFailed.length == relaysTotal}">
+                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{relaysFailed.join('<br>\n')}}">
+                          {{relaysTotal-relaysFailed.length}}/{{relaysTotal}}
                         </span>
                       </span>
                     </td>
@@ -412,6 +432,7 @@
                   <span ng-switch-when="syncing">
                     <span class="hidden-xs" translate>Syncing</span> ({{completion[deviceCfg.deviceID]._total | number:0}}%)
                   </span>
+                  <span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="disconnected"><span class="hidden-xs" translate>Disconnected</span><span class="visible-xs">&#9724;</span></span>
                   <span ng-switch-when="unused"><span class="hidden-xs" translate>Unused</span><span class="visible-xs">&#9724;</span></span>
                 </span>
@@ -421,16 +442,20 @@
               <div class="panel-body">
                 <table class="table table-condensed table-striped">
                   <tbody>
-                    <tr ng-if="connections[deviceCfg.deviceID]">
+                    <tr ng-if="connections[deviceCfg.deviceID].connected">
                       <th><span class="fa fa-fw fa-cloud-download"></span>&nbsp;<span translate>Download Rate</span></th>
                       <td class="text-right">{{connections[deviceCfg.deviceID].inbps | binary}}B/s ({{connections[deviceCfg.deviceID].inBytesTotal | binary}}B)</td>
                     </tr>
-                    <tr ng-if="connections[deviceCfg.deviceID]">
+                    <tr ng-if="connections[deviceCfg.deviceID].connected">
                       <th><span class="fa fa-fw fa-cloud-upload"></span>&nbsp;<span translate>Upload Rate</span></th>
                       <td class="text-right">{{connections[deviceCfg.deviceID].outbps | binary}}B/s ({{connections[deviceCfg.deviceID].outBytesTotal | binary}}B)</td>
                     </tr>
                     <tr>
-                      <th><span class="fa fa-fw fa-link"></span>&nbsp;<span translate>Address</span></th>
+                      <th>
+                        <span class="fa fa-fw fa-link"></span>
+                        <span translate ng-if="connections[deviceCfg.deviceID].type.indexOf('direct') == 0" >Address</span>
+                        <span translate ng-if="connections[deviceCfg.deviceID].type.indexOf('relay') == 0" >Relayed via</span>
+                      </th>
                       <td class="text-right">{{deviceAddr(deviceCfg)}}</td>
                     </tr>
                     <tr ng-if="deviceCfg.compression != 'metadata'">
@@ -444,11 +469,11 @@
                       <th><span class="fa fa-fw fa-thumbs-o-up"></span>&nbsp;<span translate>Introducer</span></th>
                       <td translate class="text-right">Yes</td>
                     </tr>
-                    <tr ng-if="connections[deviceCfg.deviceID]">
+                    <tr ng-if="connections[deviceCfg.deviceID].version">
                       <th><span class="fa fa-fw fa-tag"></span>&nbsp;<span translate>Version</span></th>
                       <td class="text-right">{{connections[deviceCfg.deviceID].clientVersion}}</td>
                     </tr>
-                    <tr ng-if="!connections[deviceCfg.deviceID]">
+                    <tr ng-if="!connections[deviceCfg.deviceID].connected">
                       <th><span class="fa fa-fw fa-eye"></span>&nbsp;<span translate>Last seen</span></th>
                       <td translate ng-if="!deviceStats[deviceCfg.deviceID].lastSeenDays || deviceStats[deviceCfg.deviceID].lastSeenDays >= 365" class="text-right">Never</td>
                       <td ng-if="deviceStats[deviceCfg.deviceID].lastSeenDays < 365" class="text-right">{{deviceStats[deviceCfg.deviceID].lastSeen | date:"yyyy-MM-dd HH:mm:ss"}}</td>
@@ -465,6 +490,12 @@
                   <button type="button" class="btn btn-sm btn-default" ng-click="editDevice(deviceCfg)">
                     <span class="fa fa-pencil"></span>&nbsp;<span translate>Edit</span>
                   </button>
+                  <button ng-if="!connections[deviceCfg.deviceID].paused" type="button" class="btn btn-sm btn-default" ng-click="pauseDevice(deviceCfg.deviceID)">
+                    <span class="fa fa-pause"></span>&nbsp;<span translate>Pause</span>
+                  </button>
+                  <button ng-if="connections[deviceCfg.deviceID].paused" type="button" class="btn btn-sm btn-default" ng-click="resumeDevice(deviceCfg.deviceID)">
+                    <span class="fa fa-play"></span>&nbsp;<span translate>Resume</span>
+                  </button>
                 </span>
                 <div class="clearfix"></div>
               </div>

gui/syncthing/core/aboutModalView.html

@@ -60,6 +60,7 @@
         <li class="auto-generated">Marc Laporte</li>
         <li class="auto-generated">Marc Pujol</li>
         <li class="auto-generated">Marcin Dziadus</li>
+        <li class="auto-generated">Mateon1</li>
         <li class="auto-generated">Matt Burke</li>
         <li class="auto-generated">Michael Jephcote</li>
         <li class="auto-generated">Michael Tilli</li>

gui/syncthing/core/eventService.js

@@ -63,6 +63,8 @@ angular.module('syncthing.core')
             DEVICE_DISCONNECTED:  'DeviceDisconnected',   // Generated each time a connection to a device has been terminated
             DEVICE_DISCOVERED:    'DeviceDiscovered',   // Emitted when a new device is discovered using local discovery
             DEVICE_REJECTED:      'DeviceRejected',   // Emitted when there is a connection from a device we are not configured to talk to
+            DEVICE_PAUSED:        'DevicePaused',   // Emitted when a device has been paused
+            DEVICE_RESUMED:       'DeviceResumed',   // Emitted when a device has been resumed
             DOWNLOAD_PROGRESS:    'DownloadProgress',   // Emitted during file downloads for each folder for each file
             FOLDER_COMPLETION:    'FolderCompletion',   //Emitted when the local or remote contents for a folder changes
             FOLDER_REJECTED:      'FolderRejected',   // Emitted when a device sends index information for a folder we do not have, or have but do not share with the device in question
@@ -76,6 +78,7 @@ angular.module('syncthing.core')
             STARTUP_COMPLETED:    'StartupCompleted',   // Emitted exactly once, when initialization is complete and Syncthing is ready to start exchanging data with other devices
             STATE_CHANGED:        'StateChanged',   // Emitted when a folder changes state
             FOLDER_ERRORS:        'FolderErrors',   // Emitted when a folder has errors preventing a full sync
+            FOLDER_SCAN_PROGRESS: 'FolderScanProgress',   // Emitted every ScanProgressIntervalS seconds, indicating how far into the scan it is at.
 
             start: function() {
                 $http.get(urlbase + '/events?limit=1')

gui/syncthing/core/syncthingController.js

@@ -48,6 +48,7 @@ angular.module('syncthing.core')
         $scope.failedCurrentPage = 1;
         $scope.failedCurrentFolder = undefined;
         $scope.failedPageSize = 10;
+        $scope.scanProgress = {};
 
         $scope.localStateTotal = {
             bytes: 0,
@@ -163,6 +164,12 @@ angular.module('syncthing.core')
                 if (data.to === 'syncing') {
                     $scope.failed[data.folder] = [];
                 }
+
+                // If a folder has started scanning, then any scan progress is
+                // also obsolete.
+                if (data.to === 'scanning') {
+                    delete $scope.scanProgress[data.folder];
+                }
             }
         });
 
@@ -171,7 +178,7 @@ angular.module('syncthing.core')
         });
 
         $scope.$on(Events.DEVICE_DISCONNECTED, function (event, arg) {
-            delete $scope.connections[arg.data.id];
+            $scope.connections[arg.data.id].connected = false;
             refreshDeviceStats();
         });
 
@@ -182,6 +189,7 @@ angular.module('syncthing.core')
                     outbps: 0,
                     inBytesTotal: 0,
                     outBytesTotal: 0,
+                    type: arg.data.type,
                     address: arg.data.addr
                 };
                 $scope.completion[arg.data.id] = {
@@ -214,6 +222,14 @@ angular.module('syncthing.core')
             $scope.deviceRejections[arg.data.device] = arg;
         });
 
+        $scope.$on(Events.DEVICE_PAUSED, function (event, arg) {
+            $scope.connections[arg.data.device].paused = true;
+        });
+
+        $scope.$on(Events.DEVICE_RESUMED, function (event, arg) {
+            $scope.connections[arg.data.device].paused = false;
+        });
+
         $scope.$on(Events.FOLDER_REJECTED, function (event, arg) {
             $scope.folderRejections[arg.data.folder + "-" + arg.data.device] = arg;
         });
@@ -301,6 +317,15 @@ angular.module('syncthing.core')
             $scope.failed[data.folder] = data.errors;
         });
 
+        $scope.$on(Events.FOLDER_SCAN_PROGRESS, function (event, arg) {
+            var data = arg.data;
+            $scope.scanProgress[data.folder] = {
+                current: data.current,
+                total: data.total
+            };
+            console.log("FolderScanProgress", data);
+        });
+
         $scope.emitHTTPError = function (data, status, headers, config) {
             $scope.$emit('HTTPError', {data: data, status: status, headers: headers, config: config});
         };
@@ -352,15 +377,26 @@ angular.module('syncthing.core')
             $http.get(urlbase + '/system/status').success(function (data) {
                 $scope.myID = data.myID;
                 $scope.system = data;
-                $scope.announceServersTotal = data.extAnnounceOK ? Object.keys(data.extAnnounceOK).length : 0;
-                var failed = [];
-                for (var server in data.extAnnounceOK) {
-                    if (!data.extAnnounceOK[server]) {
-                        failed.push(server);
+
+                $scope.discoveryTotal = data.discoveryMethods;
+                var discoveryFailed = [];
+                for (var disco in data.discoveryErrors) {
+                    if (data.discoveryErrors[disco]) {
+                        discoveryFailed.push(disco + ": " + data.discoveryErrors[disco]);
                     }
                 }
-                $scope.announceServersFailed = failed;
+                $scope.discoveryFailed = discoveryFailed;
 
+                var relaysFailed = [];
+                var relaysTotal = 0;
+                for (var relay in data.relayClientStatus) {
+                    if (!data.relayClientStatus[relay]) {
+                        relaysFailed.push(relay);
+                    }
+                    relaysTotal++;
+                }
+                $scope.relaysFailed = relaysFailed;
+                $scope.relaysTotal = relaysTotal;
 
                 console.log("refreshSystem", data);
             }).error($scope.emitHTTPError);
@@ -615,12 +651,28 @@ angular.module('syncthing.core')
             return Math.floor(pct);
         };
 
+        $scope.scanPercentage = function (folder) {
+            if (!$scope.scanProgress[folder]) {
+                return undefined;
+            }
+            var pct = 100 * $scope.scanProgress[folder].current / $scope.scanProgress[folder].total;
+            return Math.floor(pct);
+        }
+
         $scope.deviceStatus = function (deviceCfg) {
             if ($scope.deviceFolders(deviceCfg).length === 0) {
                 return 'unused';
             }
 
-            if ($scope.connections[deviceCfg.deviceID]) {
+            if (typeof $scope.connections[deviceCfg.deviceID] === 'undefined') {
+                return 'unknown';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].paused) {
+                return 'paused';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].connected) {
                 if ($scope.completion[deviceCfg.deviceID] && $scope.completion[deviceCfg.deviceID]._total === 100) {
                     return 'insync';
                 } else {
@@ -638,7 +690,15 @@ angular.module('syncthing.core')
                 return 'warning';
             }
 
-            if ($scope.connections[deviceCfg.deviceID]) {
+            if (typeof $scope.connections[deviceCfg.deviceID] === 'undefined') {
+                return 'info';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].paused) {
+                return 'default';
+            }
+
+            if ($scope.connections[deviceCfg.deviceID].connected) {
                 if ($scope.completion[deviceCfg.deviceID] && $scope.completion[deviceCfg.deviceID]._total === 100) {
                     return 'success';
                 } else {
@@ -652,7 +712,7 @@ angular.module('syncthing.core')
 
         $scope.deviceAddr = function (deviceCfg) {
             var conn = $scope.connections[deviceCfg.deviceID];
-            if (conn) {
+            if (conn && conn.connected) {
                 return conn.address;
             }
             return '?';
@@ -697,6 +757,14 @@ angular.module('syncthing.core')
             return device.deviceID.substr(0, 6);
         };
 
+        $scope.pauseDevice = function (device) {
+            $http.post(urlbase + "/system/pause?device=" + device);
+        };
+
+        $scope.resumeDevice = function (device) {
+            $http.post(urlbase + "/system/resume?device=" + device);
+        };
+
         $scope.editSettings = function () {
             // Make a working copy
             $scope.tmpOptions = angular.copy($scope.config.options);
@@ -975,13 +1043,16 @@ angular.module('syncthing.core')
         };
 
         $scope.errorList = function () {
+            if (!$scope.errors) {
+                return [];
+            }
             return $scope.errors.filter(function (e) {
-                return e.time > $scope.seenError;
+                return e.when > $scope.seenError;
             });
         };
 
         $scope.clearErrors = function () {
-            $scope.seenError = $scope.errors[$scope.errors.length - 1].time;
+            $scope.seenError = $scope.errors[$scope.errors.length - 1].when;
             $http.post(urlbase + '/system/error/clear');
         };
 

gui/syncthing/device/editDeviceModalView.html

@@ -13,9 +13,9 @@
             <label translate for="deviceID">Device ID</label>
             <input ng-if="!editingExisting" name="deviceID" id="deviceID" class="form-control text-monospace" type="text" ng-model="currentDevice.deviceID" required valid-deviceid list="discovery-list" />
             <datalist id="discovery-list" ng-if="!editingExisting">
-              <option ng-repeat="(id,address) in discovery" value="{{ id }}" />
+              <option ng-repeat="(id, data) in discovery" value="{{id}}" />
             </datalist>
-            <div ng-if="editingExisting" class="well well-sm text-monospace">{{currentDevice.deviceID}}</div>
+            <div ng-if="editingExisting" class="well well-sm text-monospace" select-on-click>{{currentDevice.deviceID}}</div>
             <p class="help-block">
               <span translate ng-if="deviceEditor.deviceID.$valid || deviceEditor.deviceID.$pristine">The device ID to enter here can be found in the "Edit > Show ID" dialog on the other device. Spaces and dashes are optional (ignored).</span>
               <span translate ng-show="!editingExisting && (deviceEditor.deviceID.$valid || deviceEditor.deviceID.$pristine)">When adding a new device, keep in mind that this device must be added on the other side too.</span>
@@ -32,7 +32,7 @@
           <div class="form-group">
             <label translate for="addresses">Addresses</label>
             <input ng-disabled="currentDevice.deviceID == myID" id="addresses" class="form-control" type="text" ng-model="currentDevice._addressesStr"></input>
-            <p translate class="help-block">Enter comma separated "ip:port" addresses or "dynamic" to perform automatic discovery of the address.</p>
+            <p translate class="help-block">Enter comma separated  ("tcp://ip:port", "tcp://host:port") addresses or "dynamic" to perform automatic discovery of the address.</p>
           </div>
           <div class="form-group">
             <label translate>Compression</label>

gui/syncthing/device/idqrModalView.html

@@ -1,4 +1,4 @@
 <modal id="idqr" status="info" icon="qrcode" title="{{'Device Identification' | translate}} - {{deviceName(thisDevice())}}" large="yes" close="yes">
-  <div class="well well-sm text-monospace text-center">{{myID}}</div>
+  <div class="well well-sm text-monospace text-center" select-on-click>{{myID}}</div>
   <img ng-if="myID" class="center-block img-thumbnail" ng-src="qr/?text={{myID}}"/>
 </modal>

gui/syncthing/folder/editFolderModalView.html

@@ -13,12 +13,11 @@
             <div class="col-md-12">
               <div class="form-group" ng-class="{'has-error': folderEditor.folderID.$invalid && folderEditor.folderID.$dirty}">
                 <label for="folderID"><span translate>Folder ID</span></label>
-                <input name="folderID" ng-readonly="editingExisting" id="folderID" class="form-control" type="text" ng-model="currentFolder.id" required unique-folder ng-pattern="/^[a-zA-Z0-9-_.]{1,64}$/"></input>
+                <input name="folderID" ng-readonly="editingExisting" id="folderID" class="form-control" type="text" ng-model="currentFolder.id" required unique-folder></input>
                 <p class="help-block">
                   <span translate ng-if="folderEditor.folderID.$valid || folderEditor.folderID.$pristine">Short identifier for the folder. Must be the same on all cluster devices.</span>
                   <span translate ng-if="folderEditor.folderID.$error.uniqueFolder">The folder ID must be unique.</span>
                   <span translate ng-if="folderEditor.folderID.$error.required && folderEditor.folderID.$dirty">The folder ID cannot be blank.</span>
-                  <span translate ng-if="folderEditor.folderID.$error.pattern && folderEditor.folderID.$dirty">The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.</span>
                 </p>
               </div>
               <div class="form-group" ng-class="{'has-error': folderEditor.folderPath.$invalid && folderEditor.folderPath.$dirty}">
@@ -98,7 +97,7 @@
                 <label translate for="trashcanClean">Clean out after</label>
                 <div class="input-group">
                   <input name="trashcanClean" id="trashcanClean" class="form-control text-right" type="number" ng-model="currentFolder.trashcanClean" required min="0"></input>
-                  <div class="input-group-addon">days</div>
+                  <div class="input-group-addon" translate>days</div>
                 </div>
                 <p class="help-block">
                   <span translate ng-if="folderEditor.trashcanClean.$valid || folderEditor.trashcanClean.$pristine">The number of days to keep files in the trash can. Zero means forever.</span>

gui/syncthing/settings/settingsModalView.html

@@ -108,7 +108,7 @@
 
             <div class="form-group">
               <label translate>API Key</label>
-              <div class="well well-sm text-monospace">{{tmpGUI.apiKey || "-"}}</div>
+              <div class="well well-sm text-monospace" select-on-click>{{tmpGUI.apiKey || "-"}}</div>
               <button type="button" class="btn btn-sm btn-default" ng-click="setAPIKey(tmpGUI)">
                 <span class="fa fa-repeat"></span>&nbsp;<span translate>Generate</span>
               </button>

gui/syncthing/usagereport/usageReportPreviewModalView.html

@@ -1,5 +1,7 @@
 <modal id="urPreview" status="success" icon="bar-chart" title="{{'Anonymous Usage Reporting' | translate}}" large="yes" close="yes" tabindex="-1">
   <p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
   <p translate translate-value-url="<a href=&quot;https://data.syncthing.net&quot; target=&quot;_blank&quot;>https://data.syncthing.net</a>">The aggregated statistics are publicly available at {%url%}.</p>
-  <pre><small>{{reportData | json}}</small></pre>
+  <form>
+  <textarea class="form-control" rows="20">{{reportData | json}}</textarea>
+  </form>
 </modal>

lib/beacon/beacon.go

@@ -6,7 +6,12 @@
 
 package beacon
 
-import "net"
+import (
+	"net"
+	stdsync "sync"
+
+	"github.com/thejerf/suture"
+)
 
 type recv struct {
 	data []byte
@@ -14,34 +19,30 @@ type recv struct {
 }
 
 type Interface interface {
+	suture.Service
 	Send(data []byte)
 	Recv() ([]byte, net.Addr)
+	Error() error
 }
 
 type readerFrom interface {
 	ReadFrom([]byte) (int, net.Addr, error)
 }
 
-func genericReader(conn readerFrom, outbox chan<- recv) {
-	bs := make([]byte, 65536)
-	for {
-		n, addr, err := conn.ReadFrom(bs)
-		if err != nil {
-			l.Warnln("multicast read:", err)
-			return
-		}
-		if debug {
-			l.Debugf("recv %d bytes from %s", n, addr)
-		}
+type errorHolder struct {
+	err error
+	mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
+}
 
-		c := make([]byte, n)
-		copy(c, bs)
-		select {
-		case outbox <- recv{c, addr}:
-		default:
-			if debug {
-				l.Debugln("dropping message")
-			}
-		}
-	}
+func (e *errorHolder) setError(err error) {
+	e.mut.Lock()
+	e.err = err
+	e.mut.Unlock()
+}
+
+func (e *errorHolder) Error() error {
+	e.mut.Lock()
+	err := e.err
+	e.mut.Unlock()
+	return err
 }

lib/beacon/broadcast.go

@@ -19,6 +19,8 @@ type Broadcast struct {
 	port   int
 	inbox  chan []byte
 	outbox chan recv
+	br     *broadcastReader
+	bw     *broadcastWriter
 }
 
 func NewBroadcast(port int) *Broadcast {
@@ -31,9 +33,7 @@ func NewBroadcast(port int) *Broadcast {
 			FailureBackoff:   60 * time.Second,
 			// Only log restarts in debug mode.
 			Log: func(line string) {
-				if debug {
-					l.Debugln(line)
-				}
+				l.Debugln(line)
 			},
 		}),
 		port:   port,
@@ -41,14 +41,16 @@ func NewBroadcast(port int) *Broadcast {
 		outbox: make(chan recv, 16),
 	}
 
-	b.Add(&broadcastReader{
+	b.br = &broadcastReader{
 		port:   port,
 		outbox: b.outbox,
-	})
-	b.Add(&broadcastWriter{
+	}
+	b.Add(b.br)
+	b.bw = &broadcastWriter{
 		port:  port,
 		inbox: b.inbox,
-	})
+	}
+	b.Add(b.bw)
 
 	return b
 }
@@ -62,38 +64,38 @@ func (b *Broadcast) Recv() ([]byte, net.Addr) {
 	return recv.data, recv.src
 }
 
+func (b *Broadcast) Error() error {
+	if err := b.br.Error(); err != nil {
+		return err
+	}
+	return b.bw.Error()
+}
+
 type broadcastWriter struct {
-	port   int
-	inbox  chan []byte
-	conn   *net.UDPConn
-	failed bool // Have we already logged a failure reason?
+	port  int
+	inbox chan []byte
+	conn  *net.UDPConn
+	errorHolder
 }
 
 func (w *broadcastWriter) Serve() {
-	if debug {
-		l.Debugln(w, "starting")
-		defer l.Debugln(w, "stopping")
-	}
+	l.Debugln(w, "starting")
+	defer l.Debugln(w, "stopping")
 
 	var err error
 	w.conn, err = net.ListenUDP("udp4", nil)
 	if err != nil {
-		if !w.failed {
-			l.Warnln("Local discovery over IPv4 unavailable:", err)
-			w.failed = true
-		}
+		l.Debugln(err)
+		w.setError(err)
 		return
 	}
 	defer w.conn.Close()
 
-	w.failed = false
-
 	for bs := range w.inbox {
 		addrs, err := net.InterfaceAddrs()
 		if err != nil {
-			if debug {
-				l.Debugln("Local discovery (broadcast writer):", err)
-			}
+			l.Debugln(err)
+			w.setError(err)
 			continue
 		}
 
@@ -110,35 +112,43 @@ func (w *broadcastWriter) Serve() {
 			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
 		}
 
-		if debug {
-			l.Debugln("addresses:", dsts)
-		}
+		l.Debugln("addresses:", dsts)
 
+		success := 0
 		for _, ip := range dsts {
 			dst := &net.UDPAddr{IP: ip, Port: w.port}
 
-			w.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
+			w.conn.SetWriteDeadline(time.Now().Add(time.Second))
 			_, err := w.conn.WriteTo(bs, dst)
+			w.conn.SetWriteDeadline(time.Time{})
+
 			if err, ok := err.(net.Error); ok && err.Timeout() {
 				// Write timeouts should not happen. We treat it as a fatal
 				// error on the socket.
-				l.Infoln("Local discovery (broadcast writer):", err)
-				w.failed = true
+				l.Debugln(err)
+				w.setError(err)
 				return
-			} else if err, ok := err.(net.Error); ok && err.Temporary() {
-				// A transient error. Lets hope for better luck in the future.
-				if debug {
-					l.Debugln(err)
-				}
-				continue
-			} else if err != nil {
-				// Some other error that we don't expect. Bail and retry.
-				l.Infoln("Local discovery (broadcast writer):", err)
-				w.failed = true
-				return
-			} else if debug {
-				l.Debugf("sent %d bytes to %s", len(bs), dst)
 			}
+
+			if err, ok := err.(net.Error); ok && err.Temporary() {
+				// A transient error. Lets hope for better luck in the future.
+				l.Debugln(err)
+				continue
+			}
+
+			if err != nil {
+				// Some other error that we don't expect. Bail and retry.
+				l.Debugln(err)
+				w.setError(err)
+				return
+			}
+
+			l.Debugf("sent %d bytes to %s", len(bs), dst)
+			success++
+		}
+
+		if success > 0 {
+			w.setError(nil)
 		}
 	}
 }
@@ -155,22 +165,18 @@ type broadcastReader struct {
 	port   int
 	outbox chan recv
 	conn   *net.UDPConn
-	failed bool
+	errorHolder
 }
 
 func (r *broadcastReader) Serve() {
-	if debug {
-		l.Debugln(r, "starting")
-		defer l.Debugln(r, "stopping")
-	}
+	l.Debugln(r, "starting")
+	defer l.Debugln(r, "stopping")
 
 	var err error
 	r.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
 	if err != nil {
-		if !r.failed {
-			l.Warnln("Local discovery over IPv4 unavailable:", err)
-			r.failed = true
-		}
+		l.Debugln(err)
+		r.setError(err)
 		return
 	}
 	defer r.conn.Close()
@@ -179,27 +185,21 @@ func (r *broadcastReader) Serve() {
 	for {
 		n, addr, err := r.conn.ReadFrom(bs)
 		if err != nil {
-			if !r.failed {
-				l.Infoln("Local discovery (broadcast reader):", err)
-				r.failed = true
-			}
+			l.Debugln(err)
+			r.setError(err)
 			return
 		}
 
-		r.failed = false
+		r.setError(nil)
 
-		if debug {
-			l.Debugf("recv %d bytes from %s", n, addr)
-		}
+		l.Debugf("recv %d bytes from %s", n, addr)
 
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
 		case r.outbox <- recv{c, addr}:
 		default:
-			if debug {
-				l.Debugln("dropping message")
-			}
+			l.Debugln("dropping message")
 		}
 	}
 

lib/beacon/debug.go

@@ -10,10 +10,13 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/logger"
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
+	l = logger.DefaultLogger.NewFacility("beacon", "Multicast and broadcast discovery")
 )
+
+func init() {
+	l.SetDebug("beacon", strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all")
+}

lib/beacon/multicast.go

@@ -8,101 +8,221 @@ package beacon
 
 import (
 	"errors"
+	"fmt"
 	"net"
+	"time"
 
+	"github.com/thejerf/suture"
 	"golang.org/x/net/ipv6"
 )
 
 type Multicast struct {
-	conn   *ipv6.PacketConn
+	*suture.Supervisor
 	addr   *net.UDPAddr
 	inbox  chan []byte
 	outbox chan recv
-	intfs  []net.Interface
+	mr     *multicastReader
+	mw     *multicastWriter
 }
 
-func NewMulticast(addr string) (*Multicast, error) {
-	gaddr, err := net.ResolveUDPAddr("udp6", addr)
-	if err != nil {
-		return nil, err
+func NewMulticast(addr string) *Multicast {
+	m := &Multicast{
+		Supervisor: suture.New("multicastBeacon", suture.Spec{
+			// Don't retry too frenetically: an error to open a socket or
+			// whatever is usually something that is either permanent or takes
+			// a while to get solved...
+			FailureThreshold: 2,
+			FailureBackoff:   60 * time.Second,
+			// Only log restarts in debug mode.
+			Log: func(line string) {
+				l.Debugln(line)
+			},
+		}),
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
 	}
 
-	conn, err := net.ListenPacket("udp6", addr)
+	m.mr = &multicastReader{
+		addr:   addr,
+		outbox: m.outbox,
+		stop:   make(chan struct{}),
+	}
+	m.Add(m.mr)
+
+	m.mw = &multicastWriter{
+		addr:  addr,
+		inbox: m.inbox,
+		stop:  make(chan struct{}),
+	}
+	m.Add(m.mw)
+
+	return m
+}
+
+func (m *Multicast) Send(data []byte) {
+	m.inbox <- data
+}
+
+func (m *Multicast) Recv() ([]byte, net.Addr) {
+	recv := <-m.outbox
+	return recv.data, recv.src
+}
+
+func (m *Multicast) Error() error {
+	if err := m.mr.Error(); err != nil {
+		return err
+	}
+	return m.mw.Error()
+}
+
+type multicastWriter struct {
+	addr  string
+	inbox <-chan []byte
+	errorHolder
+	stop chan struct{}
+}
+
+func (w *multicastWriter) Serve() {
+	l.Debugln(w, "starting")
+	defer l.Debugln(w, "stopping")
+
+	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
 	if err != nil {
-		return nil, err
+		l.Debugln(err)
+		w.setError(err)
+		return
+	}
+
+	conn, err := net.ListenPacket("udp6", ":0")
+	if err != nil {
+		l.Debugln(err)
+		w.setError(err)
+		return
+	}
+
+	pconn := ipv6.NewPacketConn(conn)
+
+	wcm := &ipv6.ControlMessage{
+		HopLimit: 1,
+	}
+
+	for bs := range w.inbox {
+		intfs, err := net.Interfaces()
+		if err != nil {
+			l.Debugln(err)
+			w.setError(err)
+			return
+		}
+
+		success := 0
+		for _, intf := range intfs {
+			wcm.IfIndex = intf.Index
+			pconn.SetWriteDeadline(time.Now().Add(time.Second))
+			_, err = pconn.WriteTo(bs, wcm, gaddr)
+			pconn.SetWriteDeadline(time.Time{})
+
+			if err != nil {
+				l.Debugln(err, "on write to", gaddr, intf.Name)
+				w.setError(err)
+				continue
+			}
+
+			l.Debugf("sent %d bytes to %v on %s", len(bs), gaddr, intf.Name)
+
+			success++
+		}
+
+		if success > 0 {
+			w.setError(nil)
+		} else {
+			l.Debugln(err)
+			w.setError(err)
+		}
+	}
+}
+
+func (w *multicastWriter) Stop() {
+	close(w.stop)
+}
+
+func (w *multicastWriter) String() string {
+	return fmt.Sprintf("multicastWriter@%p", w)
+}
+
+type multicastReader struct {
+	addr   string
+	outbox chan<- recv
+	errorHolder
+	stop chan struct{}
+}
+
+func (r *multicastReader) Serve() {
+	l.Debugln(r, "starting")
+	defer l.Debugln(r, "stopping")
+
+	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
+	if err != nil {
+		l.Debugln(err)
+		r.setError(err)
+		return
+	}
+
+	conn, err := net.ListenPacket("udp6", r.addr)
+	if err != nil {
+		l.Debugln(err)
+		r.setError(err)
+		return
 	}
 
 	intfs, err := net.Interfaces()
 	if err != nil {
-		return nil, err
+		l.Debugln(err)
+		r.setError(err)
+		return
 	}
 
-	p := ipv6.NewPacketConn(conn)
+	pconn := ipv6.NewPacketConn(conn)
 	joined := 0
 	for _, intf := range intfs {
-		err := p.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
-		if debug {
-			if err != nil {
-				l.Debugln("IPv6 join", intf.Name, "failed:", err)
-			} else {
-				l.Debugln("IPv6 join", intf.Name, "success")
-			}
+		err := pconn.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
+		if err != nil {
+			l.Debugln("IPv6 join", intf.Name, "failed:", err)
+		} else {
+			l.Debugln("IPv6 join", intf.Name, "success")
 		}
 		joined++
 	}
 
 	if joined == 0 {
-		return nil, errors.New("no multicast interfaces available")
+		l.Debugln("no multicast interfaces available")
+		r.setError(errors.New("no multicast interfaces available"))
+		return
 	}
 
-	b := &Multicast{
-		conn:   p,
-		addr:   gaddr,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-		intfs:  intfs,
-	}
+	bs := make([]byte, 65536)
+	for {
+		n, _, addr, err := pconn.ReadFrom(bs)
+		if err != nil {
+			l.Debugln(err)
+			r.setError(err)
+			continue
+		}
+		l.Debugf("recv %d bytes from %s", n, addr)
 
-	go genericReader(ipv6ReaderAdapter{b.conn}, b.outbox)
-	go b.writer()
-
-	return b, nil
-}
-
-func (b *Multicast) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Multicast) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Multicast) writer() {
-	wcm := &ipv6.ControlMessage{
-		HopLimit: 1,
-	}
-
-	for bs := range b.inbox {
-		for _, intf := range b.intfs {
-			wcm.IfIndex = intf.Index
-			_, err := b.conn.WriteTo(bs, wcm, b.addr)
-			if err != nil && debug {
-				l.Debugln(err, "on write to", b.addr)
-			} else if debug {
-				l.Debugf("sent %d bytes to %v on %s", len(bs), b.addr, intf.Name)
-			}
+		c := make([]byte, n)
+		copy(c, bs)
+		select {
+		case r.outbox <- recv{c, addr}:
+		default:
+			l.Debugln("dropping message")
 		}
 	}
 }
 
-// This makes ReadFrom on an *ipv6.PacketConn behave like ReadFrom on a
-// net.PacketConn.
-type ipv6ReaderAdapter struct {
-	c *ipv6.PacketConn
+func (r *multicastReader) Stop() {
+	close(r.stop)
 }
 
-func (i ipv6ReaderAdapter) ReadFrom(bs []byte) (int, net.Addr, error) {
-	n, _, src, err := i.c.ReadFrom(bs)
-	return n, src, err
+func (r *multicastReader) String() string {
+	return fmt.Sprintf("multicastReader@%p", r)
 }

lib/config/config.go

@@ -9,6 +9,7 @@ package config
 
 import (
 	"encoding/xml"
+	"fmt"
 	"io"
 	"math/rand"
 	"os"
@@ -19,17 +20,43 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"golang.org/x/crypto/bcrypt"
 )
 
 const (
 	OldestHandledVersion = 5
-	CurrentVersion       = 11
+	CurrentVersion       = 12
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
+var (
+	// DefaultDiscoveryServers should be substituted when the configuration
+	// contains <globalAnnounceServer>default</globalAnnounceServer>. This is
+	// done by the "consumer" of the configuration, as we don't want these
+	// saved to the config.
+	DefaultDiscoveryServers = []string{
+		"https://discovery-v4-1.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 194.126.249.5, Sweden
+		"https://discovery-v4-2.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 45.55.230.38, USA
+		"https://discovery-v4-3.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 128.199.95.124, Singapore
+		"https://discovery-v6-1.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 2001:470:28:4d6::5, Sweden
+		"https://discovery-v6-2.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 2604:a880:800:10::182:a001, USA
+		"https://discovery-v6-3.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 2400:6180:0:d0::d9:d001, Singapore
+	}
+
+	// DefaultDiscoveryServersIP is used by the usage reporting.
+	// XXX: Detect Android, and use this is we still don't have working DNS?
+	DefaultDiscoveryServersIP = []string{
+		"https://194.126.249.5/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA",
+		"https://45.55.230.38/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS",
+		"https://128.199.95.124/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4",
+		"https://[2001:470:28:4d6::5]/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA",
+		"https://[2604:a880:800:10::182:a001]/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS",
+		"https://[2400:6180:0:d0::d9:d001]/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4",
+	}
+)
+
 type Configuration struct {
 	Version        int                   `xml:"version,attr" json:"version"`
 	Folders        []FolderConfiguration `xml:"folder" json:"folders"`
@@ -67,20 +94,23 @@ func (cfg Configuration) Copy() Configuration {
 }
 
 type FolderConfiguration struct {
-	ID              string                      `xml:"id,attr" json:"id"`
-	RawPath         string                      `xml:"path,attr" json:"path"`
-	Devices         []FolderDeviceConfiguration `xml:"device" json:"devices"`
-	ReadOnly        bool                        `xml:"ro,attr" json:"readOnly"`
-	RescanIntervalS int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
-	IgnorePerms     bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
-	AutoNormalize   bool                        `xml:"autoNormalize,attr" json:"autoNormalize"`
-	MinDiskFreePct  float64                     `xml:"minDiskFreePct" json:"minDiskFreePct"`
-	Versioning      VersioningConfiguration     `xml:"versioning" json:"versioning"`
-	Copiers         int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
-	Pullers         int                         `xml:"pullers" json:"pullers"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
-	Hashers         int                         `xml:"hashers" json:"hashers"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
-	Order           PullOrder                   `xml:"order" json:"order"`
-	IgnoreDelete    bool                        `xml:"ignoreDelete" json:"ignoreDelete"`
+	ID                    string                      `xml:"id,attr" json:"id"`
+	RawPath               string                      `xml:"path,attr" json:"path"`
+	Devices               []FolderDeviceConfiguration `xml:"device" json:"devices"`
+	ReadOnly              bool                        `xml:"ro,attr" json:"readOnly"`
+	RescanIntervalS       int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
+	IgnorePerms           bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
+	AutoNormalize         bool                        `xml:"autoNormalize,attr" json:"autoNormalize"`
+	MinDiskFreePct        float64                     `xml:"minDiskFreePct" json:"minDiskFreePct"`
+	Versioning            VersioningConfiguration     `xml:"versioning" json:"versioning"`
+	Copiers               int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
+	Pullers               int                         `xml:"pullers" json:"pullers"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
+	Hashers               int                         `xml:"hashers" json:"hashers"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
+	Order                 PullOrder                   `xml:"order" json:"order"`
+	IgnoreDelete          bool                        `xml:"ignoreDelete" json:"ignoreDelete"`
+	ScanProgressIntervalS int                         `xml:"scanProgressIntervalS" json:"scanProgressIntervalS"` // Set to a negative value to disable. Value of 0 will get replaced with value of 2 (default value)
+	PullerSleepS          int                         `xml:"pullerSleepS" json:"pullerSleepS"`
+	PullerPauseS          int                         `xml:"pullerPauseS" json:"pullerPauseS"`
 
 	Invalid string `xml:"-" json:"invalid"` // Set at runtime when there is an error, not saved
 }
@@ -212,15 +242,19 @@ type FolderDeviceConfiguration struct {
 }
 
 type OptionsConfiguration struct {
-	ListenAddress           []string `xml:"listenAddress" json:"listenAddress" default:"0.0.0.0:22000"`
-	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"udp4://announce.syncthing.net:22026, udp6://announce-v6.syncthing.net:22026"`
+	ListenAddress           []string `xml:"listenAddress" json:"listenAddress" default:"tcp://0.0.0.0:22000"`
+	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default"`
 	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true"`
 	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true"`
-	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21025"`
-	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff32::5222]:21026"`
+	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027"`
+	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027"`
+	RelayServers            []string `xml:"relayServer" json:"relayServers" default:"dynamic+https://relays.syncthing.net"`
 	MaxSendKbps             int      `xml:"maxSendKbps" json:"maxSendKbps"`
 	MaxRecvKbps             int      `xml:"maxRecvKbps" json:"maxRecvKbps"`
 	ReconnectIntervalS      int      `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
+	RelaysEnabled           bool     `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
+	RelayReconnectIntervalM int      `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
+	RelayWithoutGlobalAnn   bool     `xml:"relayWithoutGlobalAnn" json:"relayWithoutGlobalAnn" default:"false"`
 	StartBrowser            bool     `xml:"startBrowser" json:"startBrowser" default:"true"`
 	UPnPEnabled             bool     `xml:"upnpEnabled" json:"upnpEnabled" default:"true"`
 	UPnPLeaseM              int      `xml:"upnpLeaseMinutes" json:"upnpLeaseMinutes" default:"60"`
@@ -228,6 +262,9 @@ type OptionsConfiguration struct {
 	UPnPTimeoutS            int      `xml:"upnpTimeoutSeconds" json:"upnpTimeoutSeconds" default:"10"`
 	URAccepted              int      `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
 	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
+	URURL                   string   `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
+	URPostInsecurely        bool     `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
+	URInitialDelayS         int      `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
 	RestartOnWakeup         bool     `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true"`
 	AutoUpgradeIntervalH    int      `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12"` // 0 for off
 	KeepTemporariesH        int      `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"`         // 0 for off
@@ -236,9 +273,9 @@ type OptionsConfiguration struct {
 	SymlinksEnabled         bool     `xml:"symlinksEnabled" json:"symlinksEnabled" default:"true"`
 	LimitBandwidthInLan     bool     `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
 	DatabaseBlockCacheMiB   int      `xml:"databaseBlockCacheMiB" json:"databaseBlockCacheMiB" default:"0"`
-	PingTimeoutS            int      `xml:"pingTimeoutS" json:"pingTimeoutS" default:"30"`
-	PingIdleTimeS           int      `xml:"pingIdleTimeS" json:"pingIdleTimeS" default:"60"`
 	MinHomeDiskFreePct      float64  `xml:"minHomeDiskFreePct" json:"minHomeDiskFreePct" default:"1"`
+	ReleasesURL             string   `xml:"releasesURL" json:"releasesURL" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=30"`
+	AlwaysLocalNets         []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
 }
 
 func (orig OptionsConfiguration) Copy() OptionsConfiguration {
@@ -346,6 +383,9 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		}
 	}
 
+	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
+	cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
+
 	if cfg.Version < OldestHandledVersion {
 		l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version)
 	}
@@ -369,6 +409,9 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 	if cfg.Version == 10 {
 		convertV10V11(cfg)
 	}
+	if cfg.Version == 11 {
+		convertV11V12(cfg)
+	}
 
 	// Hash old cleartext passwords
 	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
@@ -420,9 +463,6 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		cfg.Options.ReconnectIntervalS = 5
 	}
 
-	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
-	cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
-
 	if cfg.GUI.APIKey == "" {
 		cfg.GUI.APIKey = randomString(32)
 	}
@@ -467,6 +507,52 @@ func convertV10V11(cfg *Configuration) {
 	cfg.Version = 11
 }
 
+func convertV11V12(cfg *Configuration) {
+	// Change listen address schema
+	for i, addr := range cfg.Options.ListenAddress {
+		if len(addr) > 0 && !strings.HasPrefix(addr, "tcp://") {
+			cfg.Options.ListenAddress[i] = fmt.Sprintf("tcp://%s", addr)
+		}
+	}
+
+	for i, device := range cfg.Devices {
+		for j, addr := range device.Addresses {
+			if addr != "dynamic" && addr != "" {
+				cfg.Devices[i].Addresses[j] = fmt.Sprintf("tcp://%s", addr)
+			}
+		}
+	}
+
+	// Use new discovery server
+	var newDiscoServers []string
+	var useDefault bool
+	for _, addr := range cfg.Options.GlobalAnnServers {
+		if addr == "udp4://announce.syncthing.net:22026" {
+			useDefault = true
+		} else if addr == "udp6://announce-v6.syncthing.net:22026" {
+			useDefault = true
+		} else {
+			newDiscoServers = append(newDiscoServers, addr)
+		}
+	}
+	if useDefault {
+		newDiscoServers = append(newDiscoServers, "default")
+	}
+	cfg.Options.GlobalAnnServers = newDiscoServers
+
+	// Use new multicast group
+	if cfg.Options.LocalAnnMCAddr == "[ff32::5222]:21026" {
+		cfg.Options.LocalAnnMCAddr = "[ff12::8384]:21027"
+	}
+
+	// Use new local discovery port
+	if cfg.Options.LocalAnnPort == 21025 {
+		cfg.Options.LocalAnnPort = 21027
+	}
+
+	cfg.Version = 12
+}
+
 func convertV9V10(cfg *Configuration) {
 	// Enable auto normalization on existing folders.
 	for i := range cfg.Folders {

lib/config/config_test.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 var device1, device2, device3, device4 protocol.DeviceID
@@ -31,15 +31,19 @@ func init() {
 
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddress:           []string{"0.0.0.0:22000"},
-		GlobalAnnServers:        []string{"udp4://announce.syncthing.net:22026", "udp6://announce-v6.syncthing.net:22026"},
+		ListenAddress:           []string{"tcp://0.0.0.0:22000"},
+		GlobalAnnServers:        []string{"default"},
 		GlobalAnnEnabled:        true,
 		LocalAnnEnabled:         true,
-		LocalAnnPort:            21025,
-		LocalAnnMCAddr:          "[ff32::5222]:21026",
+		LocalAnnPort:            21027,
+		LocalAnnMCAddr:          "[ff12::8384]:21027",
+		RelayServers:            []string{"dynamic+https://relays.syncthing.net"},
 		MaxSendKbps:             0,
 		MaxRecvKbps:             0,
 		ReconnectIntervalS:      60,
+		RelaysEnabled:           true,
+		RelayReconnectIntervalM: 10,
+		RelayWithoutGlobalAnn:   false,
 		StartBrowser:            true,
 		UPnPEnabled:             true,
 		UPnPLeaseM:              60,
@@ -53,9 +57,11 @@ func TestDefaultValues(t *testing.T) {
 		SymlinksEnabled:         true,
 		LimitBandwidthInLan:     false,
 		DatabaseBlockCacheMiB:   0,
-		PingTimeoutS:            30,
-		PingIdleTimeS:           60,
 		MinHomeDiskFreePct:      1,
+		URURL:                   "https://data.syncthing.net/newdata",
+		URInitialDelayS:         1800,
+		URPostInsecurely:        false,
+		ReleasesURL:             "https://api.github.com/repos/syncthing/syncthing/releases?per_page=30",
 	}
 
 	cfg := New(device1)
@@ -100,13 +106,13 @@ func TestDeviceConfig(t *testing.T) {
 			{
 				DeviceID:    device1,
 				Name:        "node one",
-				Addresses:   []string{"a"},
+				Addresses:   []string{"tcp://a"},
 				Compression: protocol.CompressMetadata,
 			},
 			{
 				DeviceID:    device4,
 				Name:        "node two",
-				Addresses:   []string{"b"},
+				Addresses:   []string{"tcp://b"},
 				Compression: protocol.CompressMetadata,
 			},
 		}
@@ -142,15 +148,19 @@ func TestNoListenAddress(t *testing.T) {
 
 func TestOverriddenValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddress:           []string{":23000"},
+		ListenAddress:           []string{"tcp://:23000"},
 		GlobalAnnServers:        []string{"udp4://syncthing.nym.se:22026"},
 		GlobalAnnEnabled:        false,
 		LocalAnnEnabled:         false,
 		LocalAnnPort:            42123,
 		LocalAnnMCAddr:          "quux:3232",
+		RelayServers:            []string{"relay://123.123.123.123:1234", "relay://125.125.125.125:1255"},
 		MaxSendKbps:             1234,
 		MaxRecvKbps:             2341,
 		ReconnectIntervalS:      6000,
+		RelaysEnabled:           false,
+		RelayReconnectIntervalM: 20,
+		RelayWithoutGlobalAnn:   true,
 		StartBrowser:            false,
 		UPnPEnabled:             false,
 		UPnPLeaseM:              90,
@@ -164,9 +174,11 @@ func TestOverriddenValues(t *testing.T) {
 		SymlinksEnabled:         false,
 		LimitBandwidthInLan:     true,
 		DatabaseBlockCacheMiB:   42,
-		PingTimeoutS:            60,
-		PingIdleTimeS:           120,
 		MinHomeDiskFreePct:      5.2,
+		URURL:                   "https://localhost/newdata",
+		URInitialDelayS:         800,
+		URPostInsecurely:        true,
+		ReleasesURL:             "https://localhost/releases",
 	}
 
 	cfg, err := Load("testdata/overridenvalues.xml", device1)
@@ -255,15 +267,15 @@ func TestDeviceAddressesStatic(t *testing.T) {
 	expected := map[protocol.DeviceID]DeviceConfiguration{
 		device1: {
 			DeviceID:  device1,
-			Addresses: []string{"192.0.2.1", "192.0.2.2"},
+			Addresses: []string{"tcp://192.0.2.1", "tcp://192.0.2.2"},
 		},
 		device2: {
 			DeviceID:  device2,
-			Addresses: []string{"192.0.2.3:6070", "[2001:db8::42]:4242"},
+			Addresses: []string{"tcp://192.0.2.3:6070", "tcp://[2001:db8::42]:4242"},
 		},
 		device3: {
 			DeviceID:  device3,
-			Addresses: []string{"[2001:db8::44]:4444", "192.0.2.4:6090"},
+			Addresses: []string{"tcp://[2001:db8::44]:4444", "tcp://192.0.2.4:6090"},
 		},
 		device4: {
 			DeviceID:    device4,
@@ -330,12 +342,12 @@ func TestIssue1750(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if cfg.Options().ListenAddress[0] != ":23000" {
-		t.Errorf("%q != %q", cfg.Options().ListenAddress[0], ":23000")
+	if cfg.Options().ListenAddress[0] != "tcp://:23000" {
+		t.Errorf("%q != %q", cfg.Options().ListenAddress[0], "tcp://:23000")
 	}
 
-	if cfg.Options().ListenAddress[1] != ":23001" {
-		t.Errorf("%q != %q", cfg.Options().ListenAddress[1], ":23001")
+	if cfg.Options().ListenAddress[1] != "tcp://:23001" {
+		t.Errorf("%q != %q", cfg.Options().ListenAddress[1], "tcp://:23001")
 	}
 
 	if cfg.Options().GlobalAnnServers[0] != "udp4://syncthing.nym.se:22026" {

lib/config/debug.go

@@ -10,10 +10,13 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/logger"
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "config") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
+	l = logger.DefaultLogger.NewFacility("config", "Configuration loading and saving")
 )
+
+func init() {
+	l.SetDebug("config", strings.Contains(os.Getenv("STTRACE"), "config") || os.Getenv("STTRACE") == "all")
+}

lib/config/testdata/overridenvalues.xml

@@ -7,10 +7,15 @@
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>42123</localAnnouncePort>
         <localAnnounceMCAddr>quux:3232</localAnnounceMCAddr>
+        <relayServer>relay://123.123.123.123:1234</relayServer>
+        <relayServer>relay://125.125.125.125:1255</relayServer>
         <parallelRequests>32</parallelRequests>
         <maxSendKbps>1234</maxSendKbps>
         <maxRecvKbps>2341</maxRecvKbps>
         <reconnectionIntervalS>6000</reconnectionIntervalS>
+        <relaysEnabled>false</relaysEnabled>
+        <relayReconnectIntervalM>20</relayReconnectIntervalM>
+        <relayWithoutGlobalAnn>true</relayWithoutGlobalAnn>
         <startBrowser>false</startBrowser>
         <upnpEnabled>false</upnpEnabled>
         <upnpLeaseMinutes>90</upnpLeaseMinutes>
@@ -24,8 +29,10 @@
         <symlinksEnabled>false</symlinksEnabled>
         <limitBandwidthInLan>true</limitBandwidthInLan>
         <databaseBlockCacheMiB>42</databaseBlockCacheMiB>
-        <pingTimeoutS>60</pingTimeoutS>
-        <pingIdleTimeS>120</pingIdleTimeS>
         <minHomeDiskFreePct>5.2</minHomeDiskFreePct>
+        <urURL>https://localhost/newdata</urURL>
+        <urInitialDelayS>800</urInitialDelayS>
+        <urPostInsecurely>true</urPostInsecurely>
+        <releasesURL>https://localhost/releases</releasesURL>
     </options>
 </configuration>

lib/config/testdata/v12.xml

@@ -0,0 +1,13 @@
+<configuration version="12">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFreePct>1</minDiskFreePct>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/config/wrapper.go

@@ -9,9 +9,9 @@ package config
 import (
 	"os"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
@@ -60,10 +60,8 @@ type Wrapper struct {
 	deviceMap map[protocol.DeviceID]DeviceConfiguration
 	folderMap map[string]FolderConfiguration
 	replaces  chan Configuration
+	subs      []Committer
 	mut       sync.Mutex
-
-	subs []Committer
-	sMut sync.Mutex
 }
 
 // Wrap wraps an existing Configuration structure and ties it to a file on
@@ -73,7 +71,6 @@ func Wrap(path string, cfg Configuration) *Wrapper {
 		cfg:  cfg,
 		path: path,
 		mut:  sync.NewMutex(),
-		sMut: sync.NewMutex(),
 	}
 	w.replaces = make(chan Configuration)
 	return w
@@ -109,9 +106,24 @@ func (w *Wrapper) Stop() {
 // Subscribe registers the given handler to be called on any future
 // configuration changes.
 func (w *Wrapper) Subscribe(c Committer) {
-	w.sMut.Lock()
+	w.mut.Lock()
 	w.subs = append(w.subs, c)
-	w.sMut.Unlock()
+	w.mut.Unlock()
+}
+
+// Unsubscribe de-registers the given handler from any future calls to
+// configuration changes
+func (w *Wrapper) Unsubscribe(c Committer) {
+	w.mut.Lock()
+	for i := range w.subs {
+		if w.subs[i] == c {
+			copy(w.subs[i:], w.subs[i+1:])
+			w.subs[len(w.subs)-1] = nil
+			w.subs = w.subs[:len(w.subs)-1]
+			break
+		}
+	}
+	w.mut.Unlock()
 }
 
 // Raw returns the currently wrapped Configuration object.
@@ -130,13 +142,9 @@ func (w *Wrapper) replaceLocked(to Configuration) CommitResponse {
 	from := w.cfg
 
 	for _, sub := range w.subs {
-		if debug {
-			l.Debugln(sub, "verifying configuration")
-		}
+		l.Debugln(sub, "verifying configuration")
 		if err := sub.VerifyConfiguration(from, to); err != nil {
-			if debug {
-				l.Debugln(sub, "rejected config:", err)
-			}
+			l.Debugln(sub, "rejected config:", err)
 			return CommitResponse{
 				ValidationError: err,
 			}
@@ -145,14 +153,10 @@ func (w *Wrapper) replaceLocked(to Configuration) CommitResponse {
 
 	allOk := true
 	for _, sub := range w.subs {
-		if debug {
-			l.Debugln(sub, "committing configuration")
-		}
+		l.Debugln(sub, "committing configuration")
 		ok := sub.CommitConfiguration(from, to)
 		if !ok {
-			if debug {
-				l.Debugln(sub, "requires restart")
-			}
+			l.Debugln(sub, "requires restart")
 			allOk = false
 		}
 	}
@@ -302,3 +306,15 @@ func (w *Wrapper) Save() error {
 	events.Default.Log(events.ConfigSaved, w.cfg)
 	return nil
 }
+
+func (w *Wrapper) GlobalDiscoveryServers() []string {
+	var servers []string
+	for _, srv := range w.cfg.Options.GlobalAnnServers {
+		if srv == "default" {
+			servers = append(servers, DefaultDiscoveryServers...)
+		} else {
+			servers = append(servers, srv)
+		}
+	}
+	return uniqueStrings(servers)
+}

lib/connections/connections.go

@@ -0,0 +1,461 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package connections
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/juju/ratelimit"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/relay"
+	"github.com/syncthing/syncthing/lib/relay/client"
+
+	"github.com/thejerf/suture"
+)
+
+type DialerFactory func(*url.URL, *tls.Config) (*tls.Conn, error)
+type ListenerFactory func(*url.URL, *tls.Config, chan<- model.IntermediateConnection)
+
+var (
+	dialers   = make(map[string]DialerFactory, 0)
+	listeners = make(map[string]ListenerFactory, 0)
+)
+
+type Model interface {
+	protocol.Model
+	AddConnection(conn model.Connection)
+	ConnectedTo(remoteID protocol.DeviceID) bool
+	IsPaused(remoteID protocol.DeviceID) bool
+}
+
+// The connection service listens on TLS and dials configured unconnected
+// devices. Successful connections are handed to the model.
+type connectionSvc struct {
+	*suture.Supervisor
+	cfg                  *config.Wrapper
+	myID                 protocol.DeviceID
+	model                Model
+	tlsCfg               *tls.Config
+	discoverer           discover.Finder
+	conns                chan model.IntermediateConnection
+	relaySvc             *relay.Svc
+	bepProtocolName      string
+	tlsDefaultCommonName string
+	lans                 []*net.IPNet
+	writeRateLimit       *ratelimit.Bucket
+	readRateLimit        *ratelimit.Bucket
+
+	lastRelayCheck map[protocol.DeviceID]time.Time
+
+	mut           sync.RWMutex
+	connType      map[protocol.DeviceID]model.ConnectionType
+	relaysEnabled bool
+}
+
+func NewConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, relaySvc *relay.Svc,
+	bepProtocolName string, tlsDefaultCommonName string, lans []*net.IPNet) suture.Service {
+	svc := &connectionSvc{
+		Supervisor:           suture.NewSimple("connectionSvc"),
+		cfg:                  cfg,
+		myID:                 myID,
+		model:                mdl,
+		tlsCfg:               tlsCfg,
+		discoverer:           discoverer,
+		relaySvc:             relaySvc,
+		conns:                make(chan model.IntermediateConnection),
+		bepProtocolName:      bepProtocolName,
+		tlsDefaultCommonName: tlsDefaultCommonName,
+		lans:                 lans,
+
+		connType:       make(map[protocol.DeviceID]model.ConnectionType),
+		relaysEnabled:  cfg.Options().RelaysEnabled,
+		lastRelayCheck: make(map[protocol.DeviceID]time.Time),
+	}
+	cfg.Subscribe(svc)
+
+	if svc.cfg.Options().MaxSendKbps > 0 {
+		svc.writeRateLimit = ratelimit.NewBucketWithRate(float64(1000*svc.cfg.Options().MaxSendKbps), int64(5*1000*svc.cfg.Options().MaxSendKbps))
+	}
+	if svc.cfg.Options().MaxRecvKbps > 0 {
+		svc.readRateLimit = ratelimit.NewBucketWithRate(float64(1000*svc.cfg.Options().MaxRecvKbps), int64(5*1000*svc.cfg.Options().MaxRecvKbps))
+	}
+
+	// There are several moving parts here; one routine per listening address
+	// to handle incoming connections, one routine to periodically attempt
+	// outgoing connections, one routine to the the common handling
+	// regardless of whether the connection was incoming or outgoing.
+	// Furthermore, a relay service which handles incoming requests to connect
+	// via the relays.
+	//
+	// TODO: Clean shutdown, and/or handling config changes on the fly. We
+	// partly do this now - new devices and addresses will be picked up, but
+	// not new listen addresses and we don't support disconnecting devices
+	// that are removed and so on...
+
+	svc.Add(serviceFunc(svc.connect))
+	for _, addr := range svc.cfg.Options().ListenAddress {
+		uri, err := url.Parse(addr)
+		if err != nil {
+			l.Infoln("Failed to parse listen address:", addr, err)
+			continue
+		}
+
+		listener, ok := listeners[uri.Scheme]
+		if !ok {
+			l.Infoln("Unknown listen address scheme:", uri.String())
+			continue
+		}
+
+		l.Debugln("listening on", uri)
+
+		svc.Add(serviceFunc(func() {
+			listener(uri, svc.tlsCfg, svc.conns)
+		}))
+	}
+	svc.Add(serviceFunc(svc.handle))
+
+	if svc.relaySvc != nil {
+		svc.Add(serviceFunc(svc.acceptRelayConns))
+	}
+
+	return svc
+}
+
+func (s *connectionSvc) handle() {
+next:
+	for c := range s.conns {
+		cs := c.Conn.ConnectionState()
+
+		// We should have negotiated the next level protocol "bep/1.0" as part
+		// of the TLS handshake. Unfortunately this can't be a hard error,
+		// because there are implementations out there that don't support
+		// protocol negotiation (iOS for one...).
+		if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != s.bepProtocolName {
+			l.Infof("Peer %s did not negotiate bep/1.0", c.Conn.RemoteAddr())
+		}
+
+		// We should have received exactly one certificate from the other
+		// side. If we didn't, they don't have a device ID and we drop the
+		// connection.
+		certs := cs.PeerCertificates
+		if cl := len(certs); cl != 1 {
+			l.Infof("Got peer certificate list of length %d != 1 from %s; protocol error", cl, c.Conn.RemoteAddr())
+			c.Conn.Close()
+			continue
+		}
+		remoteCert := certs[0]
+		remoteID := protocol.NewDeviceID(remoteCert.Raw)
+
+		// The device ID should not be that of ourselves. It can happen
+		// though, especially in the presence of NAT hairpinning, multiple
+		// clients between the same NAT gateway, and global discovery.
+		if remoteID == s.myID {
+			l.Infof("Connected to myself (%s) - should not happen", remoteID)
+			c.Conn.Close()
+			continue
+		}
+
+		// If we have a relay connection, and the new incoming connection is
+		// not a relay connection, we should drop that, and prefer the this one.
+		s.mut.RLock()
+		ct, ok := s.connType[remoteID]
+		s.mut.RUnlock()
+		if ok && !ct.IsDirect() && c.Type.IsDirect() {
+			l.Debugln("Switching connections", remoteID)
+			s.model.Close(remoteID, fmt.Errorf("switching connections"))
+		} else if s.model.ConnectedTo(remoteID) {
+			// We should not already be connected to the other party. TODO: This
+			// could use some better handling. If the old connection is dead but
+			// hasn't timed out yet we may want to drop *that* connection and keep
+			// this one. But in case we are two devices connecting to each other
+			// in parallel we don't want to do that or we end up with no
+			// connections still established...
+			l.Infof("Connected to already connected device (%s)", remoteID)
+			c.Conn.Close()
+			continue
+		} else if s.model.IsPaused(remoteID) {
+			l.Infof("Connection from paused device (%s)", remoteID)
+			c.Conn.Close()
+			continue
+		}
+
+		for deviceID, deviceCfg := range s.cfg.Devices() {
+			if deviceID == remoteID {
+				// Verify the name on the certificate. By default we set it to
+				// "syncthing" when generating, but the user may have replaced
+				// the certificate and used another name.
+				certName := deviceCfg.CertName
+				if certName == "" {
+					certName = s.tlsDefaultCommonName
+				}
+				err := remoteCert.VerifyHostname(certName)
+				if err != nil {
+					// Incorrect certificate name is something the user most
+					// likely wants to know about, since it's an advanced
+					// config. Warn instead of Info.
+					l.Warnf("Bad certificate from %s (%v): %v", remoteID, c.Conn.RemoteAddr(), err)
+					c.Conn.Close()
+					continue next
+				}
+
+				// If rate limiting is set, and based on the address we should
+				// limit the connection, then we wrap it in a limiter.
+
+				limit := s.shouldLimit(c.Conn.RemoteAddr())
+
+				wr := io.Writer(c.Conn)
+				if limit && s.writeRateLimit != nil {
+					wr = NewWriteLimiter(c.Conn, s.writeRateLimit)
+				}
+
+				rd := io.Reader(c.Conn)
+				if limit && s.readRateLimit != nil {
+					rd = NewReadLimiter(c.Conn, s.readRateLimit)
+				}
+
+				name := fmt.Sprintf("%s-%s (%s)", c.Conn.LocalAddr(), c.Conn.RemoteAddr(), c.Type)
+				protoConn := protocol.NewConnection(remoteID, rd, wr, s.model, name, deviceCfg.Compression)
+
+				l.Infof("Established secure connection to %s at %s", remoteID, name)
+				l.Debugf("cipher suite: %04X in lan: %t", c.Conn.ConnectionState().CipherSuite, !limit)
+
+				s.model.AddConnection(model.Connection{
+					c.Conn,
+					protoConn,
+					c.Type,
+				})
+				s.mut.Lock()
+				s.connType[remoteID] = c.Type
+				s.mut.Unlock()
+				continue next
+			}
+		}
+
+		if !s.cfg.IgnoredDevice(remoteID) {
+			events.Default.Log(events.DeviceRejected, map[string]string{
+				"device":  remoteID.String(),
+				"address": c.Conn.RemoteAddr().String(),
+			})
+		}
+
+		l.Infof("Connection from %s (%s) with ignored device ID %s", c.Conn.RemoteAddr(), c.Type, remoteID)
+		c.Conn.Close()
+	}
+}
+
+func (s *connectionSvc) connect() {
+	delay := time.Second
+	for {
+	nextDevice:
+		for deviceID, deviceCfg := range s.cfg.Devices() {
+			if deviceID == s.myID {
+				continue
+			}
+
+			if s.model.IsPaused(deviceID) {
+				continue
+			}
+
+			connected := s.model.ConnectedTo(deviceID)
+
+			s.mut.RLock()
+			ct, ok := s.connType[deviceID]
+			relaysEnabled := s.relaysEnabled
+			s.mut.RUnlock()
+			if connected && ok && ct.IsDirect() {
+				continue
+			}
+
+			var addrs []string
+			var relays []discover.Relay
+			for _, addr := range deviceCfg.Addresses {
+				if addr == "dynamic" {
+					if s.discoverer != nil {
+						if t, r, err := s.discoverer.Lookup(deviceID); err == nil {
+							addrs = append(addrs, t...)
+							relays = append(relays, r...)
+						}
+					}
+				} else {
+					addrs = append(addrs, addr)
+				}
+			}
+
+			for _, addr := range addrs {
+				uri, err := url.Parse(addr)
+				if err != nil {
+					l.Infoln("Failed to parse connection url:", addr, err)
+					continue
+				}
+
+				dialer, ok := dialers[uri.Scheme]
+				if !ok {
+					l.Infoln("Unknown address schema", uri)
+					continue
+				}
+
+				l.Debugln("dial", deviceCfg.DeviceID, uri)
+				conn, err := dialer(uri, s.tlsCfg)
+				if err != nil {
+					l.Debugln("dial failed", deviceCfg.DeviceID, uri, err)
+					continue
+				}
+
+				if connected {
+					s.model.Close(deviceID, fmt.Errorf("switching connections"))
+				}
+
+				s.conns <- model.IntermediateConnection{
+					conn, model.ConnectionTypeDirectDial,
+				}
+				continue nextDevice
+			}
+
+			// Only connect via relays if not already connected
+			// Also, do not set lastRelayCheck time if we have no relays,
+			// as otherwise when we do discover relays, we might have to
+			// wait up to RelayReconnectIntervalM to connect again.
+			// Also, do not try relays if we are explicitly told not to.
+			if connected || len(relays) == 0 || !relaysEnabled {
+				continue nextDevice
+			}
+
+			reconIntv := time.Duration(s.cfg.Options().RelayReconnectIntervalM) * time.Minute
+			if last, ok := s.lastRelayCheck[deviceID]; ok && time.Since(last) < reconIntv {
+				l.Debugln("Skipping connecting via relay to", deviceID, "last checked at", last)
+				continue nextDevice
+			} else {
+				l.Debugln("Trying relay connections to", deviceID, relays)
+			}
+
+			s.lastRelayCheck[deviceID] = time.Now()
+
+			for _, addr := range relays {
+				uri, err := url.Parse(addr.URL)
+				if err != nil {
+					l.Infoln("Failed to parse relay connection url:", addr, err)
+					continue
+				}
+
+				inv, err := client.GetInvitationFromRelay(uri, deviceID, s.tlsCfg.Certificates)
+				if err != nil {
+					l.Debugf("Failed to get invitation for %s from %s: %v", deviceID, uri, err)
+					continue
+				} else {
+					l.Debugln("Succesfully retrieved relay invitation", inv, "from", uri)
+				}
+
+				conn, err := client.JoinSession(inv)
+				if err != nil {
+					l.Debugf("Failed to join relay session %s: %v", inv, err)
+					continue
+				} else {
+					l.Debugln("Sucessfully joined relay session", inv)
+				}
+
+				err = osutil.SetTCPOptions(conn.(*net.TCPConn))
+				if err != nil {
+					l.Infoln(err)
+				}
+
+				var tc *tls.Conn
+
+				if inv.ServerSocket {
+					tc = tls.Server(conn, s.tlsCfg)
+				} else {
+					tc = tls.Client(conn, s.tlsCfg)
+				}
+				err = tc.Handshake()
+				if err != nil {
+					l.Infof("TLS handshake (BEP/relay %s): %v", inv, err)
+					tc.Close()
+					continue
+				}
+				s.conns <- model.IntermediateConnection{
+					tc, model.ConnectionTypeRelayDial,
+				}
+				continue nextDevice
+			}
+		}
+
+		time.Sleep(delay)
+		delay *= 2
+		if maxD := time.Duration(s.cfg.Options().ReconnectIntervalS) * time.Second; delay > maxD {
+			delay = maxD
+		}
+	}
+}
+
+func (s *connectionSvc) acceptRelayConns() {
+	for {
+		conn := s.relaySvc.Accept()
+		s.conns <- model.IntermediateConnection{
+			Conn: conn,
+			Type: model.ConnectionTypeRelayAccept,
+		}
+	}
+}
+
+func (s *connectionSvc) shouldLimit(addr net.Addr) bool {
+	if s.cfg.Options().LimitBandwidthInLan {
+		return true
+	}
+
+	tcpaddr, ok := addr.(*net.TCPAddr)
+	if !ok {
+		return true
+	}
+	for _, lan := range s.lans {
+		if lan.Contains(tcpaddr.IP) {
+			return false
+		}
+	}
+	return !tcpaddr.IP.IsLoopback()
+}
+
+func (s *connectionSvc) VerifyConfiguration(from, to config.Configuration) error {
+	return nil
+}
+
+func (s *connectionSvc) CommitConfiguration(from, to config.Configuration) bool {
+	s.mut.Lock()
+	s.relaysEnabled = to.Options.RelaysEnabled
+	s.mut.Unlock()
+
+	// We require a restart if a device as been removed.
+
+	newDevices := make(map[protocol.DeviceID]bool, len(to.Devices))
+	for _, dev := range to.Devices {
+		newDevices[dev.DeviceID] = true
+	}
+
+	for _, dev := range from.Devices {
+		if !newDevices[dev.DeviceID] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// serviceFunc wraps a function to create a suture.Service without stop
+// functionality.
+type serviceFunc func()
+
+func (f serviceFunc) Serve() { f() }
+func (f serviceFunc) Stop()  {}

lib/connections/connections_tcp.go

@@ -0,0 +1,99 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package connections
+
+import (
+	"crypto/tls"
+	"net"
+	"net/url"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/osutil"
+)
+
+func init() {
+	dialers["tcp"] = tcpDialer
+	listeners["tcp"] = tcpListener
+}
+
+func tcpDialer(uri *url.URL, tlsCfg *tls.Config) (*tls.Conn, error) {
+	host, port, err := net.SplitHostPort(uri.Host)
+	if err != nil && strings.HasPrefix(err.Error(), "missing port") {
+		// addr is on the form "1.2.3.4"
+		uri.Host = net.JoinHostPort(uri.Host, "22000")
+	} else if err == nil && port == "" {
+		// addr is on the form "1.2.3.4:"
+		uri.Host = net.JoinHostPort(host, "22000")
+	}
+
+	raddr, err := net.ResolveTCPAddr("tcp", uri.Host)
+	if err != nil {
+		l.Debugln(err)
+		return nil, err
+	}
+
+	conn, err := net.DialTCP("tcp", nil, raddr)
+	if err != nil {
+		l.Debugln(err)
+		return nil, err
+	}
+
+	err = osutil.SetTCPOptions(conn)
+	if err != nil {
+		l.Infoln(err)
+	}
+
+	tc := tls.Client(conn, tlsCfg)
+	err = tc.Handshake()
+	if err != nil {
+		tc.Close()
+		return nil, err
+	}
+
+	return tc, nil
+}
+
+func tcpListener(uri *url.URL, tlsCfg *tls.Config, conns chan<- model.IntermediateConnection) {
+	tcaddr, err := net.ResolveTCPAddr("tcp", uri.Host)
+	if err != nil {
+		l.Fatalln("listen (BEP/tcp):", err)
+		return
+	}
+	listener, err := net.ListenTCP("tcp", tcaddr)
+	if err != nil {
+		l.Fatalln("listen (BEP/tcp):", err)
+		return
+	}
+
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			l.Warnln("Accepting connection (BEP/tcp):", err)
+			continue
+		}
+
+		l.Debugln("connect from", conn.RemoteAddr())
+
+		err = osutil.SetTCPOptions(conn.(*net.TCPConn))
+		if err != nil {
+			l.Infoln(err)
+		}
+
+		tc := tls.Server(conn, tlsCfg)
+		err = tc.Handshake()
+		if err != nil {
+			l.Infoln("TLS handshake (BEP/tcp):", err)
+			tc.Close()
+			continue
+		}
+
+		conns <- model.IntermediateConnection{
+			tc, model.ConnectionTypeDirectAccept,
+		}
+	}
+}

lib/connections/debug.go

@@ -0,0 +1,22 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package connections
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/logger"
+)
+
+var (
+	l = logger.DefaultLogger.NewFacility("connections", "Connection handling")
+)
+
+func init() {
+	l.SetDebug("connections", strings.Contains(os.Getenv("STTRACE"), "connections") || os.Getenv("STTRACE") == "all")
+}

lib/connections/limitedreader.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-package main
+package connections
 
 import (
 	"io"
@@ -12,13 +12,20 @@ import (
 	"github.com/juju/ratelimit"
 )
 
-type limitedReader struct {
-	r      io.Reader
+type LimitedReader struct {
+	reader io.Reader
 	bucket *ratelimit.Bucket
 }
 
-func (r *limitedReader) Read(buf []byte) (int, error) {
-	n, err := r.r.Read(buf)
+func NewReadLimiter(r io.Reader, b *ratelimit.Bucket) *LimitedReader {
+	return &LimitedReader{
+		reader: r,
+		bucket: b,
+	}
+}
+
+func (r *LimitedReader) Read(buf []byte) (int, error) {
+	n, err := r.reader.Read(buf)
 	if r.bucket != nil {
 		r.bucket.Wait(int64(n))
 	}

lib/connections/limitedwriter.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-package main
+package connections
 
 import (
 	"io"
@@ -12,14 +12,21 @@ import (
 	"github.com/juju/ratelimit"
 )
 
-type limitedWriter struct {
-	w      io.Writer
+type LimitedWriter struct {
+	writer io.Writer
 	bucket *ratelimit.Bucket
 }
 
-func (w *limitedWriter) Write(buf []byte) (int, error) {
+func NewWriteLimiter(w io.Writer, b *ratelimit.Bucket) *LimitedWriter {
+	return &LimitedWriter{
+		writer: w,
+		bucket: b,
+	}
+}
+
+func (w *LimitedWriter) Write(buf []byte) (int, error) {
 	if w.bucket != nil {
 		w.bucket.Wait(int64(len(buf)))
 	}
-	return w.w.Write(buf)
+	return w.writer.Write(buf)
 }

lib/db/blockmap.go

@@ -16,12 +16,9 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"sort"
 
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/osutil"
-	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/protocol"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -112,48 +109,21 @@ func (m *BlockMap) blockKey(hash []byte, file string) []byte {
 }
 
 type BlockFinder struct {
-	db      *leveldb.DB
-	folders []string
-	mut     sync.RWMutex
+	db *leveldb.DB
 }
 
-func NewBlockFinder(db *leveldb.DB, cfg *config.Wrapper) *BlockFinder {
+func NewBlockFinder(db *leveldb.DB) *BlockFinder {
 	if blockFinder != nil {
 		return blockFinder
 	}
 
 	f := &BlockFinder{
-		db:  db,
-		mut: sync.NewRWMutex(),
+		db: db,
 	}
 
-	f.CommitConfiguration(config.Configuration{}, cfg.Raw())
-	cfg.Subscribe(f)
-
 	return f
 }
 
-// VerifyConfiguration implementes the config.Committer interface
-func (f *BlockFinder) VerifyConfiguration(from, to config.Configuration) error {
-	return nil
-}
-
-// CommitConfiguration implementes the config.Committer interface
-func (f *BlockFinder) CommitConfiguration(from, to config.Configuration) bool {
-	folders := make([]string, len(to.Folders))
-	for i, folder := range to.Folders {
-		folders[i] = folder.ID
-	}
-
-	sort.Strings(folders)
-
-	f.mut.Lock()
-	f.folders = folders
-	f.mut.Unlock()
-
-	return true
-}
-
 func (f *BlockFinder) String() string {
 	return fmt.Sprintf("BlockFinder@%p", f)
 }
@@ -163,10 +133,7 @@ func (f *BlockFinder) String() string {
 // they are happy with the block) or false to continue iterating for whatever
 // reason. The iterator finally returns the result, whether or not a
 // satisfying block was eventually found.
-func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, int32) bool) bool {
-	f.mut.RLock()
-	folders := f.folders
-	f.mut.RUnlock()
+func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string, string, int32) bool) bool {
 	for _, folder := range folders {
 		key := toBlockKey(hash, folder, "")
 		iter := f.db.NewIterator(util.BytesPrefix(key), nil)

lib/db/blockmap_test.go

@@ -9,8 +9,7 @@ package db
 import (
 	"testing"
 
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
@@ -30,6 +29,7 @@ func genBlocks(n int) []protocol.BlockInfo {
 }
 
 var f1, f2, f3 protocol.FileInfo
+var folders = []string{"folder1", "folder2"}
 
 func init() {
 	blocks := genBlocks(30)
@@ -57,16 +57,7 @@ func setup() (*leveldb.DB, *BlockFinder) {
 	if err != nil {
 		panic(err)
 	}
-
-	wrapper := config.Wrap("", config.Configuration{})
-	wrapper.SetFolder(config.FolderConfiguration{
-		ID: "folder1",
-	})
-	wrapper.SetFolder(config.FolderConfiguration{
-		ID: "folder2",
-	})
-
-	return db, NewBlockFinder(db, wrapper)
+	return db, NewBlockFinder(db)
 }
 
 func dbEmpty(db *leveldb.DB) bool {
@@ -94,21 +85,21 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		if folder != "folder1" || file != "f1" || index != 0 {
 			t.Fatal("Mismatch")
 		}
 		return true
 	})
 
-	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		if folder != "folder1" || file != "f2" || index != 0 {
 			t.Fatal("Mismatch")
 		}
 		return true
 	})
 
-	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		t.Fatal("Unexpected block")
 		return true
 	})
@@ -123,17 +114,17 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		t.Fatal("Unexpected block")
 		return false
 	})
 
-	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		t.Fatal("Unexpected block")
 		return false
 	})
 
-	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		if folder != "folder1" || file != "f3" || index != 0 {
 			t.Fatal("Mismatch")
 		}
@@ -180,7 +171,7 @@ func TestBlockFinderLookup(t *testing.T) {
 	}
 
 	counter := 0
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		counter++
 		switch counter {
 		case 1:
@@ -196,6 +187,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		}
 		return false
 	})
+
 	if counter != 2 {
 		t.Fatal("Incorrect count", counter)
 	}
@@ -208,7 +200,7 @@ func TestBlockFinderLookup(t *testing.T) {
 	}
 
 	counter = 0
-	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
 		counter++
 		switch counter {
 		case 1:
@@ -220,6 +212,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		}
 		return false
 	})
+
 	if counter != 1 {
 		t.Fatal("Incorrect count")
 	}
@@ -240,7 +233,7 @@ func TestBlockFinderFix(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !f.Iterate(f1.Blocks[0].Hash, iterFn) {
+	if !f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
 		t.Fatal("Block not found")
 	}
 
@@ -249,11 +242,11 @@ func TestBlockFinderFix(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if f.Iterate(f1.Blocks[0].Hash, iterFn) {
+	if f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
 		t.Fatal("Unexpected block")
 	}
 
-	if !f.Iterate(f2.Blocks[0].Hash, iterFn) {
+	if !f.Iterate(folders, f2.Blocks[0].Hash, iterFn) {
 		t.Fatal("Block not found")
 	}
 }

lib/db/debug.go

@@ -10,11 +10,13 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/logger"
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debug   = strings.Contains(os.Getenv("STTRACE"), "files") || os.Getenv("STTRACE") == "all"
-	debugDB = strings.Contains(os.Getenv("STTRACE"), "db") || os.Getenv("STTRACE") == "all"
-	l       = logger.DefaultLogger
+	l = logger.DefaultLogger.NewFacility("db", "The database layer")
 )
+
+func init() {
+	l.SetDebug("db", strings.Contains(os.Getenv("STTRACE"), "db") || os.Getenv("STTRACE") == "all")
+}

lib/db/leveldb.go

@@ -15,7 +15,7 @@ import (
 	"runtime"
 	"sort"
 
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
@@ -178,20 +178,14 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 	limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
 
 	batch := new(leveldb.Batch)
-	if debugDB {
-		l.Debugf("new batch %p", batch)
-	}
+	l.Debugf("new batch %p", batch)
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -220,15 +214,11 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 
 		cmp := bytes.Compare(newName, oldName)
 
-		if debugDB {
-			l.Debugf("generic replace; folder=%q device=%v moreFs=%v moreDb=%v cmp=%d newName=%q oldName=%q", folder, protocol.DeviceIDFromBytes(device), moreFs, moreDb, cmp, newName, oldName)
-		}
+		l.Debugf("generic replace; folder=%q device=%v moreFs=%v moreDb=%v cmp=%d newName=%q oldName=%q", folder, protocol.DeviceIDFromBytes(device), moreFs, moreDb, cmp, newName, oldName)
 
 		switch {
 		case moreFs && (!moreDb || cmp == -1):
-			if debugDB {
-				l.Debugln("generic replace; missing - insert")
-			}
+			l.Debugln("generic replace; missing - insert")
 			// Database is missing this file. Insert it.
 			if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
 				maxLocalVer = lv
@@ -244,15 +234,11 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 			// File exists on both sides - compare versions. We might get an
 			// update with the same version and different flags if a device has
 			// marked a file as invalid, so handle that too.
-			if debugDB {
-				l.Debugln("generic replace; exists - compare")
-			}
+			l.Debugln("generic replace; exists - compare")
 			var ef FileInfoTruncated
 			ef.UnmarshalXDR(dbi.Value())
 			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Flags != ef.Flags {
-				if debugDB {
-					l.Debugln("generic replace; differs - insert")
-				}
+				l.Debugln("generic replace; differs - insert")
 				if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
 					maxLocalVer = lv
 				}
@@ -261,7 +247,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 				} else {
 					ldbUpdateGlobal(snap, batch, folder, device, fs[fsi])
 				}
-			} else if debugDB {
+			} else {
 				l.Debugln("generic replace; equal - ignore")
 			}
 
@@ -269,9 +255,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 			moreDb = dbi.Next()
 
 		case moreDb && (!moreFs || cmp == 1):
-			if debugDB {
-				l.Debugln("generic replace; exists - remove")
-			}
+			l.Debugln("generic replace; exists - remove")
 			if lv := deleteFn(snap, batch, folder, device, oldName, dbi); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
@@ -281,9 +265,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 		// Write out and reuse the batch every few records, to avoid the batch
 		// growing too large and thus allocating unnecessarily much memory.
 		if batch.Len() > batchFlushSize {
-			if debugDB {
-				l.Debugf("db.Write %p", batch)
-			}
+			l.Debugf("db.Write %p", batch)
 
 			err = db.Write(batch, nil)
 			if err != nil {
@@ -294,9 +276,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 		}
 	}
 
-	if debugDB {
-		l.Debugf("db.Write %p", batch)
-	}
+	l.Debugf("db.Write %p", batch)
 	err = db.Write(batch, nil)
 	if err != nil {
 		panic(err)
@@ -309,13 +289,9 @@ func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) i
 	// TODO: Return the remaining maxLocalVer?
 	return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
 		// Database has a file that we are missing. Remove it.
-		if debugDB {
-			l.Debugf("delete; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name)
-		}
+		l.Debugf("delete; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name)
 		ldbRemoveFromGlobal(db, batch, folder, device, name)
-		if debugDB {
-			l.Debugf("batch.Delete %p %x", batch, dbi.Key())
-		}
+		l.Debugf("batch.Delete %p %x", batch, dbi.Key())
 		batch.Delete(dbi.Key())
 		return 0
 	})
@@ -325,20 +301,14 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 	runtime.GC()
 
 	batch := new(leveldb.Batch)
-	if debugDB {
-		l.Debugf("new batch %p", batch)
-	}
+	l.Debugf("new batch %p", batch)
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -347,9 +317,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 	for _, f := range fs {
 		name := []byte(f.Name)
 		fk = deviceKeyInto(fk[:cap(fk)], folder, device, name)
-		if debugDB {
-			l.Debugf("snap.Get %p %x", snap, fk)
-		}
+		l.Debugf("snap.Get %p %x", snap, fk)
 		bs, err := snap.Get(fk, nil)
 		if err == leveldb.ErrNotFound {
 			if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
@@ -384,9 +352,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 		// Write out and reuse the batch every few records, to avoid the batch
 		// growing too large and thus allocating unnecessarily much memory.
 		if batch.Len() > batchFlushSize {
-			if debugDB {
-				l.Debugf("db.Write %p", batch)
-			}
+			l.Debugf("db.Write %p", batch)
 
 			err = db.Write(batch, nil)
 			if err != nil {
@@ -397,9 +363,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 		}
 	}
 
-	if debugDB {
-		l.Debugf("db.Write %p", batch)
-	}
+	l.Debugf("db.Write %p", batch)
 	err = db.Write(batch, nil)
 	if err != nil {
 		panic(err)
@@ -409,9 +373,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 }
 
 func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) int64 {
-	if debugDB {
-		l.Debugf("insert; folder=%q device=%v %v", folder, protocol.DeviceIDFromBytes(device), file)
-	}
+	l.Debugf("insert; folder=%q device=%v %v", folder, protocol.DeviceIDFromBytes(device), file)
 
 	if file.LocalVersion == 0 {
 		file.LocalVersion = clock(0)
@@ -419,9 +381,7 @@ func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) in
 
 	name := []byte(file.Name)
 	nk := deviceKey(folder, device, name)
-	if debugDB {
-		l.Debugf("batch.Put %p %x", batch, nk)
-	}
+	l.Debugf("batch.Put %p %x", batch, nk)
 	batch.Put(nk, file.MustMarshalXDR())
 
 	return file.LocalVersion
@@ -431,9 +391,7 @@ func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) in
 // file. If the device is already present in the list, the version is updated.
 // If the file does not have an entry in the global list, it is created.
 func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file protocol.FileInfo) bool {
-	if debugDB {
-		l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version)
-	}
+	l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version)
 	name := []byte(file.Name)
 	gk := globalKey(folder, name)
 	svl, err := db.Get(gk, nil)
@@ -499,10 +457,8 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 	fl.versions = append(fl.versions, nv)
 
 done:
-	if debugDB {
-		l.Debugf("batch.Put %p %x", batch, gk)
-		l.Debugf("new global after update: %v", fl)
-	}
+	l.Debugf("batch.Put %p %x", batch, gk)
+	l.Debugf("new global after update: %v", fl)
 	batch.Put(gk, fl.MustMarshalXDR())
 
 	return true
@@ -519,9 +475,7 @@ func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
 // given file. If the version list is empty after this, the file entry is
 // removed entirely.
 func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byte) {
-	if debugDB {
-		l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
-	}
+	l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
 
 	gk := globalKey(folder, file)
 	svl, err := db.Get(gk, nil)
@@ -545,15 +499,11 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
 	}
 
 	if len(fl.versions) == 0 {
-		if debugDB {
-			l.Debugf("batch.Delete %p %x", batch, gk)
-		}
+		l.Debugf("batch.Delete %p %x", batch, gk)
 		batch.Delete(gk)
 	} else {
-		if debugDB {
-			l.Debugf("batch.Put %p %x", batch, gk)
-			l.Debugf("new global after remove: %v", fl)
-		}
+		l.Debugf("batch.Put %p %x", batch, gk)
+		l.Debugf("new global after remove: %v", fl)
 		batch.Put(gk, fl.MustMarshalXDR())
 	}
 }
@@ -565,13 +515,9 @@ func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterat
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -598,13 +544,9 @@ func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []b
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -659,19 +601,13 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf,
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
-	if debugDB {
-		l.Debugf("snap.Get %p %x", snap, k)
-	}
+	l.Debugf("snap.Get %p %x", snap, k)
 	bs, err := snap.Get(k, nil)
 	if err == leveldb.ErrNotFound {
 		return nil, false
@@ -691,9 +627,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf,
 	}
 
 	k = deviceKey(folder, vl.versions[0].device, file)
-	if debugDB {
-		l.Debugf("snap.Get %p %x", snap, k)
-	}
+	l.Debugf("snap.Get %p %x", snap, k)
 	bs, err = snap.Get(k, nil)
 	if err != nil {
 		panic(err)
@@ -713,13 +647,9 @@ func ldbWithGlobal(db *leveldb.DB, folder, prefix []byte, truncate bool, fn Iter
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -739,9 +669,7 @@ func ldbWithGlobal(db *leveldb.DB, folder, prefix []byte, truncate bool, fn Iter
 		}
 		name := globalKeyName(dbi.Key())
 		fk = deviceKeyInto(fk[:cap(fk)], folder, vl.versions[0].device, name)
-		if debugDB {
-			l.Debugf("snap.Get %p %x", snap, fk)
-		}
+		l.Debugf("snap.Get %p %x", snap, fk)
 		bs, err := snap.Get(fk, nil)
 		if err != nil {
 			l.Debugf("folder: %q (%x)", folder, folder)
@@ -802,13 +730,9 @@ func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterat
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -854,9 +778,7 @@ nextFile:
 					continue nextFile
 				}
 				fk = deviceKeyInto(fk[:cap(fk)], folder, vl.versions[i].device, name)
-				if debugDB {
-					l.Debugf("snap.Get %p %x", snap, fk)
-				}
+				l.Debugf("snap.Get %p %x", snap, fk)
 				bs, err := snap.Get(fk, nil)
 				if err != nil {
 					var id protocol.DeviceID
@@ -886,9 +808,7 @@ nextFile:
 					continue nextFile
 				}
 
-				if debugDB {
-					l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.versions[0].version)
-				}
+				l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.versions[0].version)
 
 				if cont := fn(gf); !cont {
 					return
@@ -908,13 +828,9 @@ func ldbListFolders(db *leveldb.DB) []string {
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -945,13 +861,9 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -995,13 +907,9 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 	if err != nil {
 		panic(err)
 	}
-	if debugDB {
-		l.Debugf("created snapshot %p", snap)
-	}
+	l.Debugf("created snapshot %p", snap)
 	defer func() {
-		if debugDB {
-			l.Debugf("close snapshot %p", snap)
-		}
+		l.Debugf("close snapshot %p", snap)
 		snap.Release()
 	}()
 
@@ -1011,9 +919,7 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 	defer dbi.Release()
 
 	batch := new(leveldb.Batch)
-	if debugDB {
-		l.Debugf("new batch %p", batch)
-	}
+	l.Debugf("new batch %p", batch)
 
 	var fk []byte
 	for dbi.Next() {
@@ -1033,9 +939,7 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 		var newVL versionList
 		for _, version := range vl.versions {
 			fk = deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
-			if debugDB {
-				l.Debugf("snap.Get %p %x", snap, fk)
-			}
+			l.Debugf("snap.Get %p %x", snap, fk)
 			_, err := snap.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
 				continue
@@ -1051,8 +955,6 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 			batch.Put(dbi.Key(), newVL.MustMarshalXDR())
 		}
 	}
-	if debugDB {
-		l.Infoln("db check completed for %q", folder)
-	}
+	l.Debugf("db check completed for %q", folder)
 	db.Write(batch, nil)
 }

lib/db/set.go

@@ -13,8 +13,8 @@
 package db
 
 import (
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -62,18 +62,14 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
 		}
 		return true
 	})
-	if debug {
-		l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion)
-	}
+	l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion)
 	clock(s.localVersion[protocol.LocalDeviceID])
 
 	return &s
 }
 
 func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
-	if debug {
-		l.Debugf("%s Replace(%v, [%d])", s.folder, device, len(fs))
-	}
+	l.Debugf("%s Replace(%v, [%d])", s.folder, device, len(fs))
 	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
@@ -89,9 +85,7 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
 }
 
 func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
-	if debug {
-		l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs))
-	}
+	l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs))
 	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
@@ -114,51 +108,37 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 }
 
 func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) {
-	if debug {
-		l.Debugf("%s WithNeed(%v)", s.folder, device)
-	}
+	l.Debugf("%s WithNeed(%v)", s.folder, device)
 	ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
 }
 
 func (s *FileSet) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
-	if debug {
-		l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
-	}
+	l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
 	ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
 }
 
 func (s *FileSet) WithHave(device protocol.DeviceID, fn Iterator) {
-	if debug {
-		l.Debugf("%s WithHave(%v)", s.folder, device)
-	}
+	l.Debugf("%s WithHave(%v)", s.folder, device)
 	ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
 }
 
 func (s *FileSet) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
-	if debug {
-		l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
-	}
+	l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
 	ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
 }
 
 func (s *FileSet) WithGlobal(fn Iterator) {
-	if debug {
-		l.Debugf("%s WithGlobal()", s.folder)
-	}
+	l.Debugf("%s WithGlobal()", s.folder)
 	ldbWithGlobal(s.db, []byte(s.folder), nil, false, nativeFileIterator(fn))
 }
 
 func (s *FileSet) WithGlobalTruncated(fn Iterator) {
-	if debug {
-		l.Debugf("%s WithGlobalTruncated()", s.folder)
-	}
+	l.Debugf("%s WithGlobalTruncated()", s.folder)
 	ldbWithGlobal(s.db, []byte(s.folder), nil, true, nativeFileIterator(fn))
 }
 
 func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
-	if debug {
-		l.Debugf("%s WithPrefixedGlobalTruncated()", s.folder, prefix)
-	}
+	l.Debugf("%s WithPrefixedGlobalTruncated()", s.folder, prefix)
 	ldbWithGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn))
 }
 

lib/db/set_test.go

@@ -13,8 +13,8 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )

lib/db/truncated.go

@@ -6,7 +6,7 @@
 
 package db
 
-import "github.com/syncthing/protocol"
+import "github.com/syncthing/syncthing/lib/protocol"
 
 type FileInfoTruncated struct {
 	protocol.FileInfo

lib/db/virtualmtime.go

@@ -34,9 +34,7 @@ func NewVirtualMtimeRepo(ldb *leveldb.DB, folder string) *VirtualMtimeRepo {
 }
 
 func (r *VirtualMtimeRepo) UpdateMtime(path string, diskMtime, actualMtime time.Time) {
-	if debug {
-		l.Debugf("virtual mtime: storing values for path:%s disk:%v actual:%v", path, diskMtime, actualMtime)
-	}
+	l.Debugf("virtual mtime: storing values for path:%s disk:%v actual:%v", path, diskMtime, actualMtime)
 
 	diskBytes, _ := diskMtime.MarshalBinary()
 	actualBytes, _ := actualMtime.MarshalBinary()
@@ -63,15 +61,11 @@ func (r *VirtualMtimeRepo) GetMtime(path string, diskMtime time.Time) time.Time
 			panic(fmt.Sprintf("Can't unmarshal stored mtime at path %s: %v", path, err))
 		}
 
-		if debug {
-			l.Debugf("virtual mtime: return %v instead of %v for path: %s", mtime, diskMtime, path)
-		}
+		l.Debugf("virtual mtime: return %v instead of %v for path: %s", mtime, diskMtime, path)
 		return mtime
 	}
 
-	if debug {
-		l.Debugf("virtual mtime: record exists, but mismatch inDisk: %v dbDisk: %v for path: %s", diskMtime, mtime, path)
-	}
+	l.Debugf("virtual mtime: record exists, but mismatch inDisk: %v dbDisk: %v for path: %s", diskMtime, mtime, path)
 	return diskMtime
 }
 

lib/discover/cache.go

@@ -0,0 +1,269 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"sort"
+	stdsync "sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/thejerf/suture"
+)
+
+// The CachingMux aggregates results from multiple Finders. Each Finder has
+// an associated cache time and negative cache time. The cache time sets how
+// long we cache and return successfull lookup results, the negative cache
+// time sets how long we refrain from asking about the same device ID after
+// receiving a negative answer. The value of zero disables caching (positive
+// or negative).
+type CachingMux struct {
+	*suture.Supervisor
+	finders []cachedFinder
+	caches  []*cache
+	mut     sync.Mutex
+}
+
+// A cachedFinder is a Finder with associated cache timeouts.
+type cachedFinder struct {
+	Finder
+	cacheTime    time.Duration
+	negCacheTime time.Duration
+	priority     int
+}
+
+// A prioritizedAddress is what we use to sort addresses returned from
+// different sources with different priorities.
+type prioritizedAddress struct {
+	priority int
+	addr     string
+}
+
+func NewCachingMux() *CachingMux {
+	return &CachingMux{
+		Supervisor: suture.NewSimple("discover.cachingMux"),
+		mut:        sync.NewMutex(),
+	}
+}
+
+// Add registers a new Finder, with associated cache timeouts.
+func (m *CachingMux) Add(finder Finder, cacheTime, negCacheTime time.Duration, priority int) {
+	m.mut.Lock()
+	m.finders = append(m.finders, cachedFinder{finder, cacheTime, negCacheTime, priority})
+	m.caches = append(m.caches, newCache())
+	m.mut.Unlock()
+
+	if svc, ok := finder.(suture.Service); ok {
+		m.Supervisor.Add(svc)
+	}
+}
+
+// Lookup attempts to resolve the device ID using any of the added Finders,
+// while obeying the cache settings.
+func (m *CachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	var pdirect []prioritizedAddress
+
+	m.mut.Lock()
+	for i, finder := range m.finders {
+		if cacheEntry, ok := m.caches[i].Get(deviceID); ok {
+			// We have a cache entry. Lets see what it says.
+
+			if cacheEntry.found && time.Since(cacheEntry.when) < finder.cacheTime {
+				// It's a positive, valid entry. Use it.
+				l.Debugln("cached discovery entry for", deviceID, "at", finder)
+				l.Debugln("  cache:", cacheEntry)
+				for _, addr := range cacheEntry.Direct {
+					pdirect = append(pdirect, prioritizedAddress{finder.priority, addr})
+				}
+				relays = append(relays, cacheEntry.Relays...)
+				continue
+			}
+
+			if !cacheEntry.found && time.Since(cacheEntry.when) < finder.negCacheTime {
+				// It's a negative, valid entry. We should not make another
+				// attempt right now.
+				l.Debugln("negative cache entry for", deviceID, "at", finder)
+				continue
+			}
+
+			// It's expired. Ignore and continue.
+		}
+
+		// Perform the actual lookup and cache the result.
+		if td, tr, err := finder.Lookup(deviceID); err == nil {
+			l.Debugln("lookup for", deviceID, "at", finder)
+			l.Debugln("  direct:", td)
+			l.Debugln("  relays:", tr)
+			for _, addr := range td {
+				pdirect = append(pdirect, prioritizedAddress{finder.priority, addr})
+			}
+			relays = append(relays, tr...)
+			m.caches[i].Set(deviceID, CacheEntry{
+				Direct: td,
+				Relays: tr,
+				when:   time.Now(),
+				found:  len(td)+len(tr) > 0,
+			})
+		}
+	}
+	m.mut.Unlock()
+
+	direct = uniqueSortedAddrs(pdirect)
+	relays = uniqueSortedRelays(relays)
+	l.Debugln("lookup results for", deviceID)
+	l.Debugln("  direct: ", direct)
+	l.Debugln("  relays: ", relays)
+
+	return direct, relays, nil
+}
+
+func (m *CachingMux) String() string {
+	return "discovery cache"
+}
+
+func (m *CachingMux) Error() error {
+	return nil
+}
+
+func (m *CachingMux) ChildErrors() map[string]error {
+	m.mut.Lock()
+	children := make(map[string]error, len(m.finders))
+	for _, f := range m.finders {
+		children[f.String()] = f.Error()
+	}
+	m.mut.Unlock()
+	return children
+}
+
+func (m *CachingMux) Cache() map[protocol.DeviceID]CacheEntry {
+	// Res will be the "total" cache, i.e. the union of our cache and all our
+	// children's caches.
+	res := make(map[protocol.DeviceID]CacheEntry)
+
+	m.mut.Lock()
+	for i := range m.finders {
+		// Each finder[i] has a corresponding cache at cache[i]. Go through it
+		// and populate the total, if it's newer than what's already in there.
+		// We skip any negative cache entries.
+		for k, v := range m.caches[i].Cache() {
+			if v.found && v.when.After(res[k].when) {
+				res[k] = v
+			}
+		}
+
+		// Then ask the finder itself for it's cache and do the same. If this
+		// finder is a global discovery client, it will have no cache. If it's
+		// a local discovery client, this will be it's current state.
+		for k, v := range m.finders[i].Cache() {
+			if v.found && v.when.After(res[k].when) {
+				res[k] = v
+			}
+		}
+	}
+	m.mut.Unlock()
+
+	return res
+}
+
+// A cache can be embedded wherever useful
+
+type cache struct {
+	entries map[protocol.DeviceID]CacheEntry
+	mut     stdsync.Mutex
+}
+
+func newCache() *cache {
+	return &cache{
+		entries: make(map[protocol.DeviceID]CacheEntry),
+	}
+}
+
+func (c *cache) Set(id protocol.DeviceID, ce CacheEntry) {
+	c.mut.Lock()
+	c.entries[id] = ce
+	c.mut.Unlock()
+}
+
+func (c *cache) Get(id protocol.DeviceID) (CacheEntry, bool) {
+	c.mut.Lock()
+	ce, ok := c.entries[id]
+	c.mut.Unlock()
+	return ce, ok
+}
+
+func (c *cache) Cache() map[protocol.DeviceID]CacheEntry {
+	c.mut.Lock()
+	m := make(map[protocol.DeviceID]CacheEntry, len(c.entries))
+	for k, v := range c.entries {
+		m[k] = v
+	}
+	c.mut.Unlock()
+	return m
+}
+
+func uniqueSortedAddrs(ss []prioritizedAddress) []string {
+	// We sort the addresses by priority, then filter them based on seen
+	// (first time seen is the on kept, so we retain priority).
+	sort.Sort(prioritizedAddressList(ss))
+	filtered := make([]string, 0, len(ss))
+	seen := make(map[string]struct{}, len(ss))
+	for _, s := range ss {
+		if _, ok := seen[s.addr]; !ok {
+			filtered = append(filtered, s.addr)
+			seen[s.addr] = struct{}{}
+		}
+	}
+	return filtered
+}
+
+func uniqueSortedRelays(rs []Relay) []Relay {
+	m := make(map[string]Relay, len(rs))
+	for _, r := range rs {
+		m[r.URL] = r
+	}
+
+	var ur = make([]Relay, 0, len(m))
+	for _, r := range m {
+		ur = append(ur, r)
+	}
+
+	sort.Sort(relayList(ur))
+
+	return ur
+}
+
+type relayList []Relay
+
+func (l relayList) Len() int {
+	return len(l)
+}
+
+func (l relayList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l relayList) Less(a, b int) bool {
+	return l[a].URL < l[b].URL
+}
+
+type prioritizedAddressList []prioritizedAddress
+
+func (l prioritizedAddressList) Len() int {
+	return len(l)
+}
+
+func (l prioritizedAddressList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l prioritizedAddressList) Less(a, b int) bool {
+	if l[a].priority != l[b].priority {
+		return l[a].priority < l[b].priority
+	}
+	return l[a].addr < l[b].addr
+}

lib/discover/cache_test.go

@@ -0,0 +1,93 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func TestCacheUnique(t *testing.T) {
+	direct0 := []string{"tcp://192.0.2.44:22000", "tcp://192.0.2.42:22000"} // prio 0
+	direct1 := []string{"tcp://192.0.2.43:22000", "tcp://192.0.2.42:22000"} // prio 1
+
+	// what we expect from just direct0
+	direct0Sorted := []string{"tcp://192.0.2.42:22000", "tcp://192.0.2.44:22000"}
+
+	// what we expect from direct0+direct1
+	totalSorted := []string{
+		// first prio 0, sorted
+		"tcp://192.0.2.42:22000", "tcp://192.0.2.44:22000",
+		// then prio 1
+		"tcp://192.0.2.43:22000",
+		// no duplicate .42
+	}
+
+	relays := []Relay{{URL: "relay://192.0.2.44:443"}, {URL: "tcp://192.0.2.45:443"}}
+
+	c := NewCachingMux()
+	c.ServeBackground()
+	defer c.Stop()
+
+	// Add a fake discovery service and verify we get it's answers through the
+	// cache.
+
+	f1 := &fakeDiscovery{direct0, relays}
+	c.Add(f1, time.Minute, 0, 0)
+
+	dir, rel, err := c.Lookup(protocol.LocalDeviceID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(dir, direct0Sorted) {
+		t.Errorf("Incorrect direct; %+v != %+v", dir, direct0Sorted)
+	}
+	if !reflect.DeepEqual(rel, relays) {
+		t.Errorf("Incorrect relays; %+v != %+v", rel, relays)
+	}
+
+	// Add one more that answers in the same way and check that we don't
+	// duplicate or otherwise mess up the responses now.
+
+	f2 := &fakeDiscovery{direct1, relays}
+	c.Add(f2, time.Minute, 0, 1)
+
+	dir, rel, err = c.Lookup(protocol.LocalDeviceID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(dir, totalSorted) {
+		t.Errorf("Incorrect direct; %+v != %+v", dir, totalSorted)
+	}
+	if !reflect.DeepEqual(rel, relays) {
+		t.Errorf("Incorrect relays; %+v != %+v", rel, relays)
+	}
+}
+
+type fakeDiscovery struct {
+	direct []string
+	relays []Relay
+}
+
+func (f *fakeDiscovery) Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	return f.direct, f.relays, nil
+}
+
+func (f *fakeDiscovery) Error() error {
+	return nil
+}
+
+func (f *fakeDiscovery) String() string {
+	return "fake"
+}
+
+func (f *fakeDiscovery) Cache() map[protocol.DeviceID]CacheEntry {
+	return nil
+}

lib/discover/client.go

@@ -1,50 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"fmt"
-	"net/url"
-	"time"
-
-	"github.com/syncthing/protocol"
-)
-
-type Factory func(*url.URL, *Announce) (Client, error)
-
-var (
-	factories                      = make(map[string]Factory)
-	DefaultErrorRetryInternval     = 60 * time.Second
-	DefaultGlobalBroadcastInterval = 1800 * time.Second
-)
-
-func Register(proto string, factory Factory) {
-	factories[proto] = factory
-}
-
-func New(addr string, pkt *Announce) (Client, error) {
-	uri, err := url.Parse(addr)
-	if err != nil {
-		return nil, err
-	}
-	factory, ok := factories[uri.Scheme]
-	if !ok {
-		return nil, fmt.Errorf("Unsupported scheme: %s", uri.Scheme)
-	}
-	client, err := factory(uri, pkt)
-	if err != nil {
-		return nil, err
-	}
-	return client, nil
-}
-
-type Client interface {
-	Lookup(device protocol.DeviceID) []string
-	StatusOK() bool
-	Address() string
-	Stop()
-}

lib/discover/client_test.go

@@ -1,219 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"testing"
-
-	"github.com/syncthing/protocol"
-
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-var device protocol.DeviceID
-
-func init() {
-	device, _ = protocol.DeviceIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
-}
-
-func TestUDP4Success(t *testing.T) {
-	conn, err := net.ListenUDP("udp4", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-
-	address := fmt.Sprintf("udp4://127.0.0.1:%d", port)
-	pkt := &Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			device[:],
-			[]Address{{
-				IP:   net.IPv4(123, 123, 123, 123),
-				Port: 1234,
-			}},
-		},
-	}
-
-	client, err := New(address, pkt)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	udpclient := client.(*UDPClient)
-	if udpclient.errorRetryInterval != DefaultErrorRetryInternval {
-		t.Fatal("Incorrect retry interval")
-	}
-
-	if udpclient.listenAddress.IP != nil || udpclient.listenAddress.Port != 0 {
-		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
-	}
-
-	if client.Address() != address {
-		t.Fatal("Incorrect address")
-	}
-
-	buf := make([]byte, 2048)
-
-	// First announcement
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Announcement verification
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
-	_, addr, err := conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Reply to it.
-	_, err = conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// We should get nothing else
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-
-	// Status should be ok
-	if !client.StatusOK() {
-		t.Fatal("Wrong status")
-	}
-
-	// Do a lookup in a separate routine
-	addrs := []string{}
-	wg := sync.NewWaitGroup()
-	wg.Add(1)
-	go func() {
-		addrs = client.Lookup(device)
-		wg.Done()
-	}()
-
-	// Receive the lookup and reply
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, addr, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
-
-	// Wait for the lookup to arrive, verify that the number of answers is correct
-	wg.Wait()
-
-	if len(addrs) != 1 || addrs[0] != "123.123.123.123:1234" {
-		t.Fatal("Wrong number of answers")
-	}
-
-	client.Stop()
-}
-
-func TestUDP4Failure(t *testing.T) {
-	conn, err := net.ListenUDP("udp4", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-
-	address := fmt.Sprintf("udp4://127.0.0.1:%d/?listenaddress=127.0.0.1&retry=5", port)
-
-	pkt := &Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			device[:],
-			[]Address{{
-				IP:   net.IPv4(123, 123, 123, 123),
-				Port: 1234,
-			}},
-		},
-	}
-
-	client, err := New(address, pkt)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	udpclient := client.(*UDPClient)
-	if udpclient.errorRetryInterval != time.Second*5 {
-		t.Fatal("Incorrect retry interval")
-	}
-
-	if !udpclient.listenAddress.IP.Equal(net.IPv4(127, 0, 0, 1)) || udpclient.listenAddress.Port != 0 {
-		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
-	}
-
-	if client.Address() != address {
-		t.Fatal("Incorrect address")
-	}
-
-	buf := make([]byte, 2048)
-
-	// First announcement
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Announcement verification
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
-	_, _, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Don't reply
-	// We should get nothing else
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-
-	// Status should be failure
-	if client.StatusOK() {
-		t.Fatal("Wrong status")
-	}
-
-	// Do a lookup in a separate routine
-	addrs := []string{}
-	wg := sync.NewWaitGroup()
-	wg.Add(1)
-	go func() {
-		addrs = client.Lookup(device)
-		wg.Done()
-	}()
-
-	// Receive the lookup and don't reply
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, _, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Wait for the lookup to timeout, verify that the number of answers is none
-	wg.Wait()
-
-	if len(addrs) != 0 {
-		t.Fatal("Wrong number of answers")
-	}
-
-	client.Stop()
-}

lib/discover/client_udp.go

@@ -1,243 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"encoding/hex"
-	"io"
-	"net"
-	"net/url"
-	"strconv"
-	"time"
-
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-func init() {
-	for _, proto := range []string{"udp", "udp4", "udp6"} {
-		Register(proto, func(uri *url.URL, pkt *Announce) (Client, error) {
-			c := &UDPClient{
-				wg:  sync.NewWaitGroup(),
-				mut: sync.NewRWMutex(),
-			}
-			err := c.Start(uri, pkt)
-			if err != nil {
-				return nil, err
-			}
-			return c, nil
-		})
-	}
-}
-
-type UDPClient struct {
-	url *url.URL
-
-	id protocol.DeviceID
-
-	stop          chan struct{}
-	wg            sync.WaitGroup
-	listenAddress *net.UDPAddr
-
-	globalBroadcastInterval time.Duration
-	errorRetryInterval      time.Duration
-
-	status bool
-	mut    sync.RWMutex
-}
-
-func (d *UDPClient) Start(uri *url.URL, pkt *Announce) error {
-	d.url = uri
-	d.id = protocol.DeviceIDFromBytes(pkt.This.ID)
-	d.stop = make(chan struct{})
-
-	params := uri.Query()
-	// The address must not have a port, as otherwise both announce and lookup
-	// sockets would try to bind to the same port.
-	addr, err := net.ResolveUDPAddr(d.url.Scheme, params.Get("listenaddress")+":0")
-	if err != nil {
-		return err
-	}
-	d.listenAddress = addr
-
-	broadcastSeconds, err := strconv.ParseUint(params.Get("broadcast"), 0, 0)
-	if err != nil {
-		d.globalBroadcastInterval = DefaultGlobalBroadcastInterval
-	} else {
-		d.globalBroadcastInterval = time.Duration(broadcastSeconds) * time.Second
-	}
-
-	retrySeconds, err := strconv.ParseUint(params.Get("retry"), 0, 0)
-	if err != nil {
-		d.errorRetryInterval = DefaultErrorRetryInternval
-	} else {
-		d.errorRetryInterval = time.Duration(retrySeconds) * time.Second
-	}
-
-	d.wg.Add(1)
-	go d.broadcast(pkt.MustMarshalXDR())
-	return nil
-}
-
-func (d *UDPClient) broadcast(pkt []byte) {
-	defer d.wg.Done()
-
-	conn, err := net.ListenUDP(d.url.Scheme, d.listenAddress)
-	for err != nil {
-		if debug {
-			l.Debugf("discover %s: broadcast listen: %v; trying again in %v", d.url, err, d.errorRetryInterval)
-		}
-		select {
-		case <-d.stop:
-			return
-		case <-time.After(d.errorRetryInterval):
-		}
-		conn, err = net.ListenUDP(d.url.Scheme, d.listenAddress)
-	}
-	defer conn.Close()
-
-	remote, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	for err != nil {
-		if debug {
-			l.Debugf("discover %s: broadcast resolve: %v; trying again in %v", d.url, err, d.errorRetryInterval)
-		}
-		select {
-		case <-d.stop:
-			return
-		case <-time.After(d.errorRetryInterval):
-		}
-		remote, err = net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	}
-
-	timer := time.NewTimer(0)
-	for {
-		select {
-		case <-d.stop:
-			return
-
-		case <-timer.C:
-			var ok bool
-
-			if debug {
-				l.Debugf("discover %s: broadcast: Sending self announcement to %v", d.url, remote)
-			}
-
-			_, err := conn.WriteTo(pkt, remote)
-			if err != nil {
-				if debug {
-					l.Debugf("discover %s: broadcast: Failed to send self announcement: %s", d.url, err)
-				}
-				ok = false
-			} else {
-				// Verify that the announce server responds positively for our device ID
-
-				time.Sleep(1 * time.Second)
-
-				res := d.Lookup(d.id)
-				if debug {
-					l.Debugf("discover %s: broadcast: Self-lookup returned: %v", d.url, res)
-				}
-				ok = len(res) > 0
-			}
-
-			d.mut.Lock()
-			d.status = ok
-			d.mut.Unlock()
-
-			if ok {
-				timer.Reset(d.globalBroadcastInterval)
-			} else {
-				timer.Reset(d.errorRetryInterval)
-			}
-		}
-	}
-}
-
-func (d *UDPClient) Lookup(device protocol.DeviceID) []string {
-	extIP, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	conn, err := net.DialUDP(d.url.Scheme, d.listenAddress, extIP)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-	defer conn.Close()
-
-	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	buf := Query{QueryMagic, device[:]}.MustMarshalXDR()
-	_, err = conn.Write(buf)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	buf = make([]byte, 2048)
-	n, err := conn.Read(buf)
-	if err != nil {
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			// Expected if the server doesn't know about requested device ID
-			return nil
-		}
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return nil
-	}
-
-	var pkt Announce
-	err = pkt.UnmarshalXDR(buf[:n])
-	if err != nil && err != io.EOF {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s\n%s", d.url, device, err, hex.Dump(buf[:n]))
-		}
-		return nil
-	}
-
-	var addrs []string
-	for _, a := range pkt.This.Addresses {
-		deviceAddr := net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
-		addrs = append(addrs, deviceAddr)
-	}
-	if debug {
-		l.Debugf("discover %s: Lookup(%s) result: %v", d.url, device, addrs)
-	}
-	return addrs
-}
-
-func (d *UDPClient) Stop() {
-	if d.stop != nil {
-		close(d.stop)
-		d.wg.Wait()
-	}
-}
-
-func (d *UDPClient) StatusOK() bool {
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-	return d.status
-}
-
-func (d *UDPClient) Address() string {
-	return d.url.String()
-}

lib/discover/debug.go

@@ -10,10 +10,13 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/logger"
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
+	l = logger.DefaultLogger.NewFacility("discover", "Remote device discovery")
 )
+
+func init() {
+	l.SetDebug("discover", strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all")
+}

lib/discover/discover.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2014 The Syncthing Authors.
+// Copyright (C) 2015 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
@@ -7,424 +7,48 @@
 package discover
 
 import (
-	"bytes"
-	"encoding/hex"
-	"errors"
-	"io"
-	"net"
-	"strconv"
 	"time"
 
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/beacon"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/thejerf/suture"
 )
 
-type Discoverer struct {
-	myID            protocol.DeviceID
-	listenAddrs     []string
-	localBcastIntv  time.Duration
-	localBcastStart time.Time
-	cacheLifetime   time.Duration
-	negCacheCutoff  time.Duration
-	beacons         []beacon.Interface
-	extPort         uint16
-	localBcastTick  <-chan time.Time
-	forcedBcastTick chan time.Time
-
-	registryLock sync.RWMutex
-	registry     map[protocol.DeviceID][]CacheEntry
-	lastLookup   map[protocol.DeviceID]time.Time
-
-	clients []Client
-	mut     sync.RWMutex
+// A Finder provides lookup services of some kind.
+type Finder interface {
+	Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error)
+	Error() error
+	String() string
+	Cache() map[protocol.DeviceID]CacheEntry
 }
 
 type CacheEntry struct {
-	Address string
-	Seen    time.Time
+	Direct []string  `json:"direct"`
+	Relays []Relay   `json:"relays"`
+	when   time.Time // When did we get the result
+	found  bool      // Is it a success (cacheTime applies) or a failure (negCacheTime applies)?
 }
 
-var (
-	ErrIncorrectMagic = errors.New("incorrect magic number")
-)
-
-func NewDiscoverer(id protocol.DeviceID, addresses []string) *Discoverer {
-	return &Discoverer{
-		myID:           id,
-		listenAddrs:    addresses,
-		localBcastIntv: 30 * time.Second,
-		cacheLifetime:  5 * time.Minute,
-		negCacheCutoff: 3 * time.Minute,
-		registry:       make(map[protocol.DeviceID][]CacheEntry),
-		lastLookup:     make(map[protocol.DeviceID]time.Time),
-		registryLock:   sync.NewRWMutex(),
-		mut:            sync.NewRWMutex(),
-	}
+// A FinderService is a Finder that has background activity and must be run as
+// a suture.Service.
+type FinderService interface {
+	Finder
+	suture.Service
 }
 
-func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
-	if localPort > 0 {
-		d.startLocalIPv4Broadcasts(localPort)
-	}
-
-	if len(localMCAddr) > 0 {
-		d.startLocalIPv6Multicasts(localMCAddr)
-	}
-
-	if len(d.beacons) == 0 {
-		l.Warnln("Local discovery unavailable")
-		return
-	}
-
-	d.localBcastTick = time.Tick(d.localBcastIntv)
-	d.forcedBcastTick = make(chan time.Time)
-	d.localBcastStart = time.Now()
-	go d.sendLocalAnnouncements()
+type FinderMux interface {
+	Finder
+	ChildStatus() map[string]error
 }
 
-func (d *Discoverer) startLocalIPv4Broadcasts(localPort int) {
-	bb := beacon.NewBroadcast(localPort)
-	d.beacons = append(d.beacons, bb)
-	go d.recvAnnouncements(bb)
-	bb.ServeBackground()
+// The RelayStatusProvider answers questions about current relay status.
+type RelayStatusProvider interface {
+	Relays() []string
+	RelayStatus(uri string) (time.Duration, bool)
 }
 
-func (d *Discoverer) startLocalIPv6Multicasts(localMCAddr string) {
-	mb, err := beacon.NewMulticast(localMCAddr)
-	if err != nil {
-		if debug {
-			l.Debugln("beacon.NewMulticast:", err)
-		}
-		l.Infoln("Local discovery over IPv6 unavailable")
-		return
-	}
-	d.beacons = append(d.beacons, mb)
-	go d.recvAnnouncements(mb)
-}
-
-func (d *Discoverer) StartGlobal(servers []string, extPort uint16) {
-	d.mut.Lock()
-	defer d.mut.Unlock()
-
-	if len(d.clients) > 0 {
-		d.stopGlobal()
-	}
-
-	d.extPort = extPort
-	pkt := d.announcementPkt()
-	wg := sync.NewWaitGroup()
-	clients := make(chan Client, len(servers))
-	for _, address := range servers {
-		wg.Add(1)
-		go func(addr string) {
-			defer wg.Done()
-			client, err := New(addr, pkt)
-			if err != nil {
-				l.Infoln("Error creating discovery client", addr, err)
-				return
-			}
-			clients <- client
-		}(address)
-	}
-
-	wg.Wait()
-	close(clients)
-
-	for client := range clients {
-		d.clients = append(d.clients, client)
-	}
-}
-
-func (d *Discoverer) StopGlobal() {
-	d.mut.Lock()
-	defer d.mut.Unlock()
-	d.stopGlobal()
-}
-
-func (d *Discoverer) stopGlobal() {
-	for _, client := range d.clients {
-		client.Stop()
-	}
-	d.clients = []Client{}
-}
-
-func (d *Discoverer) ExtAnnounceOK() map[string]bool {
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-
-	ret := make(map[string]bool)
-	for _, client := range d.clients {
-		ret[client.Address()] = client.StatusOK()
-	}
-	return ret
-}
-
-func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
-	d.registryLock.RLock()
-	cached := d.filterCached(d.registry[device])
-	lastLookup := d.lastLookup[device]
-	d.registryLock.RUnlock()
-
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-
-	if len(cached) > 0 {
-		// There are cached address entries.
-		addrs := make([]string, len(cached))
-		for i := range cached {
-			addrs[i] = cached[i].Address
-		}
-		return addrs
-	}
-
-	if time.Since(lastLookup) < d.negCacheCutoff {
-		// We have recently tried to lookup this address and failed. Lets
-		// chill for a while.
-		return nil
-	}
-
-	if len(d.clients) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
-		// Only perform external lookups if we have at least one external
-		// server client and one local announcement interval has passed. This is
-		// to avoid finding local peers on their remote address at startup.
-		results := make(chan []string, len(d.clients))
-		wg := sync.NewWaitGroup()
-		for _, client := range d.clients {
-			wg.Add(1)
-			go func(c Client) {
-				defer wg.Done()
-				results <- c.Lookup(device)
-			}(client)
-		}
-
-		wg.Wait()
-		close(results)
-
-		cached := []CacheEntry{}
-		seen := make(map[string]struct{})
-		now := time.Now()
-
-		var addrs []string
-		for result := range results {
-			for _, addr := range result {
-				_, ok := seen[addr]
-				if !ok {
-					cached = append(cached, CacheEntry{
-						Address: addr,
-						Seen:    now,
-					})
-					seen[addr] = struct{}{}
-					addrs = append(addrs, addr)
-				}
-			}
-		}
-
-		d.registryLock.Lock()
-		d.registry[device] = cached
-		d.lastLookup[device] = time.Now()
-		d.registryLock.Unlock()
-
-		return addrs
-	}
-
-	return nil
-}
-
-func (d *Discoverer) Hint(device string, addrs []string) {
-	resAddrs := resolveAddrs(addrs)
-	var id protocol.DeviceID
-	id.UnmarshalText([]byte(device))
-	d.registerDevice(nil, Device{
-		Addresses: resAddrs,
-		ID:        id[:],
-	})
-}
-
-func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
-	d.registryLock.RLock()
-	devices := make(map[protocol.DeviceID][]CacheEntry, len(d.registry))
-	for device, addrs := range d.registry {
-		addrsCopy := make([]CacheEntry, len(addrs))
-		copy(addrsCopy, addrs)
-		devices[device] = addrsCopy
-	}
-	d.registryLock.RUnlock()
-	return devices
-}
-
-func (d *Discoverer) announcementPkt() *Announce {
-	var addrs []Address
-	if d.extPort != 0 {
-		addrs = []Address{{Port: d.extPort}}
-	} else {
-		for _, astr := range d.listenAddrs {
-			addr, err := net.ResolveTCPAddr("tcp", astr)
-			if err != nil {
-				l.Warnln("discover: %v: not announcing %s", err, astr)
-				continue
-			} else if debug {
-				l.Debugf("discover: resolved %s as %#v", astr, addr)
-			}
-			if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
-				addrs = append(addrs, Address{Port: uint16(addr.Port)})
-			} else if bs := addr.IP.To4(); bs != nil {
-				addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
-			} else if bs := addr.IP.To16(); bs != nil {
-				addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
-			}
-		}
-	}
-	return &Announce{
-		Magic: AnnouncementMagic,
-		This:  Device{d.myID[:], addrs},
-	}
-}
-
-func (d *Discoverer) sendLocalAnnouncements() {
-	var addrs = resolveAddrs(d.listenAddrs)
-
-	var pkt = Announce{
-		Magic: AnnouncementMagic,
-		This:  Device{d.myID[:], addrs},
-	}
-	msg := pkt.MustMarshalXDR()
-
-	for {
-		for _, b := range d.beacons {
-			b.Send(msg)
-		}
-
-		select {
-		case <-d.localBcastTick:
-		case <-d.forcedBcastTick:
-		}
-	}
-}
-
-func (d *Discoverer) recvAnnouncements(b beacon.Interface) {
-	for {
-		buf, addr := b.Recv()
-
-		var pkt Announce
-		err := pkt.UnmarshalXDR(buf)
-		if err != nil && err != io.EOF {
-			if debug {
-				l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
-			}
-			continue
-		}
-
-		if debug {
-			l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
-		}
-
-		var newDevice bool
-		if bytes.Compare(pkt.This.ID, d.myID[:]) != 0 {
-			newDevice = d.registerDevice(addr, pkt.This)
-		}
-
-		if newDevice {
-			select {
-			case d.forcedBcastTick <- time.Now():
-			}
-		}
-	}
-}
-
-func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
-	var id protocol.DeviceID
-	copy(id[:], device.ID)
-
-	d.registryLock.Lock()
-	defer d.registryLock.Unlock()
-
-	current := d.filterCached(d.registry[id])
-
-	orig := current
-
-	for _, a := range device.Addresses {
-		var deviceAddr string
-		if len(a.IP) > 0 {
-			deviceAddr = net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
-		} else if addr != nil {
-			ua := addr.(*net.UDPAddr)
-			ua.Port = int(a.Port)
-			deviceAddr = ua.String()
-		}
-		for i := range current {
-			if current[i].Address == deviceAddr {
-				current[i].Seen = time.Now()
-				goto done
-			}
-		}
-		current = append(current, CacheEntry{
-			Address: deviceAddr,
-			Seen:    time.Now(),
-		})
-	done:
-	}
-
-	if debug {
-		l.Debugf("discover: Caching %s addresses: %v", id, current)
-	}
-
-	d.registry[id] = current
-
-	if len(current) > len(orig) {
-		addrs := make([]string, len(current))
-		for i := range current {
-			addrs[i] = current[i].Address
-		}
-		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
-			"device": id.String(),
-			"addrs":  addrs,
-		})
-	}
-
-	return len(current) > len(orig)
-}
-
-func (d *Discoverer) filterCached(c []CacheEntry) []CacheEntry {
-	for i := 0; i < len(c); {
-		if ago := time.Since(c[i].Seen); ago > d.cacheLifetime {
-			if debug {
-				l.Debugf("discover: Removing cached address %s - seen %v ago", c[i].Address, ago)
-			}
-			c[i] = c[len(c)-1]
-			c = c[:len(c)-1]
-		} else {
-			i++
-		}
-	}
-	return c
-}
-
-func addrToAddr(addr *net.TCPAddr) Address {
-	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
-		return Address{Port: uint16(addr.Port)}
-	} else if bs := addr.IP.To4(); bs != nil {
-		return Address{IP: bs, Port: uint16(addr.Port)}
-	} else if bs := addr.IP.To16(); bs != nil {
-		return Address{IP: bs, Port: uint16(addr.Port)}
-	}
-	return Address{}
-}
-
-func resolveAddrs(addrs []string) []Address {
-	var raddrs []Address
-	for _, addrStr := range addrs {
-		addrRes, err := net.ResolveTCPAddr("tcp", addrStr)
-		if err != nil {
-			continue
-		}
-		addr := addrToAddr(addrRes)
-		if len(addr.IP) > 0 {
-			raddrs = append(raddrs, addr)
-		} else {
-			raddrs = append(raddrs, Address{Port: addr.Port})
-		}
-	}
-	return raddrs
+// The AddressLister answers questions about what addresses we are listening
+// on.
+type AddressLister interface {
+	ExternalAddresses() []string
+	AllAddresses() []string
 }

lib/discover/discover_test.go

@@ -1,139 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"net/url"
-	"time"
-
-	"testing"
-
-	"github.com/syncthing/protocol"
-)
-
-type DummyClient struct {
-	url          *url.URL
-	lookups      []protocol.DeviceID
-	lookupRet    []string
-	stops        int
-	statusRet    bool
-	statusChecks int
-}
-
-func (c *DummyClient) Lookup(device protocol.DeviceID) []string {
-	c.lookups = append(c.lookups, device)
-	return c.lookupRet
-}
-
-func (c *DummyClient) StatusOK() bool {
-	c.statusChecks++
-	return c.statusRet
-}
-
-func (c *DummyClient) Stop() {
-	c.stops++
-}
-
-func (c *DummyClient) Address() string {
-	return c.url.String()
-}
-
-func TestGlobalDiscovery(t *testing.T) {
-	c1 := &DummyClient{
-		statusRet: false,
-		lookupRet: []string{"test.com:1234"},
-	}
-
-	c2 := &DummyClient{
-		statusRet: true,
-		lookupRet: []string{},
-	}
-
-	c3 := &DummyClient{
-		statusRet: true,
-		lookupRet: []string{"best.com:2345"},
-	}
-
-	clients := []*DummyClient{c1, c2}
-
-	Register("test1", func(uri *url.URL, pkt *Announce) (Client, error) {
-		c := clients[0]
-		clients = clients[1:]
-		c.url = uri
-		return c, nil
-	})
-
-	Register("test2", func(uri *url.URL, pkt *Announce) (Client, error) {
-		c3.url = uri
-		return c3, nil
-	})
-
-	d := NewDiscoverer(device, []string{})
-	d.localBcastStart = time.Time{}
-	servers := []string{
-		"test1://123.123.123.123:1234",
-		"test1://23.23.23.23:234",
-		"test2://234.234.234.234.2345",
-	}
-	d.StartGlobal(servers, 1234)
-
-	if len(d.clients) != 3 {
-		t.Fatal("Wrong number of clients")
-	}
-
-	status := d.ExtAnnounceOK()
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if status[c.url.String()] != c.statusRet || c.statusChecks != 1 {
-			t.Fatal("Wrong status")
-		}
-	}
-
-	addrs := d.Lookup(device)
-	if len(addrs) != 2 {
-		t.Fatal("Wrong number of addresses", addrs)
-	}
-
-	for _, addr := range []string{"test.com:1234", "best.com:2345"} {
-		found := false
-		for _, laddr := range addrs {
-			if laddr == addr {
-				found = true
-				break
-			}
-		}
-		if !found {
-			t.Fatal("Couldn't find", addr)
-		}
-	}
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if len(c.lookups) != 1 || c.lookups[0] != device {
-			t.Fatal("Wrong lookups")
-		}
-	}
-
-	addrs = d.Lookup(device)
-	if len(addrs) != 2 {
-		t.Fatal("Wrong number of addresses", addrs)
-	}
-
-	// Answer should be cached, so number of lookups should have not increased
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if len(c.lookups) != 1 || c.lookups[0] != device {
-			t.Fatal("Wrong lookups")
-		}
-	}
-
-	d.StopGlobal()
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if c.stops != 1 {
-			t.Fatal("Wrong number of stops")
-		}
-	}
-}

lib/discover/doc.go

@@ -1,8 +1,75 @@
-// Copyright (C) 2014 The Syncthing Authors.
+// Copyright (C) 2015 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-// Package discover implements the device discovery protocol.
+/*
+Package discover implements the local and global device discovery protocols.
+
+Global Discovery
+================
+
+Announcements
+-------------
+
+A device should announce itself at startup. It does this by an HTTPS POST to
+the announce server URL (with the path usually being "/", but this is of
+course up to the discovery server). The POST has a JSON payload listing direct
+connection addresses (if any) and relay addresses (if any).
+
+	{
+		direct: ["tcp://192.0.2.45:22000", "tcp://:22202"],
+		relays: [{"url": "relay://192.0.2.99:22028", "latency": 142}]
+	}
+
+It's OK for either of the "direct" or "relays" fields to be either the empty
+list ([]), null, or missing entirely. An announcment with both fields missing
+or empty is however not useful...
+
+Any empty or unspecified IP addresses (i.e. addresses like tcp://:22000,
+tcp://0.0.0.0:22000, tcp://[::]:22000) are interpreted as referring to the
+source IP address of the announcement.
+
+The device ID of the announcing device is not part of the announcement.
+Instead, the server requires that the client perform certificate
+authentication. The device ID is deduced from the presented certificate.
+
+The server response is empty, with code 200 (OK) on success. If no certificate
+was presented, status 403 (Forbidden) is returned. If the posted data doesn't
+conform to the expected format, 400 (Bad Request) is returned.
+
+In successfull responses, the server may return a "Reannounce-After" header
+containing the number of seconds after which the client should perform a new
+announcement.
+
+In error responses, the server may return a "Retry-After" header containing
+the number of seconds after which the client should retry.
+
+Performing announcements significantly more often than indicated by the
+Reannounce-After or Retry-After headers may result in the client being
+throttled. In such cases the server may respond with status code 429 (Too Many
+Requests).
+
+Queries
+=======
+
+Queries are performed as HTTPS GET requests to the announce server URL. The
+requested device ID is passed as the query parameter "device", in canonical
+string form, i.e. https://announce.syncthing.net/?device=ABC12345-....
+
+Successfull responses will have status code 200 (OK) and carry a JSON payload
+of the same format as the announcement above. The response will not contain
+empty or unspecified addresses.
+
+If the "device" query parameter is missing or malformed, the status code 400
+(Bad Request) is returned.
+
+If the device ID is of a valid format but not found in the registry, 404 (Not
+Found) is returned.
+
+If the client has exceeded a rate limit, the server may respond with 429 (Too
+Many Requests).
+
+*/
 package discover

lib/discover/global.go

@@ -0,0 +1,373 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	stdsync "sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type globalClient struct {
+	server         string
+	addrList       AddressLister
+	relayStat      RelayStatusProvider
+	announceClient httpClient
+	queryClient    httpClient
+	noAnnounce     bool
+	stop           chan struct{}
+	errorHolder
+}
+
+type httpClient interface {
+	Get(url string) (*http.Response, error)
+	Post(url, ctype string, data io.Reader) (*http.Response, error)
+}
+
+const (
+	defaultReannounceInterval  = 30 * time.Minute
+	announceErrorRetryInterval = 5 * time.Minute
+)
+
+type announcement struct {
+	Direct []string `json:"direct"`
+	Relays []Relay  `json:"relays"`
+}
+
+type serverOptions struct {
+	insecure   bool   // don't check certificate
+	noAnnounce bool   // don't announce
+	id         string // expected server device ID
+}
+
+func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
+	server, opts, err := parseOptions(server)
+	if err != nil {
+		return nil, err
+	}
+
+	var devID protocol.DeviceID
+	if opts.id != "" {
+		devID, err = protocol.DeviceIDFromString(opts.id)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// The http.Client used for announcements. It needs to have our
+	// certificate to prove our identity, and may or may not verify the server
+	// certificate depending on the insecure setting.
+	var announceClient httpClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: opts.insecure,
+				Certificates:       []tls.Certificate{cert},
+			},
+		},
+	}
+	if opts.id != "" {
+		announceClient = newIDCheckingHTTPClient(announceClient, devID)
+	}
+
+	// The http.Client used for queries. We don't need to present our
+	// certificate here, so lets not include it. May be insecure if requested.
+	var queryClient httpClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: opts.insecure,
+			},
+		},
+	}
+	if opts.id != "" {
+		queryClient = newIDCheckingHTTPClient(queryClient, devID)
+	}
+
+	cl := &globalClient{
+		server:         server,
+		addrList:       addrList,
+		relayStat:      relayStat,
+		announceClient: announceClient,
+		queryClient:    queryClient,
+		noAnnounce:     opts.noAnnounce,
+		stop:           make(chan struct{}),
+	}
+	cl.setError(errors.New("not announced"))
+
+	return cl, nil
+}
+
+// Lookup returns the list of addresses where the given device is available;
+// direct, and via relays.
+func (c *globalClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	qURL, err := url.Parse(c.server)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	q := qURL.Query()
+	q.Set("device", device.String())
+	qURL.RawQuery = q.Encode()
+
+	resp, err := c.queryClient.Get(qURL.String())
+	if err != nil {
+		l.Debugln("globalClient.Lookup", qURL, err)
+		return nil, nil, err
+	}
+	if resp.StatusCode != 200 {
+		resp.Body.Close()
+		l.Debugln("globalClient.Lookup", qURL, resp.Status)
+		return nil, nil, errors.New(resp.Status)
+	}
+
+	// TODO: Handle 429 and Retry-After?
+
+	var ann announcement
+	err = json.NewDecoder(resp.Body).Decode(&ann)
+	resp.Body.Close()
+	return ann.Direct, ann.Relays, err
+}
+
+func (c *globalClient) String() string {
+	return "global@" + c.server
+}
+
+func (c *globalClient) Serve() {
+	if c.noAnnounce {
+		// We're configured to not do announcements, only lookups. To maintain
+		// the same interface, we just pause here if Serve() is run.
+		<-c.stop
+		return
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+
+	eventSub := events.Default.Subscribe(events.ExternalPortMappingChanged | events.RelayStateChanged)
+	defer events.Default.Unsubscribe(eventSub)
+
+	for {
+		select {
+		case <-eventSub.C():
+			c.sendAnnouncement(timer)
+
+		case <-timer.C:
+			c.sendAnnouncement(timer)
+
+		case <-c.stop:
+			return
+		}
+	}
+}
+
+func (c *globalClient) sendAnnouncement(timer *time.Timer) {
+
+	var ann announcement
+	if c.addrList != nil {
+		ann.Direct = c.addrList.ExternalAddresses()
+	}
+
+	if c.relayStat != nil {
+		for _, relay := range c.relayStat.Relays() {
+			latency, ok := c.relayStat.RelayStatus(relay)
+			if ok {
+				ann.Relays = append(ann.Relays, Relay{
+					URL:     relay,
+					Latency: int32(latency / time.Millisecond),
+				})
+			}
+		}
+	}
+
+	if len(ann.Direct)+len(ann.Relays) == 0 {
+		c.setError(errors.New("nothing to announce"))
+		l.Debugln("Nothing to announce")
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+
+	// The marshal doesn't fail, I promise.
+	postData, _ := json.Marshal(ann)
+
+	l.Debugf("Announcement: %s", postData)
+
+	resp, err := c.announceClient.Post(c.server, "application/json", bytes.NewReader(postData))
+	if err != nil {
+		l.Debugln("announce POST:", err)
+		c.setError(err)
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+	l.Debugln("announce POST:", resp.Status)
+	resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		l.Debugln("announce POST:", resp.Status)
+		c.setError(errors.New(resp.Status))
+
+		if h := resp.Header.Get("Retry-After"); h != "" {
+			// The server has a recommendation on when we should
+			// retry. Follow it.
+			if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
+				l.Debugln("announce Retry-After:", secs, err)
+				timer.Reset(time.Duration(secs) * time.Second)
+				return
+			}
+		}
+
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+
+	c.setError(nil)
+
+	if h := resp.Header.Get("Reannounce-After"); h != "" {
+		// The server has a recommendation on when we should
+		// reannounce. Follow it.
+		if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
+			l.Debugln("announce Reannounce-After:", secs, err)
+			timer.Reset(time.Duration(secs) * time.Second)
+			return
+		}
+	}
+
+	timer.Reset(defaultReannounceInterval)
+}
+
+func (c *globalClient) Stop() {
+	close(c.stop)
+}
+
+func (c *globalClient) Cache() map[protocol.DeviceID]CacheEntry {
+	// The globalClient doesn't do caching
+	return nil
+}
+
+// parseOptions parses and strips away any ?query=val options, setting the
+// corresponding field in the serverOptions struct. Unknown query options are
+// ignored and removed.
+func parseOptions(dsn string) (server string, opts serverOptions, err error) {
+	p, err := url.Parse(dsn)
+	if err != nil {
+		return "", serverOptions{}, err
+	}
+
+	// Grab known options from the query string
+	q := p.Query()
+	opts.id = q.Get("id")
+	opts.insecure = opts.id != "" || queryBool(q, "insecure")
+	opts.noAnnounce = queryBool(q, "noannounce")
+
+	// Check for disallowed combinations
+	if p.Scheme == "http" {
+		if !opts.insecure {
+			return "", serverOptions{}, errors.New("http without insecure not supported")
+		}
+		if !opts.noAnnounce {
+			return "", serverOptions{}, errors.New("http without noannounce not supported")
+		}
+	} else if p.Scheme != "https" {
+		return "", serverOptions{}, errors.New("unsupported scheme " + p.Scheme)
+	}
+
+	// Remove the query string
+	p.RawQuery = ""
+	server = p.String()
+
+	return
+}
+
+// queryBool returns the query parameter parsed as a boolean. An empty value
+// ("?foo") is considered true, as is any value string except false
+// ("?foo=false").
+func queryBool(q url.Values, key string) bool {
+	if _, ok := q[key]; !ok {
+		return false
+	}
+
+	return q.Get(key) != "false"
+}
+
+type idCheckingHTTPClient struct {
+	httpClient
+	id protocol.DeviceID
+}
+
+func newIDCheckingHTTPClient(client httpClient, id protocol.DeviceID) *idCheckingHTTPClient {
+	return &idCheckingHTTPClient{
+		httpClient: client,
+		id:         id,
+	}
+}
+
+func (c *idCheckingHTTPClient) check(resp *http.Response) error {
+	if resp.TLS == nil {
+		return errors.New("security: not TLS")
+	}
+
+	if len(resp.TLS.PeerCertificates) == 0 {
+		return errors.New("security: no certificates")
+	}
+
+	id := protocol.NewDeviceID(resp.TLS.PeerCertificates[0].Raw)
+	if !id.Equals(c.id) {
+		return errors.New("security: incorrect device id")
+	}
+
+	return nil
+}
+
+func (c *idCheckingHTTPClient) Get(url string) (*http.Response, error) {
+	resp, err := c.httpClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if err := c.check(resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+func (c *idCheckingHTTPClient) Post(url, ctype string, data io.Reader) (*http.Response, error) {
+	resp, err := c.httpClient.Post(url, ctype, data)
+	if err != nil {
+		return nil, err
+	}
+	if err := c.check(resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+type errorHolder struct {
+	err error
+	mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
+}
+
+func (e *errorHolder) setError(err error) {
+	e.mut.Lock()
+	e.err = err
+	e.mut.Unlock()
+}
+
+func (e *errorHolder) Error() error {
+	e.mut.Lock()
+	err := e.err
+	e.mut.Unlock()
+	return err
+}

lib/discover/global_test.go

@@ -0,0 +1,259 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"crypto/tls"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+)
+
+func TestParseOptions(t *testing.T) {
+	testcases := []struct {
+		in   string
+		out  string
+		opts serverOptions
+	}{
+		{"https://example.com/", "https://example.com/", serverOptions{}},
+		{"https://example.com/?insecure", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=true", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=yes", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=false&noannounce", "https://example.com/", serverOptions{noAnnounce: true}},
+		{"https://example.com/?id=abc", "https://example.com/", serverOptions{id: "abc", insecure: true}},
+	}
+
+	for _, tc := range testcases {
+		res, opts, err := parseOptions(tc.in)
+		if err != nil {
+			t.Errorf("Unexpected err %v for %v", err, tc.in)
+			continue
+		}
+		if res != tc.out {
+			t.Errorf("Incorrect server, %v!= %v for %v", res, tc.out, tc.in)
+		}
+		if opts != tc.opts {
+			t.Errorf("Incorrect options, %v!= %v for %v", opts, tc.opts, tc.in)
+		}
+	}
+}
+
+func TestGlobalOverHTTP(t *testing.T) {
+	// HTTP works for queries, but is obviously insecure and we can't do
+	// announces over it (as we don't present a certificate). As such, http://
+	// is only allowed in combination with the "insecure" and "noannounce"
+	// parameters.
+
+	if _, err := NewGlobal("http://192.0.2.42/", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without insecure and noannounce")
+	}
+
+	if _, err := NewGlobal("http://192.0.2.42/?insecure", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without noannounce")
+	}
+
+	if _, err := NewGlobal("http://192.0.2.42/?noannounce", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without insecure")
+	}
+
+	// Now lets check that lookups work over HTTP, given the correct options.
+
+	list, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	direct, relays, err := testLookup("http://" + list.Addr().String() + "?insecure&noannounce")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+		t.Errorf("incorrect direct list: %+v", direct)
+	}
+	if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+		t.Errorf("incorrect relays list: %+v", direct)
+	}
+}
+
+func TestGlobalOverHTTPS(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate a server certificate, using fewer bits than usual to hurry the
+	// process along a bit.
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	// With default options the lookup code expects the server certificate to
+	// check out according to the usual CA chains etc. That won't be the case
+	// here so we expect the lookup to fail.
+
+	url := "https://" + list.Addr().String()
+	if _, _, err := testLookup(url); err == nil {
+		t.Fatalf("unexpected nil error when we should have got a certificate error")
+	}
+
+	// With "insecure" set, whatever certificate is on the other side should
+	// be accepted.
+
+	url = "https://" + list.Addr().String() + "?insecure"
+	if direct, relays, err := testLookup(url); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	} else {
+		if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+			t.Errorf("incorrect direct list: %+v", direct)
+		}
+		if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+			t.Errorf("incorrect relays list: %+v", direct)
+		}
+	}
+
+	// With "id" set to something incorrect, the checks should fail again.
+
+	url = "https://" + list.Addr().String() + "?id=" + protocol.LocalDeviceID.String()
+	if _, _, err := testLookup(url); err == nil {
+		t.Fatalf("unexpected nil error for incorrect discovery server ID")
+	}
+
+	// With the correct device ID, the check should pass and we should get a
+	// lookup response.
+
+	id := protocol.NewDeviceID(cert.Certificate[0])
+	url = "https://" + list.Addr().String() + "?id=" + id.String()
+	if direct, relays, err := testLookup(url); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	} else {
+		if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+			t.Errorf("incorrect direct list: %+v", direct)
+		}
+		if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+			t.Errorf("incorrect relays list: %+v", direct)
+		}
+	}
+}
+
+func TestGlobalAnnounce(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate a server certificate, using fewer bits than usual to hurry the
+	// process along a bit.
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	url := "https://" + list.Addr().String() + "?insecure"
+	disco, err := NewGlobal(url, cert, new(fakeAddressLister), new(fakeRelayStatus))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go disco.Serve()
+	defer disco.Stop()
+
+	// The discovery thing should attempt an announcement immediately. We wait
+	// for it to succeed, a while.
+	t0 := time.Now()
+	for err := disco.Error(); err != nil; err = disco.Error() {
+		if time.Since(t0) > 10*time.Second {
+			t.Fatal("announce failed:", err)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	if !strings.Contains(string(s.announce), "tcp://0.0.0.0:22000") {
+		t.Errorf("announce missing direct address: %s", s.announce)
+	}
+	if !strings.Contains(string(s.announce), "relay://192.0.2.42:443") {
+		t.Errorf("announce missing relay address: %s", s.announce)
+	}
+}
+
+func testLookup(url string) ([]string, []Relay, error) {
+	disco, err := NewGlobal(url, tls.Certificate{}, nil, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	go disco.Serve()
+	defer disco.Stop()
+
+	return disco.Lookup(protocol.LocalDeviceID)
+}
+
+type fakeDiscoveryServer struct {
+	announce []byte
+}
+
+func (s *fakeDiscoveryServer) handler(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "POST" {
+		s.announce, _ = ioutil.ReadAll(r.Body)
+		w.WriteHeader(204)
+	} else {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"direct":["tcp://192.0.2.42::22000"], "relays":[{"url": "relay://192.0.2.43:443", "latency": 42}]}`))
+	}
+}
+
+type fakeAddressLister struct{}
+
+func (f *fakeAddressLister) ExternalAddresses() []string {
+	return []string{"tcp://0.0.0.0:22000"}
+}
+func (f *fakeAddressLister) AllAddresses() []string {
+	return []string{"tcp://0.0.0.0:22000", "tcp://192.168.0.1:22000"}
+}
+
+type fakeRelayStatus struct{}
+
+func (f *fakeRelayStatus) Relays() []string {
+	return []string{"relay://192.0.2.42:443"}
+}
+func (f *fakeRelayStatus) RelayStatus(uri string) (time.Duration, bool) {
+	return 42 * time.Millisecond, true
+}

lib/discover/local.go

@@ -0,0 +1,271 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/beacon"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/thejerf/suture"
+)
+
+type localClient struct {
+	*suture.Supervisor
+	myID      protocol.DeviceID
+	addrList  AddressLister
+	relayStat RelayStatusProvider
+	name      string
+
+	beacon          beacon.Interface
+	localBcastStart time.Time
+	localBcastTick  <-chan time.Time
+	forcedBcastTick chan time.Time
+
+	*cache
+}
+
+const (
+	BroadcastInterval = 30 * time.Second
+	CacheLifeTime     = 3 * BroadcastInterval
+)
+
+var (
+	ErrIncorrectMagic = errors.New("incorrect magic number")
+)
+
+func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
+	c := &localClient{
+		Supervisor:      suture.NewSimple("local"),
+		myID:            id,
+		addrList:        addrList,
+		relayStat:       relayStat,
+		localBcastTick:  time.Tick(BroadcastInterval),
+		forcedBcastTick: make(chan time.Time),
+		localBcastStart: time.Now(),
+		cache:           newCache(),
+	}
+
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(host) == 0 {
+		// A broadcast client
+		c.name = "IPv4 local"
+		bcPort, err := strconv.Atoi(port)
+		if err != nil {
+			return nil, err
+		}
+		c.startLocalIPv4Broadcasts(bcPort)
+	} else {
+		// A multicast client
+		c.name = "IPv6 local"
+		c.startLocalIPv6Multicasts(addr)
+	}
+
+	go c.sendLocalAnnouncements()
+
+	return c, nil
+}
+
+func (c *localClient) startLocalIPv4Broadcasts(localPort int) {
+	c.beacon = beacon.NewBroadcast(localPort)
+	c.Add(c.beacon)
+	go c.recvAnnouncements(c.beacon)
+}
+
+func (c *localClient) startLocalIPv6Multicasts(localMCAddr string) {
+	c.beacon = beacon.NewMulticast(localMCAddr)
+	c.Add(c.beacon)
+	go c.recvAnnouncements(c.beacon)
+}
+
+// Lookup returns a list of addresses the device is available at. Local
+// discovery never returns relays.
+func (c *localClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	if cache, ok := c.Get(device); ok {
+		if time.Since(cache.when) < CacheLifeTime {
+			direct = cache.Direct
+			relays = cache.Relays
+		}
+	}
+
+	return
+}
+
+func (c *localClient) String() string {
+	return c.name
+}
+
+func (c *localClient) Error() error {
+	return c.beacon.Error()
+}
+
+func (c *localClient) announcementPkt() Announce {
+	var addrs []Address
+	for _, addr := range c.addrList.AllAddresses() {
+		addrs = append(addrs, Address{
+			URL: addr,
+		})
+	}
+
+	var relays []Relay
+	for _, relay := range c.relayStat.Relays() {
+		latency, ok := c.relayStat.RelayStatus(relay)
+		if ok {
+			relays = append(relays, Relay{
+				URL:     relay,
+				Latency: int32(latency / time.Millisecond),
+			})
+		}
+	}
+
+	return Announce{
+		Magic: AnnouncementMagic,
+		This: Device{
+			ID:        c.myID[:],
+			Addresses: addrs,
+			Relays:    relays,
+		},
+	}
+}
+
+func (c *localClient) sendLocalAnnouncements() {
+	var pkt = c.announcementPkt()
+	msg := pkt.MustMarshalXDR()
+
+	for {
+		c.beacon.Send(msg)
+
+		select {
+		case <-c.localBcastTick:
+		case <-c.forcedBcastTick:
+		}
+	}
+}
+
+func (c *localClient) recvAnnouncements(b beacon.Interface) {
+	for {
+		buf, addr := b.Recv()
+
+		var pkt Announce
+		err := pkt.UnmarshalXDR(buf)
+		if err != nil && err != io.EOF {
+			l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
+			continue
+		}
+
+		l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
+
+		var newDevice bool
+		if bytes.Compare(pkt.This.ID, c.myID[:]) != 0 {
+			newDevice = c.registerDevice(addr, pkt.This)
+		}
+
+		if newDevice {
+			select {
+			case c.forcedBcastTick <- time.Now():
+			}
+		}
+	}
+}
+
+func (c *localClient) registerDevice(src net.Addr, device Device) bool {
+	var id protocol.DeviceID
+	copy(id[:], device.ID)
+
+	// Remember whether we already had a valid cache entry for this device.
+
+	ce, existsAlready := c.Get(id)
+	isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime
+
+	// Any empty or unspecified addresses should be set to the source address
+	// of the announcement. We also skip any addresses we can't parse.
+
+	var validAddresses []string
+	for _, addr := range device.Addresses {
+		u, err := url.Parse(addr.URL)
+		if err != nil {
+			continue
+		}
+
+		tcpAddr, err := net.ResolveTCPAddr("tcp", u.Host)
+		if err != nil {
+			continue
+		}
+
+		if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
+			host, _, err := net.SplitHostPort(src.String())
+			if err != nil {
+				continue
+			}
+			u.Host = net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port))
+			validAddresses = append(validAddresses, u.String())
+		} else {
+			validAddresses = append(validAddresses, addr.URL)
+		}
+	}
+
+	c.Set(id, CacheEntry{
+		Direct: validAddresses,
+		Relays: device.Relays,
+		when:   time.Now(),
+		found:  true,
+	})
+
+	if isNewDevice {
+		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
+			"device": id.String(),
+			"addrs":  device.Addresses,
+			"relays": device.Relays,
+		})
+	}
+
+	return isNewDevice
+}
+
+func addrToAddr(addr *net.TCPAddr) string {
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		return fmt.Sprintf(":%c", addr.Port)
+	} else if bs := addr.IP.To4(); bs != nil {
+		return net.JoinHostPort(bs.String(), strconv.Itoa(addr.Port))
+	} else if bs := addr.IP.To16(); bs != nil {
+		return net.JoinHostPort(bs.String(), strconv.Itoa(addr.Port))
+	}
+	return ""
+}
+
+func resolveAddrs(addrs []string) []string {
+	var raddrs []string
+	for _, addrStr := range addrs {
+		uri, err := url.Parse(addrStr)
+		if err != nil {
+			continue
+		}
+		addrRes, err := net.ResolveTCPAddr("tcp", uri.Host)
+		if err != nil {
+			continue
+		}
+		addr := addrToAddr(addrRes)
+		if len(addr) > 0 {
+			uri.Host = addr
+			raddrs = append(raddrs, uri.String())
+		}
+	}
+	return raddrs
+}

lib/discover/localpackets.go

@@ -5,20 +5,14 @@
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
 //go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o packets_xdr.go packets.go
+//go:generate genxdr -o localpackets_xdr.go localpackets.go
 
 package discover
 
 const (
-	AnnouncementMagic = 0x9D79BC39
-	QueryMagic        = 0x2CA856F5
+	AnnouncementMagic = 0x9D79BC40
 )
 
-type Query struct {
-	Magic    uint32
-	DeviceID []byte // max:32
-}
-
 type Announce struct {
 	Magic uint32
 	This  Device
@@ -28,9 +22,14 @@ type Announce struct {
 type Device struct {
 	ID        []byte    // max:32
 	Addresses []Address // max:16
+	Relays    []Relay   // max:16
 }
 
 type Address struct {
-	IP   []byte // max:16
-	Port uint16
+	URL string // max:2083
+}
+
+type Relay struct {
+	URL     string `json:"url"` // max:2083
+	Latency int32  `json:"latency"`
 }

lib/discover/localpackets_xdr.go

@@ -13,80 +13,6 @@ import (
 
 /*
 
-Query Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Magic                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                      Length of Device ID                      |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                  Device ID (variable length)                  \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Query {
-	unsigned int Magic;
-	opaque DeviceID<32>;
-}
-
-*/
-
-func (o Query) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
-}
-
-func (o Query) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o Query) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Query) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
-}
-
-func (o Query) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	xw.WriteUint32(o.Magic)
-	if l := len(o.DeviceID); l > 32 {
-		return xw.Tot(), xdr.ElementSizeExceeded("DeviceID", l, 32)
-	}
-	xw.WriteBytes(o.DeviceID)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *Query) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *Query) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *Query) DecodeXDRFrom(xr *xdr.Reader) error {
-	o.Magic = xr.ReadUint32()
-	o.DeviceID = xr.ReadBytesMax(32)
-	return xr.Error()
-}
-
-/*
-
 Announce Structure:
 
  0                   1                   2                   3
@@ -204,11 +130,18 @@ Device Structure:
 \                Zero or more Address Structures                \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Relays                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                 Zero or more Relay Structures                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct Device {
 	opaque ID<32>;
 	Address Addresses<16>;
+	Relay Relays<16>;
 }
 
 */
@@ -252,6 +185,16 @@ func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 			return xw.Tot(), err
 		}
 	}
+	if l := len(o.Relays); l > 16 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Relays", l, 16)
+	}
+	xw.WriteUint32(uint32(len(o.Relays)))
+	for i := range o.Relays {
+		_, err := o.Relays[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
 	return xw.Tot(), xw.Error()
 }
 
@@ -279,6 +222,17 @@ func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error {
 	for i := range o.Addresses {
 		(&o.Addresses[i]).DecodeXDRFrom(xr)
 	}
+	_RelaysSize := int(xr.ReadUint32())
+	if _RelaysSize < 0 {
+		return xdr.ElementSizeExceeded("Relays", _RelaysSize, 16)
+	}
+	if _RelaysSize > 16 {
+		return xdr.ElementSizeExceeded("Relays", _RelaysSize, 16)
+	}
+	o.Relays = make([]Relay, _RelaysSize)
+	for i := range o.Relays {
+		(&o.Relays[i]).DecodeXDRFrom(xr)
+	}
 	return xr.Error()
 }
 
@@ -289,19 +243,16 @@ Address Structure:
  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of IP                          |
+|                         Length of URL                         |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 /                                                               /
-\                     IP (variable length)                      \
+\                     URL (variable length)                     \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |             Port              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct Address {
-	opaque IP<16>;
-	unsigned int Port;
+	string URL<2083>;
 }
 
 */
@@ -331,11 +282,10 @@ func (o Address) AppendXDR(bs []byte) ([]byte, error) {
 }
 
 func (o Address) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	if l := len(o.IP); l > 16 {
-		return xw.Tot(), xdr.ElementSizeExceeded("IP", l, 16)
+	if l := len(o.URL); l > 2083 {
+		return xw.Tot(), xdr.ElementSizeExceeded("URL", l, 2083)
 	}
-	xw.WriteBytes(o.IP)
-	xw.WriteUint16(o.Port)
+	xw.WriteString(o.URL)
 	return xw.Tot(), xw.Error()
 }
 
@@ -351,7 +301,80 @@ func (o *Address) UnmarshalXDR(bs []byte) error {
 }
 
 func (o *Address) DecodeXDRFrom(xr *xdr.Reader) error {
-	o.IP = xr.ReadBytesMax(16)
-	o.Port = xr.ReadUint16()
+	o.URL = xr.ReadStringMax(2083)
+	return xr.Error()
+}
+
+/*
+
+Relay Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of URL                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     URL (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                            Latency                            |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Relay {
+	string URL<2083>;
+	int Latency;
+}
+
+*/
+
+func (o Relay) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Relay) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Relay) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Relay) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Relay) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	if l := len(o.URL); l > 2083 {
+		return xw.Tot(), xdr.ElementSizeExceeded("URL", l, 2083)
+	}
+	xw.WriteString(o.URL)
+	xw.WriteUint32(uint32(o.Latency))
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Relay) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Relay) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Relay) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.URL = xr.ReadStringMax(2083)
+	o.Latency = int32(xr.ReadUint32())
 	return xr.Error()
 }

lib/events/debug.go

@@ -10,10 +10,13 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/logger"
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "events") || os.Getenv("STTRACE") == "all"
-	dl    = logger.DefaultLogger
+	dl = logger.DefaultLogger.NewFacility("events", "Event generation and logging")
 )
+
+func init() {
+	dl.SetDebug("events", strings.Contains(os.Getenv("STTRACE"), "events") || os.Getenv("STTRACE") == "all")
+}

lib/events/events.go

@@ -25,6 +25,8 @@ const (
 	DeviceConnected
 	DeviceDisconnected
 	DeviceRejected
+	DevicePaused
+	DeviceResumed
 	LocalIndexUpdated
 	RemoteIndexUpdated
 	ItemStarted
@@ -36,6 +38,9 @@ const (
 	FolderSummary
 	FolderCompletion
 	FolderErrors
+	FolderScanProgress
+	ExternalPortMappingChanged
+	RelayStateChanged
 
 	AllEvents = (1 << iota) - 1
 )
@@ -78,6 +83,16 @@ func (t EventType) String() string {
 		return "FolderCompletion"
 	case FolderErrors:
 		return "FolderErrors"
+	case DevicePaused:
+		return "DevicePaused"
+	case DeviceResumed:
+		return "DeviceResumed"
+	case FolderScanProgress:
+		return "FolderScanProgress"
+	case ExternalPortMappingChanged:
+		return "ExternalPortMappingChanged"
+	case RelayStateChanged:
+		return "RelayStateChanged"
 	default:
 		return "Unknown"
 	}
@@ -123,9 +138,7 @@ func NewLogger() *Logger {
 
 func (l *Logger) Log(t EventType, data interface{}) {
 	l.mutex.Lock()
-	if debug {
-		dl.Debugln("log", l.nextID, t.String(), data)
-	}
+	dl.Debugln("log", l.nextID, t, data)
 	l.nextID++
 	e := Event{
 		ID:   l.nextID,
@@ -147,9 +160,7 @@ func (l *Logger) Log(t EventType, data interface{}) {
 
 func (l *Logger) Subscribe(mask EventType) *Subscription {
 	l.mutex.Lock()
-	if debug {
-		dl.Debugln("subscribe", mask)
-	}
+	dl.Debugln("subscribe", mask)
 	s := &Subscription{
 		mask:    mask,
 		events:  make(chan Event, BufferSize),
@@ -162,9 +173,7 @@ func (l *Logger) Subscribe(mask EventType) *Subscription {
 
 func (l *Logger) Unsubscribe(s *Subscription) {
 	l.mutex.Lock()
-	if debug {
-		dl.Debugln("unsubscribe")
-	}
+	dl.Debugln("unsubscribe")
 	for i, ss := range l.subs {
 		if s == ss {
 			last := len(l.subs) - 1
@@ -182,9 +191,7 @@ func (l *Logger) Unsubscribe(s *Subscription) {
 // out of the event channel is closed. Poll should not be called concurrently
 // from multiple goroutines for a single subscription.
 func (s *Subscription) Poll(timeout time.Duration) (Event, error) {
-	if debug {
-		dl.Debugln("poll", timeout)
-	}
+	dl.Debugln("poll", timeout)
 
 	if !s.timeout.Reset(timeout) {
 		select {

lib/fnmatch/fnmatch.go

@@ -27,43 +27,43 @@ func Convert(pattern string, flags int) (*regexp.Regexp, error) {
 		flags |= NoEscape | CaseFold
 		pattern = filepath.FromSlash(pattern)
 		if flags&PathName != 0 {
-			any = "[^\\\\]"
+			any = `[^\\]`
 		}
 	case "darwin":
 		flags |= CaseFold
 		fallthrough
 	default:
 		if flags&PathName != 0 {
-			any = "[^/]"
+			any = `[^/]`
 		}
 	}
 
 	if flags&NoEscape != 0 {
-		pattern = strings.Replace(pattern, "\\", "\\\\", -1)
+		pattern = strings.Replace(pattern, `\`, `\\`, -1)
 	} else {
-		pattern = strings.Replace(pattern, "\\*", "[:escapedstar:]", -1)
-		pattern = strings.Replace(pattern, "\\?", "[:escapedques:]", -1)
-		pattern = strings.Replace(pattern, "\\.", "[:escapeddot:]", -1)
+		pattern = strings.Replace(pattern, `\*`, "[:escapedstar:]", -1)
+		pattern = strings.Replace(pattern, `\?`, "[:escapedques:]", -1)
+		pattern = strings.Replace(pattern, `\.`, "[:escapeddot:]", -1)
 	}
 
 	// Characters that are special in regexps but not in glob, must be
 	// escaped.
-	for _, char := range []string{".", "+", "$", "^", "(", ")", "|"} {
-		pattern = strings.Replace(pattern, char, "\\"+char, -1)
+	for _, char := range []string{`.`, `+`, `$`, `^`, `(`, `)`, `|`} {
+		pattern = strings.Replace(pattern, char, `\`+char, -1)
 	}
 
-	pattern = strings.Replace(pattern, "**", "[:doublestar:]", -1)
-	pattern = strings.Replace(pattern, "*", any+"*", -1)
-	pattern = strings.Replace(pattern, "[:doublestar:]", ".*", -1)
-	pattern = strings.Replace(pattern, "?", any, -1)
+	pattern = strings.Replace(pattern, `**`, `[:doublestar:]`, -1)
+	pattern = strings.Replace(pattern, `*`, any+`*`, -1)
+	pattern = strings.Replace(pattern, `[:doublestar:]`, `.*`, -1)
+	pattern = strings.Replace(pattern, `?`, any, -1)
 
-	pattern = strings.Replace(pattern, "[:escapedstar:]", "\\*", -1)
-	pattern = strings.Replace(pattern, "[:escapedques:]", "\\?", -1)
-	pattern = strings.Replace(pattern, "[:escapeddot:]", "\\.", -1)
+	pattern = strings.Replace(pattern, `[:escapedstar:]`, `\*`, -1)
+	pattern = strings.Replace(pattern, `[:escapedques:]`, `\?`, -1)
+	pattern = strings.Replace(pattern, `[:escapeddot:]`, `\.`, -1)
 
-	pattern = "^" + pattern + "$"
+	pattern = `^` + pattern + `$`
 	if flags&CaseFold != 0 {
-		pattern = "(?i)" + pattern
+		pattern = `(?i)` + pattern
 	}
 	return regexp.Compile(pattern)
 }

lib/logger/logger.go

@@ -0,0 +1,379 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// Package logger implements a standardized logger with callback functionality
+package logger
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+// This package uses stdlib sync as it may be used to debug syncthing/lib/sync
+// and that would cause an implosion of the universe.
+
+type LogLevel int
+
+const (
+	LevelDebug LogLevel = iota
+	LevelVerbose
+	LevelInfo
+	LevelOK
+	LevelWarn
+	LevelFatal
+	NumLevels
+)
+
+// A MessageHandler is called with the log level and message text.
+type MessageHandler func(l LogLevel, msg string)
+
+type Logger interface {
+	AddHandler(level LogLevel, h MessageHandler)
+	SetFlags(flag int)
+	SetPrefix(prefix string)
+	Debugln(vals ...interface{})
+	Debugf(format string, vals ...interface{})
+	Verboseln(vals ...interface{})
+	Verbosef(format string, vals ...interface{})
+	Infoln(vals ...interface{})
+	Infof(format string, vals ...interface{})
+	Okln(vals ...interface{})
+	Okf(format string, vals ...interface{})
+	Warnln(vals ...interface{})
+	Warnf(format string, vals ...interface{})
+	Fatalln(vals ...interface{})
+	Fatalf(format string, vals ...interface{})
+	ShouldDebug(facility string) bool
+	SetDebug(facility string, enabled bool)
+	Facilities() map[string]string
+	FacilityDebugging() []string
+	NewFacility(facility, description string) Logger
+}
+
+type logger struct {
+	logger     *log.Logger
+	handlers   [NumLevels][]MessageHandler
+	facilities map[string]string // facility name => description
+	debug      map[string]bool   // facility name => debugging enabled
+	mut        sync.Mutex
+}
+
+// The default logger logs to standard output with a time prefix.
+var DefaultLogger = New()
+
+func New() Logger {
+	if os.Getenv("LOGGER_DISCARD") != "" {
+		// Hack to completely disable logging, for example when running benchmarks.
+		return &logger{
+			logger: log.New(ioutil.Discard, "", 0),
+		}
+	}
+
+	return &logger{
+		logger: log.New(os.Stdout, "", log.Ltime),
+	}
+}
+
+// AddHandler registers a new MessageHandler to receive messages with the
+// specified log level or above.
+func (l *logger) AddHandler(level LogLevel, h MessageHandler) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	l.handlers[level] = append(l.handlers[level], h)
+}
+
+// See log.SetFlags
+func (l *logger) SetFlags(flag int) {
+	l.logger.SetFlags(flag)
+}
+
+// See log.SetPrefix
+func (l *logger) SetPrefix(prefix string) {
+	l.logger.SetPrefix(prefix)
+}
+
+func (l *logger) callHandlers(level LogLevel, s string) {
+	for ll := LevelDebug; ll <= level; ll++ {
+		for _, h := range l.handlers[ll] {
+			h(level, strings.TrimSpace(s))
+		}
+	}
+}
+
+// Debugln logs a line with a DEBUG prefix.
+func (l *logger) Debugln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+
+// Debugf logs a formatted line with a DEBUG prefix.
+func (l *logger) Debugf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+
+// Infoln logs a line with a VERBOSE prefix.
+func (l *logger) Verboseln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "VERBOSE: "+s)
+	l.callHandlers(LevelVerbose, s)
+}
+
+// Infof logs a formatted line with a VERBOSE prefix.
+func (l *logger) Verbosef(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "VERBOSE: "+s)
+	l.callHandlers(LevelVerbose, s)
+}
+
+// Infoln logs a line with an INFO prefix.
+func (l *logger) Infoln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+// Infof logs a formatted line with an INFO prefix.
+func (l *logger) Infof(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+// Okln logs a line with an OK prefix.
+func (l *logger) Okln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+// Okf logs a formatted line with an OK prefix.
+func (l *logger) Okf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+// Warnln logs a formatted line with a WARNING prefix.
+func (l *logger) Warnln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+// Warnf logs a formatted line with a WARNING prefix.
+func (l *logger) Warnf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+// Fatalln logs a line with a FATAL prefix and exits the process with exit
+// code 1.
+func (l *logger) Fatalln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(1)
+}
+
+// Fatalf logs a formatted line with a FATAL prefix and exits the process with
+// exit code 1.
+func (l *logger) Fatalf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(1)
+}
+
+// ShouldDebug returns true if the given facility has debugging enabled.
+func (l *logger) ShouldDebug(facility string) bool {
+	l.mut.Lock()
+	res := l.debug[facility]
+	l.mut.Unlock()
+	return res
+}
+
+// SetDebug enabled or disables debugging for the given facility name.
+func (l *logger) SetDebug(facility string, enabled bool) {
+	l.mut.Lock()
+	l.debug[facility] = enabled
+	l.mut.Unlock()
+}
+
+// FacilityDebugging returns the set of facilities that have debugging
+// enabled.
+func (l *logger) FacilityDebugging() []string {
+	var enabled []string
+	l.mut.Lock()
+	for facility, isEnabled := range l.debug {
+		if isEnabled {
+			enabled = append(enabled, facility)
+		}
+	}
+	l.mut.Unlock()
+	return enabled
+}
+
+// Facilities returns the currently known set of facilities and their
+// descriptions.
+func (l *logger) Facilities() map[string]string {
+	l.mut.Lock()
+	res := make(map[string]string, len(l.facilities))
+	for facility, descr := range l.facilities {
+		res[facility] = descr
+	}
+	l.mut.Unlock()
+	return res
+}
+
+// NewFacility returns a new logger bound to the named facility.
+func (l *logger) NewFacility(facility, description string) Logger {
+	l.mut.Lock()
+	if l.facilities == nil {
+		l.facilities = make(map[string]string)
+	}
+	if description != "" {
+		l.facilities[facility] = description
+	}
+
+	if l.debug == nil {
+		l.debug = make(map[string]bool)
+	}
+	l.debug[facility] = false
+	l.mut.Unlock()
+
+	return &facilityLogger{
+		logger:   l,
+		facility: facility,
+	}
+}
+
+// A facilityLogger is a regular logger but bound to a facility name. The
+// Debugln and Debugf methods are no-ops unless debugging has been enabled for
+// this facility on the parent logger.
+type facilityLogger struct {
+	*logger
+	facility string
+}
+
+// Debugln logs a line with a DEBUG prefix.
+func (l *facilityLogger) Debugln(vals ...interface{}) {
+	if !l.ShouldDebug(l.facility) {
+		return
+	}
+	l.logger.Debugln(vals...)
+}
+
+// Debugf logs a formatted line with a DEBUG prefix.
+func (l *facilityLogger) Debugf(format string, vals ...interface{}) {
+	if !l.ShouldDebug(l.facility) {
+		return
+	}
+	l.logger.Debugf(format, vals...)
+}
+
+// A Recorder keeps a size limited record of log events.
+type Recorder struct {
+	lines   []Line
+	initial int
+	mut     sync.Mutex
+}
+
+// A Line represents a single log entry.
+type Line struct {
+	When    time.Time
+	Message string
+}
+
+func NewRecorder(l Logger, level LogLevel, size, initial int) *Recorder {
+	r := &Recorder{
+		lines:   make([]Line, 0, size),
+		initial: initial,
+	}
+	l.AddHandler(level, r.append)
+	return r
+}
+
+func (r *Recorder) Since(t time.Time) []Line {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	res := r.lines
+	for i := 0; i < len(res) && res[i].When.Before(t); i++ {
+		// nothing, just incrementing i
+	}
+	if len(res) == 0 {
+		return nil
+	}
+
+	// We must copy the result as r.lines can be mutated as soon as the lock
+	// is released.
+	cp := make([]Line, len(res))
+	copy(cp, res)
+	return cp
+}
+
+func (r *Recorder) Clear() {
+	r.mut.Lock()
+	r.lines = r.lines[:0]
+	r.mut.Unlock()
+}
+
+func (r *Recorder) append(l LogLevel, msg string) {
+	line := Line{
+		When:    time.Now(),
+		Message: msg,
+	}
+
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	if len(r.lines) == cap(r.lines) {
+		if r.initial > 0 {
+			// Shift all lines one step to the left, keeping the "initial" first intact.
+			copy(r.lines[r.initial+1:], r.lines[r.initial+2:])
+		} else {
+			copy(r.lines, r.lines[1:])
+		}
+		// Add the new one at the end
+		r.lines[len(r.lines)-1] = line
+		return
+	}
+
+	r.lines = append(r.lines, line)
+	if len(r.lines) == r.initial {
+		r.lines = append(r.lines, Line{time.Now(), "..."})
+	}
+}

lib/logger/logger_test.go

@@ -0,0 +1,140 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package logger
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestAPI(t *testing.T) {
+	l := New()
+	l.SetFlags(0)
+	l.SetPrefix("testing")
+
+	debug := 0
+	l.AddHandler(LevelDebug, checkFunc(t, LevelDebug, &debug))
+	info := 0
+	l.AddHandler(LevelInfo, checkFunc(t, LevelInfo, &info))
+	ok := 0
+	l.AddHandler(LevelOK, checkFunc(t, LevelOK, &ok))
+	warn := 0
+	l.AddHandler(LevelWarn, checkFunc(t, LevelWarn, &warn))
+
+	l.Debugf("test %d", 0)
+	l.Debugln("test", 0)
+	l.Infof("test %d", 1)
+	l.Infoln("test", 1)
+	l.Okf("test %d", 2)
+	l.Okln("test", 2)
+	l.Warnf("test %d", 3)
+	l.Warnln("test", 3)
+
+	if debug != 8 {
+		t.Errorf("Debug handler called %d != 8 times", debug)
+	}
+	if info != 6 {
+		t.Errorf("Info handler called %d != 6 times", info)
+	}
+	if ok != 4 {
+		t.Errorf("Ok handler called %d != 4 times", ok)
+	}
+	if warn != 2 {
+		t.Errorf("Warn handler called %d != 2 times", warn)
+	}
+}
+
+func checkFunc(t *testing.T, expectl LogLevel, counter *int) func(LogLevel, string) {
+	return func(l LogLevel, msg string) {
+		*counter++
+		if l < expectl {
+			t.Errorf("Incorrect message level %d < %d", l, expectl)
+		}
+	}
+}
+
+func TestFacilityDebugging(t *testing.T) {
+	l := New()
+	l.SetFlags(0)
+
+	msgs := 0
+	l.AddHandler(LevelDebug, func(l LogLevel, msg string) {
+		msgs++
+		if strings.Contains(msg, "f1") {
+			t.Fatal("Should not get message for facility f1")
+		}
+	})
+
+	f0 := l.NewFacility("f0", "foo#0")
+	f1 := l.NewFacility("f1", "foo#1")
+
+	l.SetDebug("f0", true)
+	l.SetDebug("f1", false)
+
+	f0.Debugln("Debug line from f0")
+	f1.Debugln("Debug line from f1")
+
+	if msgs != 1 {
+		t.Fatalf("Incorrent number of messages, %d != 1", msgs)
+	}
+}
+
+func TestRecorder(t *testing.T) {
+	l := New()
+	l.SetFlags(0)
+
+	// Keep the last five warnings or higher, no special initial handling.
+	r0 := NewRecorder(l, LevelWarn, 5, 0)
+	// Keep the last ten infos or higher, with the first three being permanent.
+	r1 := NewRecorder(l, LevelInfo, 10, 3)
+
+	// Log a bunch of messages.
+	for i := 0; i < 15; i++ {
+		l.Debugf("Debug#%d", i)
+		l.Infof("Info#%d", i)
+		l.Warnf("Warn#%d", i)
+	}
+
+	// r0 should contain the last five warnings
+	lines := r0.Since(time.Time{})
+	if len(lines) != 5 {
+		t.Fatalf("Incorrect length %d != 5", len(lines))
+	}
+	for i := 0; i < 5; i++ {
+		expected := fmt.Sprintf("Warn#%d", i+10)
+		if lines[i].Message != expected {
+			t.Error("Incorrect warning in r0:", lines[i].Message, "!=", expected)
+		}
+	}
+
+	// r0 should contain:
+	// - The first three messages
+	// - A "..." marker
+	// - The last six messages
+	// (totalling ten)
+	lines = r1.Since(time.Time{})
+	if len(lines) != 10 {
+		t.Fatalf("Incorrect length %d != 10", len(lines))
+	}
+	expected := []string{
+		"Info#0",
+		"Warn#0",
+		"Info#1",
+		"...",
+		"Info#12",
+		"Warn#12",
+		"Info#13",
+		"Warn#13",
+		"Info#14",
+		"Warn#14",
+	}
+	for i := 0; i < 10; i++ {
+		if lines[i].Message != expected[i] {
+			t.Error("Incorrect warning in r0:", lines[i].Message, "!=", expected[i])
+		}
+	}
+
+}

lib/model/connection.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"crypto/tls"
+	"net"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type IntermediateConnection struct {
+	*tls.Conn
+	Type ConnectionType
+}
+
+type Connection struct {
+	net.Conn
+	protocol.Connection
+	Type ConnectionType
+}
+
+const (
+	ConnectionTypeDirectAccept ConnectionType = iota
+	ConnectionTypeDirectDial
+	ConnectionTypeRelayAccept
+	ConnectionTypeRelayDial
+)
+
+type ConnectionType int
+
+func (t ConnectionType) String() string {
+	switch t {
+	case ConnectionTypeDirectAccept:
+		return "direct-accept"
+	case ConnectionTypeDirectDial:
+		return "direct-dial"
+	case ConnectionTypeRelayAccept:
+		return "relay-accept"
+	case ConnectionTypeRelayDial:
+		return "relay-dial"
+	}
+	return "unknown"
+}
+
+func (t ConnectionType) IsDirect() bool {
+	return t == ConnectionTypeDirectAccept || t == ConnectionTypeDirectDial
+}

lib/model/debug.go

@@ -10,10 +10,17 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/logger"
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "model") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
+	l = logger.DefaultLogger.NewFacility("model", "The root hub")
 )
+
+func init() {
+	l.SetDebug("model", strings.Contains(os.Getenv("STTRACE"), "model") || os.Getenv("STTRACE") == "all")
+}
+
+func shouldDebug() bool {
+	return l.ShouldDebug("model")
+}

lib/model/deviceactivity.go

@@ -7,7 +7,7 @@
 package model
 
 import (
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 

lib/model/deviceactivity_test.go

@@ -9,7 +9,7 @@ package model
 import (
 	"testing"
 
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 func TestDeviceActivity(t *testing.T) {

lib/model/model.go

@@ -22,12 +22,12 @@ import (
 	stdsync "sync"
 	"time"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/stats"
 	"github.com/syncthing/syncthing/lib/symlinks"
@@ -87,10 +87,10 @@ type Model struct {
 	folderStatRefs map[string]*stats.FolderStatisticsReference            // folder -> statsRef
 	fmut           sync.RWMutex                                           // protects the above
 
-	protoConn map[protocol.DeviceID]protocol.Connection
-	rawConn   map[protocol.DeviceID]io.Closer
-	deviceVer map[protocol.DeviceID]string
-	pmut      sync.RWMutex // protects protoConn and rawConn
+	conn         map[protocol.DeviceID]Connection
+	deviceVer    map[protocol.DeviceID]string
+	devicePaused map[protocol.DeviceID]bool
+	pmut         sync.RWMutex // protects the above
 
 	reqValidationCache map[string]time.Time // folder / file name => time when confirmed to exist
 	rvmut              sync.RWMutex         // protects reqValidationCache
@@ -107,14 +107,12 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 	m := &Model{
 		Supervisor: suture.New("model", suture.Spec{
 			Log: func(line string) {
-				if debug {
-					l.Debugln(line)
-				}
+				l.Debugln(line)
 			},
 		}),
 		cfg:                cfg,
 		db:                 ldb,
-		finder:             db.NewBlockFinder(ldb, cfg),
+		finder:             db.NewBlockFinder(ldb),
 		progressEmitter:    NewProgressEmitter(cfg),
 		id:                 id,
 		shortID:            id.Short(),
@@ -130,9 +128,9 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		folderIgnores:      make(map[string]*ignore.Matcher),
 		folderRunners:      make(map[string]service),
 		folderStatRefs:     make(map[string]*stats.FolderStatisticsReference),
-		protoConn:          make(map[protocol.DeviceID]protocol.Connection),
-		rawConn:            make(map[protocol.DeviceID]io.Closer),
+		conn:               make(map[protocol.DeviceID]Connection),
 		deviceVer:          make(map[protocol.DeviceID]string),
+		devicePaused:       make(map[protocol.DeviceID]bool),
 		reqValidationCache: make(map[string]time.Time),
 
 		fmut:  sync.NewRWMutex(),
@@ -219,8 +217,11 @@ func (m *Model) StartFolderRO(folder string) {
 
 type ConnectionInfo struct {
 	protocol.Statistics
+	Connected     bool
+	Paused        bool
 	Address       string
 	ClientVersion string
+	Type          ConnectionType
 }
 
 func (info ConnectionInfo) MarshalJSON() ([]byte, error) {
@@ -228,8 +229,11 @@ func (info ConnectionInfo) MarshalJSON() ([]byte, error) {
 		"at":            info.At,
 		"inBytesTotal":  info.InBytesTotal,
 		"outBytesTotal": info.OutBytesTotal,
+		"connected":     info.Connected,
+		"paused":        info.Paused,
 		"address":       info.Address,
 		"clientVersion": info.ClientVersion,
+		"type":          info.Type.String(),
 	})
 }
 
@@ -242,15 +246,21 @@ func (m *Model) ConnectionStats() map[string]interface{} {
 	m.pmut.RLock()
 	m.fmut.RLock()
 
-	var res = make(map[string]interface{})
-	conns := make(map[string]ConnectionInfo, len(m.protoConn))
-	for device, conn := range m.protoConn {
+	res := make(map[string]interface{})
+	devs := m.cfg.Devices()
+	conns := make(map[string]ConnectionInfo, len(devs))
+	for device := range devs {
 		ci := ConnectionInfo{
-			Statistics:    conn.Statistics(),
 			ClientVersion: m.deviceVer[device],
+			Paused:        m.devicePaused[device],
 		}
-		if nc, ok := m.rawConn[device].(remoteAddrer); ok {
-			ci.Address = nc.RemoteAddr().String()
+		if conn, ok := m.conn[device]; ok {
+			ci.Type = conn.Type
+			ci.Connected = ok
+			ci.Statistics = conn.Statistics()
+			if addr := conn.RemoteAddr(); addr != nil {
+				ci.Address = addr.String()
+			}
 		}
 
 		conns[device.String()] = ci
@@ -323,9 +333,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 	})
 
 	res := 100 * (1 - float64(need)/float64(tot))
-	if debug {
-		l.Debugf("%v Completion(%s, %q): %f (%d / %d)", m, device, folder, res, need, tot)
-	}
+	l.Debugf("%v Completion(%s, %q): %f (%d / %d)", m, device, folder, res, need, tot)
 
 	return res
 }
@@ -400,9 +408,7 @@ func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) {
 		})
 	}
 	bytes -= m.progressEmitter.BytesCompleted(folder)
-	if debug {
-		l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
-	}
+	l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
 	return
 }
 
@@ -481,9 +487,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 		return
 	}
 
-	if debug {
-		l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
-	}
+	l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
 		events.Default.Log(events.FolderRejected, map[string]string{
@@ -529,9 +533,7 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
 		return
 	}
 
-	if debug {
-		l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
-	}
+	l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
 		l.Infof("Update for unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder, deviceID)
@@ -586,8 +588,12 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 		"clientVersion": cm.ClientVersion,
 	}
 
-	if conn, ok := m.rawConn[deviceID].(*tls.Conn); ok {
-		event["addr"] = conn.RemoteAddr().String()
+	if conn, ok := m.conn[deviceID]; ok {
+		event["type"] = conn.Type.String()
+		addr := conn.RemoteAddr()
+		if addr != nil {
+			event["addr"] = addr.String()
+		}
 	}
 
 	m.pmut.Unlock()
@@ -693,12 +699,11 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 	}
 	m.fmut.RUnlock()
 
-	conn, ok := m.rawConn[device]
+	conn, ok := m.conn[device]
 	if ok {
 		closeRawConn(conn)
 	}
-	delete(m.protoConn, device)
-	delete(m.rawConn, device)
+	delete(m.conn, device)
 	delete(m.deviceVer, device)
 	m.pmut.Unlock()
 }
@@ -751,16 +756,12 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		}
 
 		if lf.IsInvalid() || lf.IsDeleted() {
-			if debug {
-				l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, deviceID, folder, name, offset, len(buf), lf)
-			}
+			l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, deviceID, folder, name, offset, len(buf), lf)
 			return protocol.ErrInvalid
 		}
 
 		if offset > lf.Size() {
-			if debug {
-				l.Debugf("%v REQ(in; nonexistent): %s: %q o=%d s=%d", m, deviceID, name, offset, len(buf))
-			}
+			l.Debugf("%v REQ(in; nonexistent): %s: %q o=%d s=%d", m, deviceID, name, offset, len(buf))
 			return protocol.ErrNoSuchFile
 		}
 
@@ -791,7 +792,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		m.rvmut.Unlock()
 	}
 
-	if debug && deviceID != protocol.LocalDeviceID {
+	if deviceID != protocol.LocalDeviceID {
 		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf))
 	}
 	m.fmut.RLock()
@@ -860,7 +861,7 @@ func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
 // ConnectedTo returns true if we are connected to the named device.
 func (m *Model) ConnectedTo(deviceID protocol.DeviceID) bool {
 	m.pmut.RLock()
-	_, ok := m.protoConn[deviceID]
+	_, ok := m.conn[deviceID]
 	m.pmut.RUnlock()
 	if ok {
 		m.deviceWasSeen(deviceID)
@@ -906,7 +907,9 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 		return fmt.Errorf("Folder %s does not exist", folder)
 	}
 
-	fd, err := osutil.CreateAtomic(filepath.Join(cfg.Path(), ".stignore"), 0644)
+	path := filepath.Join(cfg.Path(), ".stignore")
+
+	fd, err := osutil.CreateAtomic(path, 0644)
 	if err != nil {
 		l.Warnln("Saving .stignore:", err)
 		return err
@@ -920,6 +923,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 		l.Warnln("Saving .stignore:", err)
 		return err
 	}
+	osutil.HideFile(path)
 
 	return m.ScanFolder(folder)
 }
@@ -927,28 +931,24 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 // AddConnection adds a new peer connection to the model. An initial index will
 // be sent to the connected peer, thereafter index updates whenever the local
 // folder changes.
-func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
-	deviceID := protoConn.ID()
+func (m *Model) AddConnection(conn Connection) {
+	deviceID := conn.ID()
 
 	m.pmut.Lock()
-	if _, ok := m.protoConn[deviceID]; ok {
+	if _, ok := m.conn[deviceID]; ok {
 		panic("add existing device")
 	}
-	m.protoConn[deviceID] = protoConn
-	if _, ok := m.rawConn[deviceID]; ok {
-		panic("add existing device")
-	}
-	m.rawConn[deviceID] = rawConn
+	m.conn[deviceID] = conn
 
-	protoConn.Start()
+	conn.Start()
 
 	cm := m.clusterConfig(deviceID)
-	protoConn.ClusterConfig(cm)
+	conn.ClusterConfig(cm)
 
 	m.fmut.RLock()
 	for _, folder := range m.deviceFolders[deviceID] {
 		fs := m.folderFiles[folder]
-		go sendIndexes(protoConn, folder, fs, m.folderIgnores[folder])
+		go sendIndexes(conn, folder, fs, m.folderIgnores[folder])
 	}
 	m.fmut.RUnlock()
 	m.pmut.Unlock()
@@ -956,6 +956,31 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	m.deviceWasSeen(deviceID)
 }
 
+func (m *Model) PauseDevice(device protocol.DeviceID) {
+	m.pmut.Lock()
+	m.devicePaused[device] = true
+	_, ok := m.conn[device]
+	m.pmut.Unlock()
+	if ok {
+		m.Close(device, errors.New("device paused"))
+	}
+	events.Default.Log(events.DevicePaused, map[string]string{"device": device.String()})
+}
+
+func (m *Model) ResumeDevice(device protocol.DeviceID) {
+	m.pmut.Lock()
+	m.devicePaused[device] = false
+	m.pmut.Unlock()
+	events.Default.Log(events.DeviceResumed, map[string]string{"device": device.String()})
+}
+
+func (m *Model) IsPaused(device protocol.DeviceID) bool {
+	m.pmut.Lock()
+	paused := m.devicePaused[device]
+	m.pmut.Unlock()
+	return paused
+}
+
 func (m *Model) deviceStatRef(deviceID protocol.DeviceID) *stats.DeviceStatisticsReference {
 	m.fmut.Lock()
 	defer m.fmut.Unlock()
@@ -964,7 +989,7 @@ func (m *Model) deviceStatRef(deviceID protocol.DeviceID) *stats.DeviceStatistic
 		return sr
 	}
 
-	sr := stats.NewDeviceStatisticsReference(m.db, deviceID)
+	sr := stats.NewDeviceStatisticsReference(m.db, deviceID.String())
 	m.deviceStatRefs[deviceID] = sr
 	return sr
 }
@@ -986,7 +1011,7 @@ func (m *Model) folderStatRef(folder string) *stats.FolderStatisticsReference {
 }
 
 func (m *Model) receivedFile(folder string, file protocol.FileInfo) {
-	m.folderStatRef(folder).ReceivedFile(file)
+	m.folderStatRef(folder).ReceivedFile(file.Name, file.IsDeleted())
 }
 
 func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) {
@@ -994,9 +1019,8 @@ func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignore
 	name := conn.Name()
 	var err error
 
-	if debug {
-		l.Debugf("sendIndexes for %s-%s/%q starting", deviceID, name, folder)
-	}
+	l.Debugf("sendIndexes for %s-%s/%q starting", deviceID, name, folder)
+	defer l.Debugf("sendIndexes for %s-%s/%q exiting: %v", deviceID, name, folder, err)
 
 	minLocalVer, err := sendIndexTo(true, 0, conn, folder, fs, ignores)
 
@@ -1021,9 +1045,6 @@ func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignore
 		time.Sleep(250 * time.Millisecond)
 	}
 
-	if debug {
-		l.Debugf("sendIndexes for %s-%s/%q exiting: %v", deviceID, name, folder, err)
-	}
 }
 
 func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) (int64, error) {
@@ -1045,9 +1066,7 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 		}
 
 		if ignores.Match(f.Name) || symlinkInvalid(folder, f) {
-			if debug {
-				l.Debugln("not sending update for ignored/unsupported symlink", f)
-			}
+			l.Debugln("not sending update for ignored/unsupported symlink", f)
 			return true
 		}
 
@@ -1056,17 +1075,13 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 				if err = conn.Index(folder, batch, 0, nil); err != nil {
 					return false
 				}
-				if debug {
-					l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", deviceID, name, folder, len(batch), currentBatchSize)
-				}
+				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", deviceID, name, folder, len(batch), currentBatchSize)
 				initial = false
 			} else {
 				if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil {
 					return false
 				}
-				if debug {
-					l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", deviceID, name, folder, len(batch), currentBatchSize)
-				}
+				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", deviceID, name, folder, len(batch), currentBatchSize)
 			}
 
 			batch = make([]protocol.FileInfo, 0, indexBatchSize)
@@ -1080,12 +1095,12 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 
 	if initial && err == nil {
 		err = conn.Index(folder, batch, 0, nil)
-		if debug && err == nil {
+		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch))
 		}
 	} else if len(batch) > 0 && err == nil {
 		err = conn.IndexUpdate(folder, batch, 0, nil)
-		if debug && err == nil {
+		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch))
 		}
 	}
@@ -1114,16 +1129,14 @@ func (m *Model) updateLocals(folder string, fs []protocol.FileInfo) {
 
 func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) {
 	m.pmut.RLock()
-	nc, ok := m.protoConn[deviceID]
+	nc, ok := m.conn[deviceID]
 	m.pmut.RUnlock()
 
 	if !ok {
 		return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
 	}
 
-	if debug {
-		l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x f=%x op=%s", m, deviceID, folder, name, offset, size, hash, flags, options)
-	}
+	l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x f=%x op=%s", m, deviceID, folder, name, offset, size, hash, flags, options)
 
 	return nc.Request(folder, name, offset, size, hash, flags, options)
 }
@@ -1270,18 +1283,20 @@ nextSub:
 	subs = unifySubs
 
 	w := &scanner.Walker{
-		Dir:           folderCfg.Path(),
-		Subs:          subs,
-		Matcher:       ignores,
-		BlockSize:     protocol.BlockSize,
-		TempNamer:     defTempNamer,
-		TempLifetime:  time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
-		CurrentFiler:  cFiler{m, folder},
-		MtimeRepo:     db.NewVirtualMtimeRepo(m.db, folderCfg.ID),
-		IgnorePerms:   folderCfg.IgnorePerms,
-		AutoNormalize: folderCfg.AutoNormalize,
-		Hashers:       m.numHashers(folder),
-		ShortID:       m.shortID,
+		Folder:                folderCfg.ID,
+		Dir:                   folderCfg.Path(),
+		Subs:                  subs,
+		Matcher:               ignores,
+		BlockSize:             protocol.BlockSize,
+		TempNamer:             defTempNamer,
+		TempLifetime:          time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
+		CurrentFiler:          cFiler{m, folder},
+		MtimeRepo:             db.NewVirtualMtimeRepo(m.db, folderCfg.ID),
+		IgnorePerms:           folderCfg.IgnorePerms,
+		AutoNormalize:         folderCfg.AutoNormalize,
+		Hashers:               m.numHashers(folder),
+		ShortID:               m.shortID,
+		ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
 	}
 
 	runner.setState(FolderScanning)
@@ -1362,9 +1377,7 @@ nextSub:
 
 			if ignores.Match(f.Name) || symlinkInvalid(folder, f) {
 				// File has been ignored or an unsupported symlink. Set invalid bit.
-				if debug {
-					l.Debugln("setting invalid bit on ignored", f)
-				}
+				l.Debugln("setting invalid bit on ignored", f)
 				nf := protocol.FileInfo{
 					Name:     f.Name,
 					Flags:    f.Flags | protocol.FlagInvalid,
@@ -1657,7 +1670,7 @@ func (m *Model) Availability(folder, file string) []protocol.DeviceID {
 
 	availableDevices := []protocol.DeviceID{}
 	for _, device := range fs.Availability(file) {
-		_, ok := m.protoConn[device]
+		_, ok := m.conn[device]
 		if ok {
 			availableDevices = append(availableDevices, device)
 		}
@@ -1779,9 +1792,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 	for folderID, cfg := range toFolders {
 		if _, ok := fromFolders[folderID]; !ok {
 			// A folder was added.
-			if debug {
-				l.Debugln(m, "adding folder", folderID)
-			}
+			l.Debugln(m, "adding folder", folderID)
 			m.AddFolder(cfg)
 			if cfg.ReadOnly {
 				m.StartFolderRO(folderID)
@@ -1793,7 +1804,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 			// folder.
 			m.pmut.Lock()
 			for _, dev := range cfg.DeviceIDs() {
-				if conn, ok := m.rawConn[dev]; ok {
+				if conn, ok := m.conn[dev]; ok {
 					closeRawConn(conn)
 				}
 			}
@@ -1805,9 +1816,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 		toCfg, ok := toFolders[folderID]
 		if !ok {
 			// A folder was removed. Requires restart.
-			if debug {
-				l.Debugln(m, "requires restart, removing folder", folderID)
-			}
+			l.Debugln(m, "requires restart, removing folder", folderID)
 			return false
 		}
 
@@ -1819,9 +1828,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 		for dev := range fromDevs {
 			if _, ok := toDevs[dev]; !ok {
 				// A device was removed. Requires restart.
-				if debug {
-					l.Debugln(m, "requires restart, removing device", dev, "from folder", folderID)
-				}
+				l.Debugln(m, "requires restart, removing device", dev, "from folder", folderID)
 				return false
 			}
 		}
@@ -1841,7 +1848,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 				// disconnect it so that we start sharing the folder with it.
 				// We close the underlying connection and let the normal error
 				// handling kick in to clean up and reconnect.
-				if conn, ok := m.rawConn[dev]; ok {
+				if conn, ok := m.conn[dev]; ok {
 					closeRawConn(conn)
 				}
 
@@ -1854,9 +1861,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 		fromCfg.Devices = nil
 		toCfg.Devices = nil
 		if !reflect.DeepEqual(fromCfg, toCfg) {
-			if debug {
-				l.Debugln(m, "requires restart, folder", folderID, "configuration differs")
-			}
+			l.Debugln(m, "requires restart, folder", folderID, "configuration differs")
 			return false
 		}
 	}
@@ -1865,18 +1870,14 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 	toDevs := mapDeviceCfgs(from.Devices)
 	for _, dev := range from.Devices {
 		if _, ok := toDevs[dev.DeviceID]; !ok {
-			if debug {
-				l.Debugln(m, "requires restart, device", dev.DeviceID, "was removed")
-			}
+			l.Debugln(m, "requires restart, device", dev.DeviceID, "was removed")
 			return false
 		}
 	}
 
 	// All of the generic options require restart
 	if !reflect.DeepEqual(from.Options, to.Options) {
-		if debug {
-			l.Debugln(m, "requires restart, options differ")
-		}
+		l.Debugln(m, "requires restart, options differ")
 		return false
 	}
 
@@ -1916,21 +1917,15 @@ func mapDeviceCfgs(devices []config.DeviceConfiguration) map[protocol.DeviceID]s
 func filterIndex(folder string, fs []protocol.FileInfo, dropDeletes bool) []protocol.FileInfo {
 	for i := 0; i < len(fs); {
 		if fs[i].Flags&^protocol.FlagsAll != 0 {
-			if debug {
-				l.Debugln("dropping update for file with unknown bits set", fs[i])
-			}
+			l.Debugln("dropping update for file with unknown bits set", fs[i])
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
 		} else if fs[i].IsDeleted() && dropDeletes {
-			if debug {
-				l.Debugln("dropping update for undesired delete", fs[i])
-			}
+			l.Debugln("dropping update for undesired delete", fs[i])
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
 		} else if symlinkInvalid(folder, fs[i]) {
-			if debug {
-				l.Debugln("dropping update for unsupported symlink", fs[i])
-			}
+			l.Debugln("dropping update for unsupported symlink", fs[i])
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
 		} else {

lib/model/model_test.go

@@ -12,15 +12,16 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"testing"
 	"time"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
@@ -281,7 +282,11 @@ func BenchmarkRequest(b *testing.B) {
 		id:          device1,
 		requestData: []byte("some data to return"),
 	}
-	m.AddConnection(fc, fc)
+	m.AddConnection(Connection{
+		&net.TCPConn{},
+		fc,
+		ConnectionTypeDirectAccept,
+	})
 	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
@@ -314,6 +319,18 @@ func TestDeviceRename(t *testing.T) {
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+
+	fc := FakeConnection{
+		id:          device1,
+		requestData: []byte("some data to return"),
+	}
+
+	m.AddConnection(Connection{
+		&net.TCPConn{},
+		fc,
+		ConnectionTypeDirectAccept,
+	})
+
 	m.ServeBackground()
 	if cfg.Devices()[device1].Name != "" {
 		t.Errorf("Device already has a name")

lib/model/progressemitter.go

@@ -51,15 +51,11 @@ func (t *ProgressEmitter) Serve() {
 	for {
 		select {
 		case <-t.stop:
-			if debug {
-				l.Debugln("progress emitter: stopping")
-			}
+			l.Debugln("progress emitter: stopping")
 			return
 		case <-t.timer.C:
 			t.mut.Lock()
-			if debug {
-				l.Debugln("progress emitter: timer - looking after", len(t.registry))
-			}
+			l.Debugln("progress emitter: timer - looking after", len(t.registry))
 			output := make(map[string]map[string]*pullerProgress)
 			for _, puller := range t.registry {
 				if output[puller.folder] == nil {
@@ -70,10 +66,8 @@ func (t *ProgressEmitter) Serve() {
 			if !reflect.DeepEqual(t.last, output) {
 				events.Default.Log(events.DownloadProgress, output)
 				t.last = output
-				if debug {
-					l.Debugf("progress emitter: emitting %#v", output)
-				}
-			} else if debug {
+				l.Debugf("progress emitter: emitting %#v", output)
+			} else {
 				l.Debugln("progress emitter: nothing new")
 			}
 			if len(t.registry) != 0 {
@@ -95,9 +89,7 @@ func (t *ProgressEmitter) CommitConfiguration(from, to config.Configuration) boo
 	defer t.mut.Unlock()
 
 	t.interval = time.Duration(to.Options.ProgressUpdateIntervalS) * time.Second
-	if debug {
-		l.Debugln("progress emitter: updated interval", t.interval)
-	}
+	l.Debugln("progress emitter: updated interval", t.interval)
 
 	return true
 }
@@ -112,9 +104,7 @@ func (t *ProgressEmitter) Stop() {
 func (t *ProgressEmitter) Register(s *sharedPullerState) {
 	t.mut.Lock()
 	defer t.mut.Unlock()
-	if debug {
-		l.Debugln("progress emitter: registering", s.folder, s.file.Name)
-	}
+	l.Debugln("progress emitter: registering", s.folder, s.file.Name)
 	if len(t.registry) == 0 {
 		t.timer.Reset(t.interval)
 	}
@@ -125,9 +115,7 @@ func (t *ProgressEmitter) Register(s *sharedPullerState) {
 func (t *ProgressEmitter) Deregister(s *sharedPullerState) {
 	t.mut.Lock()
 	defer t.mut.Unlock()
-	if debug {
-		l.Debugln("progress emitter: deregistering", s.folder, s.file.Name)
-	}
+	l.Debugln("progress emitter: deregistering", s.folder, s.file.Name)
 	delete(t.registry, filepath.Join(s.folder, s.file.Name))
 }
 
@@ -141,9 +129,7 @@ func (t *ProgressEmitter) BytesCompleted(folder string) (bytes int64) {
 			bytes += s.Progress().BytesDone
 		}
 	}
-	if debug {
-		l.Debugf("progress emitter: bytes completed for %s: %d", folder, bytes)
-	}
+	l.Debugf("progress emitter: bytes completed for %s: %d", folder, bytes)
 	return
 }
 

lib/model/rofolder.go

@@ -48,10 +48,8 @@ func newROFolder(model *Model, folder string, interval time.Duration) *roFolder
 }
 
 func (s *roFolder) Serve() {
-	if debug {
-		l.Debugln(s, "starting")
-		defer l.Debugln(s, "exiting")
-	}
+	l.Debugln(s, "starting")
+	defer l.Debugln(s, "exiting")
 
 	defer func() {
 		s.timer.Stop()
@@ -79,9 +77,7 @@ func (s *roFolder) Serve() {
 				continue
 			}
 
-			if debug {
-				l.Debugln(s, "rescan")
-			}
+			l.Debugln(s, "rescan")
 
 			if err := s.model.internalScanFolderSubs(s.folder, nil); err != nil {
 				// Potentially sets the error twice, once in the scanner just
@@ -111,9 +107,7 @@ func (s *roFolder) Serve() {
 				continue
 			}
 
-			if debug {
-				l.Debugln(s, "forced rescan")
-			}
+			l.Debugln(s, "forced rescan")
 
 			if err := s.model.internalScanFolderSubs(s.folder, req.subs); err != nil {
 				// Potentially sets the error twice, once in the scanner just

lib/model/rwfolder.go

@@ -16,12 +16,12 @@ import (
 	"sort"
 	"time"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/symlinks"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -30,12 +30,6 @@ import (
 
 // TODO: Stop on errors
 
-const (
-	pauseIntv     = 60 * time.Second
-	nextPullIntv  = 10 * time.Second
-	shortPullIntv = time.Second
-)
-
 // A pullBlockState is passed to the puller routine for each block that needs
 // to be fetched.
 type pullBlockState struct {
@@ -67,8 +61,10 @@ const (
 )
 
 const (
-	defaultCopiers = 1
-	defaultPullers = 16
+	defaultCopiers     = 1
+	defaultPullers     = 16
+	defaultPullerSleep = 10 * time.Second
+	defaultPullerPause = 60 * time.Second
 )
 
 type dbUpdateJob struct {
@@ -92,6 +88,8 @@ type rwFolder struct {
 	pullers     int
 	shortID     uint64
 	order       config.PullOrder
+	sleep       time.Duration
+	pause       time.Duration
 
 	stop        chan struct{}
 	queue       *jobQueue
@@ -128,7 +126,7 @@ func newRWFolder(m *Model, shortID uint64, cfg config.FolderConfiguration) *rwFo
 
 		stop:        make(chan struct{}),
 		queue:       newJobQueue(),
-		pullTimer:   time.NewTimer(shortPullIntv),
+		pullTimer:   time.NewTimer(time.Second),
 		scanTimer:   time.NewTimer(time.Millisecond), // The first scan should be done immediately.
 		delayScan:   make(chan time.Duration),
 		scanNow:     make(chan rescanRequest),
@@ -144,6 +142,18 @@ func newRWFolder(m *Model, shortID uint64, cfg config.FolderConfiguration) *rwFo
 		p.pullers = defaultPullers
 	}
 
+	if cfg.PullerPauseS == 0 {
+		p.pause = defaultPullerPause
+	} else {
+		p.pause = time.Duration(cfg.PullerPauseS) * time.Second
+	}
+
+	if cfg.PullerSleepS == 0 {
+		p.sleep = defaultPullerSleep
+	} else {
+		p.sleep = time.Duration(cfg.PullerSleepS) * time.Second
+	}
+
 	return p
 }
 
@@ -157,10 +167,8 @@ func (p *rwFolder) ignorePermissions(file protocol.FileInfo) bool {
 // Serve will run scans and pulls. It will return when Stop()ed or on a
 // critical error.
 func (p *rwFolder) Serve() {
-	if debug {
-		l.Debugln(p, "starting")
-		defer l.Debugln(p, "exiting")
-	}
+	l.Debugln(p, "starting")
+	defer l.Debugln(p, "exiting")
 
 	defer func() {
 		p.pullTimer.Stop()
@@ -182,9 +190,7 @@ func (p *rwFolder) Serve() {
 		sleepNanos := (p.scanIntv.Nanoseconds()*3 + rand.Int63n(2*p.scanIntv.Nanoseconds())) / 4
 		intv := time.Duration(sleepNanos) * time.Nanosecond
 
-		if debug {
-			l.Debugln(p, "next rescan in", intv)
-		}
+		l.Debugln(p, "next rescan in", intv)
 		p.scanTimer.Reset(intv)
 	}
 
@@ -198,23 +204,13 @@ func (p *rwFolder) Serve() {
 
 		case <-p.remoteIndex:
 			prevVer = 0
-			p.pullTimer.Reset(shortPullIntv)
-			if debug {
-				l.Debugln(p, "remote index updated, rescheduling pull")
-			}
+			p.pullTimer.Reset(0)
+			l.Debugln(p, "remote index updated, rescheduling pull")
 
 		case <-p.pullTimer.C:
 			if !initialScanCompleted {
-				if debug {
-					l.Debugln(p, "skip (initial)")
-				}
-				p.pullTimer.Reset(nextPullIntv)
-				continue
-			}
-
-			if err := p.model.CheckFolderHealth(p.folder); err != nil {
-				l.Infoln("Skipping folder", p.folder, "pull due to folder error:", err)
-				p.pullTimer.Reset(nextPullIntv)
+				l.Debugln(p, "skip (initial)")
+				p.pullTimer.Reset(p.sleep)
 				continue
 			}
 
@@ -225,9 +221,7 @@ func (p *rwFolder) Serve() {
 			if newHash := curIgnores.Hash(); newHash != prevIgnoreHash {
 				// The ignore patterns have changed. We need to re-evaluate if
 				// there are files we need now that were ignored before.
-				if debug {
-					l.Debugln(p, "ignore patterns have changed, resetting prevVer")
-				}
+				l.Debugln(p, "ignore patterns have changed, resetting prevVer")
 				prevVer = 0
 				prevIgnoreHash = newHash
 			}
@@ -235,17 +229,19 @@ func (p *rwFolder) Serve() {
 			// RemoteLocalVersion() is a fast call, doesn't touch the database.
 			curVer, ok := p.model.RemoteLocalVersion(p.folder)
 			if !ok || curVer == prevVer {
-				if debug {
-					l.Debugln(p, "skip (curVer == prevVer)", prevVer, ok)
-				}
-				p.pullTimer.Reset(nextPullIntv)
+				l.Debugln(p, "skip (curVer == prevVer)", prevVer, ok)
+				p.pullTimer.Reset(p.sleep)
 				continue
 			}
 
-			if debug {
-				l.Debugln(p, "pulling", prevVer, curVer)
+			if err := p.model.CheckFolderHealth(p.folder); err != nil {
+				l.Infoln("Skipping folder", p.folder, "pull due to folder error:", err)
+				p.pullTimer.Reset(p.sleep)
+				continue
 			}
 
+			l.Debugln(p, "pulling", prevVer, curVer)
+
 			p.setState(FolderSyncing)
 			p.clearErrors()
 			tries := 0
@@ -254,9 +250,7 @@ func (p *rwFolder) Serve() {
 				tries++
 
 				changed := p.pullerIteration(curIgnores)
-				if debug {
-					l.Debugln(p, "changed", changed)
-				}
+				l.Debugln(p, "changed", changed)
 
 				if changed == 0 {
 					// No files were changed by the puller, so we are in
@@ -276,10 +270,8 @@ func (p *rwFolder) Serve() {
 						curVer = lv
 					}
 					prevVer = curVer
-					if debug {
-						l.Debugln(p, "next pull in", nextPullIntv)
-					}
-					p.pullTimer.Reset(nextPullIntv)
+					l.Debugln(p, "next pull in", p.sleep)
+					p.pullTimer.Reset(p.sleep)
 					break
 				}
 
@@ -288,10 +280,8 @@ func (p *rwFolder) Serve() {
 					// we're not making it. Probably there are write
 					// errors preventing us. Flag this with a warning and
 					// wait a bit longer before retrying.
-					l.Infof("Folder %q isn't making progress. Pausing puller for %v.", p.folder, pauseIntv)
-					if debug {
-						l.Debugln(p, "next pull in", pauseIntv)
-					}
+					l.Infof("Folder %q isn't making progress. Pausing puller for %v.", p.folder, p.pause)
+					l.Debugln(p, "next pull in", p.pause)
 
 					if folderErrors := p.currentErrors(); len(folderErrors) > 0 {
 						events.Default.Log(events.FolderErrors, map[string]interface{}{
@@ -300,7 +290,7 @@ func (p *rwFolder) Serve() {
 						})
 					}
 
-					p.pullTimer.Reset(pauseIntv)
+					p.pullTimer.Reset(p.pause)
 					break
 				}
 			}
@@ -316,9 +306,7 @@ func (p *rwFolder) Serve() {
 				continue
 			}
 
-			if debug {
-				l.Debugln(p, "rescan")
-			}
+			l.Debugln(p, "rescan")
 
 			if err := p.model.internalScanFolderSubs(p.folder, nil); err != nil {
 				// Potentially sets the error twice, once in the scanner just
@@ -345,9 +333,7 @@ func (p *rwFolder) Serve() {
 				continue
 			}
 
-			if debug {
-				l.Debugln(p, "forced rescan")
-			}
+			l.Debugln(p, "forced rescan")
 
 			if err := p.model.internalScanFolderSubs(p.folder, req.subs); err != nil {
 				// Potentially sets the error twice, once in the scanner just
@@ -409,9 +395,7 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 	pullWg := sync.NewWaitGroup()
 	doneWg := sync.NewWaitGroup()
 
-	if debug {
-		l.Debugln(p, "c", p.copiers, "p", p.pullers)
-	}
+	l.Debugln(p, "c", p.copiers, "p", p.pullers)
 
 	p.dbUpdates = make(chan dbUpdateJob)
 	updateWg.Add(1)
@@ -474,9 +458,7 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 			return true
 		}
 
-		if debug {
-			l.Debugln(p, "handling", file.Name)
-		}
+		l.Debugln(p, "handling", file.Name)
 
 		switch {
 		case file.IsDeleted():
@@ -498,9 +480,7 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 			}
 		case file.IsDirectory() && !file.IsSymlink():
 			// A new or changed directory
-			if debug {
-				l.Debugln("Creating directory", file.Name)
-			}
+			l.Debugln("Creating directory", file.Name)
 			p.handleDir(file)
 		default:
 			// A new or changed file or symlink. This is the only case where we
@@ -591,17 +571,13 @@ nextFile:
 	doneWg.Wait()
 
 	for _, file := range fileDeletions {
-		if debug {
-			l.Debugln("Deleting file", file.Name)
-		}
+		l.Debugln("Deleting file", file.Name)
 		p.deleteFile(file)
 	}
 
 	for i := range dirDeletions {
 		dir := dirDeletions[len(dirDeletions)-i-1]
-		if debug {
-			l.Debugln("Deleting dir", dir.Name)
-		}
+		l.Debugln("Deleting dir", dir.Name)
 		p.deleteDir(dir)
 	}
 
@@ -638,7 +614,7 @@ func (p *rwFolder) handleDir(file protocol.FileInfo) {
 		mode = 0777
 	}
 
-	if debug {
+	if shouldDebug() {
 		curFile, _ := p.model.CurrentFolderFile(p.folder, file.Name)
 		l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
 	}
@@ -836,9 +812,7 @@ func (p *rwFolder) renameFile(source, target protocol.FileInfo) {
 		})
 	}()
 
-	if debug {
-		l.Debugln(p, "taking rename shortcut", source.Name, "->", target.Name)
-	}
+	l.Debugln(p, "taking rename shortcut", source.Name, "->", target.Name)
 
 	from := filepath.Join(p.dir, source.Name)
 	to := filepath.Join(p.dir, target.Name)
@@ -926,9 +900,7 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		// We are supposed to copy the entire file, and then fetch nothing. We
 		// are only updating metadata, so we don't actually *need* to make the
 		// copy.
-		if debug {
-			l.Debugln(p, "taking shortcut on", file.Name)
-		}
+		l.Debugln(p, "taking shortcut on", file.Name)
 
 		events.Default.Log(events.ItemStarted, map[string]string{
 			"folder": p.folder,
@@ -965,7 +937,7 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 	}
 
 	if free, err := osutil.DiskFreeBytes(p.dir); err == nil && free < file.Size() {
-		l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, p.dir, file.Name, float64(free)/1024/1024, float64(file.Size())/1024/1024)
+		l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, p.folder, p.dir, file.Name, float64(free)/1024/1024, float64(file.Size())/1024/1024)
 		p.newError(file.Name, errors.New("insufficient space"))
 		return
 	}
@@ -988,7 +960,7 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 
 	// Check for an old temporary file which might have some blocks we could
 	// reuse.
-	tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize)
+	tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize, 0, nil)
 	if err == nil {
 		// Check for any reusable blocks in the temp file
 		tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
@@ -1033,9 +1005,7 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		mut:         sync.NewMutex(),
 	}
 
-	if debug {
-		l.Debugf("%v need file %s; copy %d, reused %v", p, file.Name, len(blocks), reused)
-	}
+	l.Debugf("%v need file %s; copy %d, reused %v", p, file.Name, len(blocks), reused)
 
 	cs := copyBlocksState{
 		sharedPullerState: &s,
@@ -1080,7 +1050,11 @@ func (p *rwFolder) shortcutFile(file protocol.FileInfo) error {
 
 // shortcutSymlink changes the symlinks type if necessary.
 func (p *rwFolder) shortcutSymlink(file protocol.FileInfo) (err error) {
-	err = symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
+	tt := symlinks.TargetFile
+	if file.IsDirectory() {
+		tt = symlinks.TargetDirectory
+	}
+	err = symlinks.ChangeType(filepath.Join(p.dir, file.Name), tt)
 	if err != nil {
 		l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
 		p.newError(file.Name, err)
@@ -1106,15 +1080,17 @@ func (p *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
 		}
 
 		folderRoots := make(map[string]string)
+		var folders []string
 		p.model.fmut.RLock()
 		for folder, cfg := range p.model.folderCfgs {
 			folderRoots[folder] = cfg.Path()
+			folders = append(folders, folder)
 		}
 		p.model.fmut.RUnlock()
 
 		for _, block := range state.blocks {
 			buf = buf[:int(block.Size)]
-			found := p.model.finder.Iterate(block.Hash, func(folder, file string, index int32) bool {
+			found := p.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
 				fd, err := os.Open(filepath.Join(folderRoots[folder], file))
 				if err != nil {
 					return false
@@ -1129,14 +1105,12 @@ func (p *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
 				hash, err := scanner.VerifyBuffer(buf, block)
 				if err != nil {
 					if hash != nil {
-						if debug {
-							l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
-						}
+						l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
 						err = p.model.finder.Fix(folder, file, index, block.Hash, hash)
 						if err != nil {
 							l.Warnln("finder fix:", err)
 						}
-					} else if debug {
+					} else {
 						l.Debugln("Finder failed to verify buffer", err)
 					}
 					return false
@@ -1211,9 +1185,7 @@ func (p *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul
 			buf, lastError := p.model.requestGlobal(selected, p.folder, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, 0, nil)
 			activity.done(selected)
 			if lastError != nil {
-				if debug {
-					l.Debugln("request:", p.folder, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
-				}
+				l.Debugln("request:", p.folder, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
 				continue
 			}
 
@@ -1221,9 +1193,7 @@ func (p *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul
 			// try pulling it from another device.
 			_, lastError = scanner.VerifyBuffer(buf, state.block)
 			if lastError != nil {
-				if debug {
-					l.Debugln("request:", p.folder, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
-				}
+				l.Debugln("request:", p.folder, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
 				continue
 			}
 
@@ -1314,7 +1284,11 @@ func (p *rwFolder) performFinish(state *sharedPullerState) error {
 		// Remove the file, and replace it with a symlink.
 		err = osutil.InWritableDir(func(path string) error {
 			os.Remove(path)
-			return symlinks.Create(path, string(content), state.file.Flags)
+			tt := symlinks.TargetFile
+			if state.file.IsDirectory() {
+				tt = symlinks.TargetDirectory
+			}
+			return symlinks.Create(path, string(content), tt)
 		}, state.realName)
 		if err != nil {
 			return err
@@ -1329,9 +1303,7 @@ func (p *rwFolder) performFinish(state *sharedPullerState) error {
 func (p *rwFolder) finisherRoutine(in <-chan *sharedPullerState) {
 	for state := range in {
 		if closed, err := state.finalClose(); closed {
-			if debug {
-				l.Debugln(p, "closing", state.file.Name)
-			}
+			l.Debugln(p, "closing", state.file.Name)
 
 			p.queue.Done(state.file.Name)
 

lib/model/rwfolder_test.go

@@ -12,7 +12,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/sync"
 
@@ -42,6 +42,8 @@ var blocks = []protocol.BlockInfo{
 	{Offset: 917504, Size: 0x20000, Hash: []uint8{0x96, 0x6b, 0x15, 0x6b, 0xc4, 0xf, 0x19, 0x18, 0xca, 0xbb, 0x5f, 0xd6, 0xbb, 0xa2, 0xc6, 0x2a, 0xac, 0xbb, 0x8a, 0xb9, 0xce, 0xec, 0x4c, 0xdb, 0x78, 0xec, 0x57, 0x5d, 0x33, 0xf9, 0x8e, 0xaf}},
 }
 
+var folders = []string{"default"}
+
 // Layout of the files: (indexes from the above array)
 // 12345678 - Required file
 // 02005008 - Existing file (currently in the index)
@@ -197,7 +199,7 @@ func TestCopierFinder(t *testing.T) {
 
 	// Verify that the blocks we say exist on file, really exist in the db.
 	for _, idx := range []int{2, 3, 4, 7} {
-		if m.finder.Iterate(blocks[idx].Hash, iterFn) == false {
+		if m.finder.Iterate(folders, blocks[idx].Hash, iterFn) == false {
 			t.Error("Didn't find block")
 		}
 	}
@@ -241,7 +243,7 @@ func TestCopierFinder(t *testing.T) {
 	}
 
 	// Verify that the fetched blocks have actually been written to the temp file
-	blks, err := scanner.HashFile(tempFile, protocol.BlockSize)
+	blks, err := scanner.HashFile(tempFile, protocol.BlockSize, 0, nil)
 	if err != nil {
 		t.Log(err)
 	}
@@ -277,7 +279,7 @@ func TestCopierCleanup(t *testing.T) {
 	// Add file to index
 	m.updateLocals("default", []protocol.FileInfo{file})
 
-	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 
@@ -286,11 +288,11 @@ func TestCopierCleanup(t *testing.T) {
 	// Update index (removing old blocks)
 	m.updateLocals("default", []protocol.FileInfo{file})
 
-	if m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Unexpected block found")
 	}
 
-	if !m.finder.Iterate(blocks[1].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 
@@ -299,11 +301,11 @@ func TestCopierCleanup(t *testing.T) {
 	// Update index (removing old blocks)
 	m.updateLocals("default", []protocol.FileInfo{file})
 
-	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Unexpected block found")
 	}
 
-	if m.finder.Iterate(blocks[1].Hash, iterFn) {
+	if m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 }
@@ -334,7 +336,7 @@ func TestLastResortPulling(t *testing.T) {
 	}
 
 	// Check that that particular block is there
-	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Expected block not found")
 	}
 
@@ -361,11 +363,11 @@ func TestLastResortPulling(t *testing.T) {
 	<-pullChan
 
 	// Verify that it did fix the incorrect hash.
-	if m.finder.Iterate(blocks[0].Hash, iterFn) {
+	if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Found unexpected block")
 	}
 
-	if !m.finder.Iterate(scanner.SHA256OfNothing, iterFn) {
+	if !m.finder.Iterate(folders, scanner.SHA256OfNothing, iterFn) {
 		t.Error("Expected block not found")
 	}
 

lib/model/sharedpullerstate.go

@@ -11,8 +11,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
@@ -193,9 +193,7 @@ func (s *sharedPullerState) failed() error {
 func (s *sharedPullerState) copyDone() {
 	s.mut.Lock()
 	s.copyNeeded--
-	if debug {
-		l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.copyNeeded)
-	}
+	l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.copyNeeded)
 	s.mut.Unlock()
 }
 
@@ -211,18 +209,14 @@ func (s *sharedPullerState) pullStarted() {
 	s.copyNeeded--
 	s.pullTotal++
 	s.pullNeeded++
-	if debug {
-		l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded start ->", s.pullNeeded)
-	}
+	l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded start ->", s.pullNeeded)
 	s.mut.Unlock()
 }
 
 func (s *sharedPullerState) pullDone() {
 	s.mut.Lock()
 	s.pullNeeded--
-	if debug {
-		l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded done ->", s.pullNeeded)
-	}
+	l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded done ->", s.pullNeeded)
 	s.mut.Unlock()
 }