Merge pull request #1539 from calmh/locations

Move index to index-v0.11.0.db (new format) and centralize location config

AUTHORS

@@ -28,7 +28,7 @@ Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jochen Voss <voss@seehuhn.de>
 Johan Vromans <jvromans@squirrel.nl>
 Karol Różycki <rozycki.karol@gmail.com>
-Kamada Ken'ichi <kamada@nanohz.org>
+Ken'ichi Kamada <kamada@nanohz.org>
 Lode Hoste <zillode@zillode.be>
 Marcin Dziadus <dziadus.marcin@gmail.com>
 Marc Laporte <marc@marclaporte.com>

Godeps/Godeps.json

@@ -19,27 +19,31 @@
 		},
 		{
 			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "ff948d7666c5e0fd18d398f6278881724d36a90b"
+			"Rev": "bccf335c34c01760bdc89f98c952fcda696e27d2"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
-			"Rev": "f9f36d11773655c0485207f0ad30dc2655f69d56"
+			"Rev": "c5abe513796336ee2869745bff0638508450e9c5"
 		},
 		{
 			"ImportPath": "github.com/kardianos/osext",
-			"Rev": "91292666f7e40f03185cdd1da7d85633c973eca7"
+			"Rev": "efacde03154693404c65e7aa7d461ac9014acd0c"
 		},
 		{
 			"ImportPath": "github.com/syncthing/protocol",
-			"Rev": "1a4398cc55c8fe82a964097eaf59f2475b020a49"
+			"Rev": "6277c0595c18d42e9db75dfe900463ef093a82d2"
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "e3f32eb300aa1e514fe8ba58d008da90a062273d"
+			"Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
 		},
 		{
 			"ImportPath": "github.com/syndtr/gosnappy/snappy",
-			"Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
+			"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
+		},
+		{
+			"ImportPath": "github.com/thejerf/suture",
+			"Rev": "ff19fb384c3fe30f42717967eaa69da91e5f317c"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
@@ -55,19 +59,19 @@
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Rev": "4ed45ec682102c643324fae5dff8dab085b6c300"
+			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Rev": "4ed45ec682102c643324fae5dff8dab085b6c300"
+			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
 		},
 		{
 			"ImportPath": "golang.org/x/text/transform",
-			"Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a"
+			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
 		},
 		{
 			"ImportPath": "golang.org/x/text/unicode/norm",
-			"Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a"
+			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
 		}
 	]
 }

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -52,7 +52,7 @@ import (
 var encodeTpl = template.Must(template.New("encoder").Parse(`
 func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }//+n
 
 func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
@@ -70,11 +70,11 @@ func (o {{.TypeName}}) MustMarshalXDR() []byte {
 func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }//+n
 
-func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
@@ -87,7 +87,7 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 				{{end}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}.encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
@@ -105,7 +105,7 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 			{{else if $fieldInfo.IsBasic}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}[i].encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
@@ -118,16 +118,16 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }//+n
 
 func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }//+n
 
-func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
+func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
@@ -139,7 +139,7 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
 				{{end}}
 			{{else}}
-				(&o.{{$fieldInfo.Name}}).decodeXDR(xr)
+				(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
 			{{end}}
 		{{else}}
 			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
@@ -155,7 +155,7 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 				{{else if $fieldInfo.IsBasic}}
 					o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
 				{{else}}
-					(&o.{{$fieldInfo.Name}}[i]).decodeXDR(xr)
+					(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
 				{{end}}
 			}
 		{{end}}
@@ -257,12 +257,18 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 			} else {
 				f = fieldInfo{
 					Name:      fn,
-					IsBasic:   false,
 					IsSlice:   true,
 					FieldType: tn,
 					Max:       max,
 				}
 			}
+
+		case *ast.SelectorExpr:
+			f = fieldInfo{
+				Name:      fn,
+				FieldType: ft.Sel.Name,
+				Max:       max,
+			}
 		}
 
 		fs = append(fs, f)
@@ -310,10 +316,9 @@ func generateDiagram(output io.Writer, s structInfo) {
 
 	for _, f := range fs {
 		tn := f.FieldType
-		sl := f.IsSlice
 		name := uncamelize(f.Name)
 
-		if sl {
+		if f.IsSlice {
 			fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
 			fmt.Fprintln(output, line)
 		}
@@ -340,13 +345,16 @@ func generateDiagram(output io.Writer, s structInfo) {
 			fmt.Fprintf(output, "/ %61s /\n", "")
 			fmt.Fprintln(output, line)
 		default:
-			if sl {
+			if f.IsSlice {
 				tn = "Zero or more " + tn + " Structures"
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			} else {
-				fmt.Fprintf(output, "| %s |\n", center(tn, 61))
+				tn = tn + " Structure"
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
+				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			}
 			fmt.Fprintln(output, line)
 		}

Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE

@@ -1,3 +1,6 @@
+This package contains an efficient token-bucket-based rate limiter.
+Copyright (C) 2015 Canonical Ltd.
+
 This software is licensed under the LGPLv3, included below.
 
 As a special exception to the GNU Lesser General Public License version 3

Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go

@@ -11,12 +11,18 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 )
 
 func executable() (string, error) {
 	switch runtime.GOOS {
 	case "linux":
-		return os.Readlink("/proc/self/exe")
+		const deletedSuffix = " (deleted)"
+		execpath, err := os.Readlink("/proc/self/exe")
+		if err != nil {
+			return execpath, err
+		}
+		return strings.TrimSuffix(execpath, deletedSuffix), nil
 	case "netbsd":
 		return os.Readlink("/proc/curproc/exe")
 	case "openbsd", "dragonfly":

Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go

@@ -7,35 +7,42 @@
 package osext
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"os"
-	oexec "os/exec"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"testing"
 )
 
-const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
+const (
+	executableEnvVar = "OSTEST_OUTPUT_EXECUTABLE"
 
-func TestExecPath(t *testing.T) {
+	executableEnvValueMatch  = "match"
+	executableEnvValueDelete = "delete"
+)
+
+func TestExecutableMatch(t *testing.T) {
 	ep, err := Executable()
 	if err != nil {
-		t.Fatalf("ExecPath failed: %v", err)
+		t.Fatalf("Executable failed: %v", err)
 	}
-	// we want fn to be of the form "dir/prog"
+
+	// fullpath to be of the form "dir/prog".
 	dir := filepath.Dir(filepath.Dir(ep))
-	fn, err := filepath.Rel(dir, ep)
+	fullpath, err := filepath.Rel(dir, ep)
 	if err != nil {
 		t.Fatalf("filepath.Rel: %v", err)
 	}
-	cmd := &oexec.Cmd{}
-	// make child start with a relative program path
-	cmd.Dir = dir
-	cmd.Path = fn
-	// forge argv[0] for child, so that we can verify we could correctly
-	// get real path of the executable without influenced by argv[0].
-	cmd.Args = []string{"-", "-test.run=XXXX"}
-	cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
+	// Make child start with a relative program path.
+	// Alter argv[0] for child to verify getting real path without argv[0].
+	cmd := &exec.Cmd{
+		Dir:  dir,
+		Path: fullpath,
+		Env:  []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueMatch)},
+	}
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		t.Fatalf("exec(self) failed: %v", err)
@@ -49,6 +56,63 @@ func TestExecPath(t *testing.T) {
 	}
 }
 
+func TestExecutableDelete(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip()
+	}
+	fpath, err := Executable()
+	if err != nil {
+		t.Fatalf("Executable failed: %v", err)
+	}
+
+	r, w := io.Pipe()
+	stderrBuff := &bytes.Buffer{}
+	stdoutBuff := &bytes.Buffer{}
+	cmd := &exec.Cmd{
+		Path:   fpath,
+		Env:    []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueDelete)},
+		Stdin:  r,
+		Stderr: stderrBuff,
+		Stdout: stdoutBuff,
+	}
+	err = cmd.Start()
+	if err != nil {
+		t.Fatalf("exec(self) start failed: %v", err)
+	}
+
+	tempPath := fpath + "_copy"
+	_ = os.Remove(tempPath)
+
+	err = copyFile(tempPath, fpath)
+	if err != nil {
+		t.Fatalf("copy file failed: %v", err)
+	}
+	err = os.Remove(fpath)
+	if err != nil {
+		t.Fatalf("remove running test file failed: %v", err)
+	}
+	err = os.Rename(tempPath, fpath)
+	if err != nil {
+		t.Fatalf("rename copy to previous name failed: %v", err)
+	}
+
+	w.Write([]byte{0})
+	w.Close()
+
+	err = cmd.Wait()
+	if err != nil {
+		t.Fatalf("exec wait failed: %v", err)
+	}
+
+	childPath := stderrBuff.String()
+	if !filepath.IsAbs(childPath) {
+		t.Fatalf("Child returned %q, want an absolute path", childPath)
+	}
+	if !sameFile(childPath, fpath) {
+		t.Fatalf("Child returned %q, not the same file as %q", childPath, fpath)
+	}
+}
+
 func sameFile(fn1, fn2 string) bool {
 	fi1, err := os.Stat(fn1)
 	if err != nil {
@@ -60,10 +124,30 @@ func sameFile(fn1, fn2 string) bool {
 	}
 	return os.SameFile(fi1, fi2)
 }
+func copyFile(dest, src string) error {
+	df, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer df.Close()
 
-func init() {
-	if e := os.Getenv(execPath_EnvVar); e != "" {
-		// first chdir to another path
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+
+	_, err = io.Copy(df, sf)
+	return err
+}
+
+func TestMain(m *testing.M) {
+	env := os.Getenv(executableEnvVar)
+	switch env {
+	case "":
+		os.Exit(m.Run())
+	case executableEnvValueMatch:
+		// First chdir to another path.
 		dir := "/"
 		if runtime.GOOS == "windows" {
 			dir = filepath.VolumeName(".")
@@ -74,6 +158,23 @@ func init() {
 		} else {
 			fmt.Fprint(os.Stderr, ep)
 		}
-		os.Exit(0)
+	case executableEnvValueDelete:
+		bb := make([]byte, 1)
+		var err error
+		n, err := os.Stdin.Read(bb)
+		if err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+			os.Exit(2)
+		}
+		if n != 1 {
+			fmt.Fprint(os.Stderr, "ERROR: n != 1, n == ", n)
+			os.Exit(2)
+		}
+		if ep, err := Executable(); err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+		} else {
+			fmt.Fprint(os.Stderr, ep)
+		}
 	}
+	os.Exit(0)
 }

Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go

@@ -13,6 +13,9 @@ type TestModel struct {
 	name     string
 	offset   int64
 	size     int
+	hash     []byte
+	flags    uint32
+	options  []Option
 	closedCh chan bool
 }
 
@@ -22,17 +25,20 @@ func newTestModel() *TestModel {
 	}
 }
 
-func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
+func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 }
 
-func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
+func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 }
 
-func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int) ([]byte, error) {
+func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	t.folder = folder
 	t.name = name
 	t.offset = offset
 	t.size = size
+	t.hash = hash
+	t.flags = flags
+	t.options = options
 	return t.data, nil
 }
 

Godeps/_workspace/src/github.com/syncthing/protocol/deviceid.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/base32"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"regexp"
@@ -67,6 +68,11 @@ func (n DeviceID) Equals(other DeviceID) bool {
 	return bytes.Compare(n[:], other[:]) == 0
 }
 
+// Short returns an integer representing bits 0-63 of the device ID.
+func (n DeviceID) Short() uint64 {
+	return binary.BigEndian.Uint64(n[:])
+}
+
 func (n *DeviceID) MarshalText() ([]byte, error) {
 	return []byte(n.String()), nil
 }

Godeps/_workspace/src/github.com/syncthing/protocol/errors.go

@@ -0,0 +1,51 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import (
+	"errors"
+)
+
+const (
+	ecNoError int32 = iota
+	ecGeneric
+	ecNoSuchFile
+	ecInvalid
+)
+
+var (
+	ErrNoError    error = nil
+	ErrGeneric          = errors.New("generic error")
+	ErrNoSuchFile       = errors.New("no such file")
+	ErrInvalid          = errors.New("file is invalid")
+)
+
+var lookupError = map[int32]error{
+	ecNoError:    ErrNoError,
+	ecGeneric:    ErrGeneric,
+	ecNoSuchFile: ErrNoSuchFile,
+	ecInvalid:    ErrInvalid,
+}
+
+var lookupCode = map[error]int32{
+	ErrNoError:    ecNoError,
+	ErrGeneric:    ecGeneric,
+	ErrNoSuchFile: ecNoSuchFile,
+	ErrInvalid:    ecInvalid,
+}
+
+func codeToError(errcode int32) error {
+	err, ok := lookupError[errcode]
+	if !ok {
+		return ErrGeneric
+	}
+	return err
+}
+
+func errorToCode(err error) int32 {
+	code, ok := lookupCode[err]
+	if !ok {
+		return ecGeneric
+	}
+	return code
+}

Godeps/_workspace/src/github.com/syncthing/protocol/message.go

@@ -1,5 +1,6 @@
 // Copyright (C) 2014 The Protocol Authors.
 
+//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
 //go:generate genxdr -o message_xdr.go message.go
 
 package protocol
@@ -17,13 +18,13 @@ type FileInfo struct {
 	Name         string // max:8192
 	Flags        uint32
 	Modified     int64
-	Version      int64
+	Version      Vector
 	LocalVersion int64
 	Blocks       []BlockInfo
 }
 
 func (f FileInfo) String() string {
-	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}",
+	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
 		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
 }
 
@@ -78,8 +79,8 @@ type RequestMessage struct {
 }
 
 type ResponseMessage struct {
-	Data  []byte
-	Error int32
+	Data []byte
+	Code int32
 }
 
 type ClusterConfigMessage struct {
@@ -101,12 +102,15 @@ func (o *ClusterConfigMessage) GetOption(key string) string {
 type Folder struct {
 	ID      string // max:64
 	Devices []Device
+	Flags   uint32
+	Options []Option // max:64
 }
 
 type Device struct {
 	ID              []byte // max:32
-	Flags           uint32
 	MaxLocalVersion int64
+	Flags           uint32
+	Options         []Option // max:64
 }
 
 type Option struct {

Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go

@@ -51,7 +51,7 @@ struct IndexMessage {
 
 func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o IndexMessage) MarshalXDR() ([]byte, error) {
@@ -69,15 +69,15 @@ func (o IndexMessage) MustMarshalXDR() []byte {
 func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteString(o.Folder)
 	xw.WriteUint32(uint32(len(o.Files)))
 	for i := range o.Files {
-		_, err := o.Files[i].encodeXDR(xw)
+		_, err := o.Files[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -88,7 +88,7 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Options)))
 	for i := range o.Options {
-		_, err := o.Options[i].encodeXDR(xw)
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -98,21 +98,21 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *IndexMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *IndexMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Folder = xr.ReadString()
 	_FilesSize := int(xr.ReadUint32())
 	o.Files = make([]FileInfo, _FilesSize)
 	for i := range o.Files {
-		(&o.Files[i]).decodeXDR(xr)
+		(&o.Files[i]).DecodeXDRFrom(xr)
 	}
 	o.Flags = xr.ReadUint32()
 	_OptionsSize := int(xr.ReadUint32())
@@ -121,7 +121,7 @@ func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error {
 	}
 	o.Options = make([]Option, _OptionsSize)
 	for i := range o.Options {
-		(&o.Options[i]).decodeXDR(xr)
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -145,9 +145,9 @@ FileInfo Structure:
 +                      Modified (64 bits)                       +
 |                                                               |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                       Version (64 bits)                       +
-|                                                               |
+/                                                               /
+\                       Vector Structure                        \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                                                               |
 +                    Local Version (64 bits)                    +
@@ -165,7 +165,7 @@ struct FileInfo {
 	string Name<8192>;
 	unsigned int Flags;
 	hyper Modified;
-	hyper Version;
+	Vector Version;
 	hyper LocalVersion;
 	BlockInfo Blocks<>;
 }
@@ -174,7 +174,7 @@ struct FileInfo {
 
 func (o FileInfo) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o FileInfo) MarshalXDR() ([]byte, error) {
@@ -192,22 +192,25 @@ func (o FileInfo) MustMarshalXDR() []byte {
 func (o FileInfo) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Name); l > 8192 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192)
 	}
 	xw.WriteString(o.Name)
 	xw.WriteUint32(o.Flags)
 	xw.WriteUint64(uint64(o.Modified))
-	xw.WriteUint64(uint64(o.Version))
+	_, err := o.Version.EncodeXDRInto(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
 	xw.WriteUint64(uint64(o.LocalVersion))
 	xw.WriteUint32(uint32(len(o.Blocks)))
 	for i := range o.Blocks {
-		_, err := o.Blocks[i].encodeXDR(xw)
+		_, err := o.Blocks[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -217,25 +220,25 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *FileInfo) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *FileInfo) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *FileInfo) decodeXDR(xr *xdr.Reader) error {
+func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Name = xr.ReadStringMax(8192)
 	o.Flags = xr.ReadUint32()
 	o.Modified = int64(xr.ReadUint64())
-	o.Version = int64(xr.ReadUint64())
+	(&o.Version).DecodeXDRFrom(xr)
 	o.LocalVersion = int64(xr.ReadUint64())
 	_BlocksSize := int(xr.ReadUint32())
 	o.Blocks = make([]BlockInfo, _BlocksSize)
 	for i := range o.Blocks {
-		(&o.Blocks[i]).decodeXDR(xr)
+		(&o.Blocks[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -266,7 +269,7 @@ struct BlockInfo {
 
 func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o BlockInfo) MarshalXDR() ([]byte, error) {
@@ -284,11 +287,11 @@ func (o BlockInfo) MustMarshalXDR() []byte {
 func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o BlockInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(uint32(o.Size))
 	if l := len(o.Hash); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64)
@@ -299,16 +302,16 @@ func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *BlockInfo) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *BlockInfo) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error {
+func (o *BlockInfo) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Size = int32(xr.ReadUint32())
 	o.Hash = xr.ReadBytesMax(64)
 	return xr.Error()
@@ -369,7 +372,7 @@ struct RequestMessage {
 
 func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o RequestMessage) MarshalXDR() ([]byte, error) {
@@ -387,11 +390,11 @@ func (o RequestMessage) MustMarshalXDR() []byte {
 func (o RequestMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o RequestMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Folder); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64)
 	}
@@ -412,7 +415,7 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Options)))
 	for i := range o.Options {
-		_, err := o.Options[i].encodeXDR(xw)
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -422,16 +425,16 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *RequestMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *RequestMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *RequestMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Folder = xr.ReadStringMax(64)
 	o.Name = xr.ReadStringMax(8192)
 	o.Offset = int64(xr.ReadUint64())
@@ -444,7 +447,7 @@ func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error {
 	}
 	o.Options = make([]Option, _OptionsSize)
 	for i := range o.Options {
-		(&o.Options[i]).decodeXDR(xr)
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -462,20 +465,20 @@ ResponseMessage Structure:
 \                    Data (variable length)                     \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Error                             |
+|                             Code                              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct ResponseMessage {
 	opaque Data<>;
-	int Error;
+	int Code;
 }
 
 */
 
 func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o ResponseMessage) MarshalXDR() ([]byte, error) {
@@ -493,30 +496,30 @@ func (o ResponseMessage) MustMarshalXDR() []byte {
 func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteBytes(o.Data)
-	xw.WriteUint32(uint32(o.Error))
+	xw.WriteUint32(uint32(o.Code))
 	return xw.Tot(), xw.Error()
 }
 
 func (o *ResponseMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *ResponseMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Data = xr.ReadBytes()
-	o.Error = int32(xr.ReadUint32())
+	o.Code = int32(xr.ReadUint32())
 	return xr.Error()
 }
 
@@ -564,7 +567,7 @@ struct ClusterConfigMessage {
 
 func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o ClusterConfigMessage) MarshalXDR() ([]byte, error) {
@@ -582,11 +585,11 @@ func (o ClusterConfigMessage) MustMarshalXDR() []byte {
 func (o ClusterConfigMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.ClientName); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("ClientName", l, 64)
 	}
@@ -597,7 +600,7 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteString(o.ClientVersion)
 	xw.WriteUint32(uint32(len(o.Folders)))
 	for i := range o.Folders {
-		_, err := o.Folders[i].encodeXDR(xw)
+		_, err := o.Folders[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -607,7 +610,7 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Options)))
 	for i := range o.Options {
-		_, err := o.Options[i].encodeXDR(xw)
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -617,22 +620,22 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ClientName = xr.ReadStringMax(64)
 	o.ClientVersion = xr.ReadStringMax(64)
 	_FoldersSize := int(xr.ReadUint32())
 	o.Folders = make([]Folder, _FoldersSize)
 	for i := range o.Folders {
-		(&o.Folders[i]).decodeXDR(xr)
+		(&o.Folders[i]).DecodeXDRFrom(xr)
 	}
 	_OptionsSize := int(xr.ReadUint32())
 	if _OptionsSize > 64 {
@@ -640,7 +643,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error {
 	}
 	o.Options = make([]Option, _OptionsSize)
 	for i := range o.Options {
-		(&o.Options[i]).decodeXDR(xr)
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -664,18 +667,28 @@ Folder Structure:
 \                Zero or more Device Structures                 \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Options                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Option Structures                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct Folder {
 	string ID<64>;
 	Device Devices<>;
+	unsigned int Flags;
+	Option Options<64>;
 }
 
 */
 
 func (o Folder) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Folder) MarshalXDR() ([]byte, error) {
@@ -693,18 +706,29 @@ func (o Folder) MustMarshalXDR() []byte {
 func (o Folder) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.ID); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64)
 	}
 	xw.WriteString(o.ID)
 	xw.WriteUint32(uint32(len(o.Devices)))
 	for i := range o.Devices {
-		_, err := o.Devices[i].encodeXDR(xw)
+		_, err := o.Devices[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	xw.WriteUint32(o.Flags)
+	if l := len(o.Options); l > 64 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64)
+	}
+	xw.WriteUint32(uint32(len(o.Options)))
+	for i := range o.Options {
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -714,21 +738,30 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Folder) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Folder) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Folder) decodeXDR(xr *xdr.Reader) error {
+func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ID = xr.ReadStringMax(64)
 	_DevicesSize := int(xr.ReadUint32())
 	o.Devices = make([]Device, _DevicesSize)
 	for i := range o.Devices {
-		(&o.Devices[i]).decodeXDR(xr)
+		(&o.Devices[i]).DecodeXDRFrom(xr)
+	}
+	o.Flags = xr.ReadUint32()
+	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize > 64 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
+	o.Options = make([]Option, _OptionsSize)
+	for i := range o.Options {
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -746,25 +779,32 @@ Device Structure:
 \                     ID (variable length)                      \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Flags                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                                                               |
 +                  Max Local Version (64 bits)                  +
 |                                                               |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Options                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Option Structures                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct Device {
 	opaque ID<32>;
-	unsigned int Flags;
 	hyper MaxLocalVersion;
+	unsigned int Flags;
+	Option Options<64>;
 }
 
 */
 
 func (o Device) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Device) MarshalXDR() ([]byte, error) {
@@ -782,35 +822,53 @@ func (o Device) MustMarshalXDR() []byte {
 func (o Device) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Device) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.ID); l > 32 {
 		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32)
 	}
 	xw.WriteBytes(o.ID)
-	xw.WriteUint32(o.Flags)
 	xw.WriteUint64(uint64(o.MaxLocalVersion))
+	xw.WriteUint32(o.Flags)
+	if l := len(o.Options); l > 64 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64)
+	}
+	xw.WriteUint32(uint32(len(o.Options)))
+	for i := range o.Options {
+		_, err := o.Options[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
 	return xw.Tot(), xw.Error()
 }
 
 func (o *Device) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Device) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Device) decodeXDR(xr *xdr.Reader) error {
+func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ID = xr.ReadBytesMax(32)
-	o.Flags = xr.ReadUint32()
 	o.MaxLocalVersion = int64(xr.ReadUint64())
+	o.Flags = xr.ReadUint32()
+	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize > 64 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
+	o.Options = make([]Option, _OptionsSize)
+	for i := range o.Options {
+		(&o.Options[i]).DecodeXDRFrom(xr)
+	}
 	return xr.Error()
 }
 
@@ -844,7 +902,7 @@ struct Option {
 
 func (o Option) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Option) MarshalXDR() ([]byte, error) {
@@ -862,11 +920,11 @@ func (o Option) MustMarshalXDR() []byte {
 func (o Option) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Option) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Option) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Key); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 64)
 	}
@@ -880,16 +938,16 @@ func (o Option) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Option) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Option) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Option) decodeXDR(xr *xdr.Reader) error {
+func (o *Option) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Key = xr.ReadStringMax(64)
 	o.Value = xr.ReadStringMax(1024)
 	return xr.Error()
@@ -921,7 +979,7 @@ struct CloseMessage {
 
 func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o CloseMessage) MarshalXDR() ([]byte, error) {
@@ -939,11 +997,11 @@ func (o CloseMessage) MustMarshalXDR() []byte {
 func (o CloseMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o CloseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Reason); l > 1024 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024)
 	}
@@ -954,16 +1012,16 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *CloseMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *CloseMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *CloseMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Reason = xr.ReadStringMax(1024)
 	o.Code = int32(xr.ReadUint32())
 	return xr.Error()
@@ -985,7 +1043,7 @@ struct EmptyMessage {
 
 func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o EmptyMessage) MarshalXDR() ([]byte, error) {
@@ -1003,25 +1061,25 @@ func (o EmptyMessage) MustMarshalXDR() []byte {
 func (o EmptyMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o EmptyMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	return xw.Tot(), xw.Error()
 }
 
 func (o *EmptyMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *EmptyMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *EmptyMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *EmptyMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go

@@ -12,23 +12,23 @@ type nativeModel struct {
 	next Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 	for i := range files {
 		files[i].Name = norm.NFD.String(files[i].Name)
 	}
-	m.next.Index(deviceID, folder, files)
+	m.next.Index(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 	for i := range files {
 		files[i].Name = norm.NFD.String(files[i].Name)
 	}
-	m.next.IndexUpdate(deviceID, folder, files)
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	name = norm.NFD.String(name)
-	return m.next.Request(deviceID, folder, name, offset, size)
+	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
 }
 
 func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go

@@ -10,16 +10,16 @@ type nativeModel struct {
 	next Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
-	m.next.Index(deviceID, folder, files)
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	m.next.Index(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
-	m.next.IndexUpdate(deviceID, folder, files)
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) {
-	return m.next.Request(deviceID, folder, name, offset, size)
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
 }
 
 func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go

@@ -24,23 +24,30 @@ type nativeModel struct {
 	next Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
-	for i, f := range files {
-		if strings.ContainsAny(f.Name, disallowedCharacters) {
-			if f.IsDeleted() {
-				// Don't complain if the file is marked as deleted, since it
-				// can't possibly exist here anyway.
-				continue
-			}
-			files[i].Flags |= FlagInvalid
-			l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name)
-		}
-		files[i].Name = filepath.FromSlash(f.Name)
-	}
-	m.next.Index(deviceID, folder, files)
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	fixupFiles(files)
+	m.next.Index(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	fixupFiles(files)
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+	name = filepath.FromSlash(name)
+	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
+}
+
+func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(deviceID, config)
+}
+
+func (m nativeModel) Close(deviceID DeviceID, err error) {
+	m.next.Close(deviceID, err)
+}
+
+func fixupFiles(files []FileInfo) {
 	for i, f := range files {
 		if strings.ContainsAny(f.Name, disallowedCharacters) {
 			if f.IsDeleted() {
@@ -53,18 +60,4 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
 		}
 		files[i].Name = filepath.FromSlash(files[i].Name)
 	}
-	m.next.IndexUpdate(deviceID, folder, files)
-}
-
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) {
-	name = filepath.FromSlash(name)
-	return m.next.Request(deviceID, folder, name, offset, size)
-}
-
-func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
-	m.next.ClusterConfig(deviceID, config)
-}
-
-func (m nativeModel) Close(deviceID DeviceID, err error) {
-	m.next.Close(deviceID, err)
 }

Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go

@@ -35,6 +35,7 @@ const (
 	stateIdxRcvd
 )
 
+// FileInfo flags
 const (
 	FlagDeleted              uint32 = 1 << 12
 	FlagInvalid                     = 1 << 13
@@ -48,6 +49,17 @@ const (
 	SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget
 )
 
+// IndexMessage message flags (for IndexUpdate)
+const (
+	FlagIndexTemporary uint32 = 1 << iota
+)
+
+// Request message flags
+const (
+	FlagRequestTemporary uint32 = 1 << iota
+)
+
+// ClusterConfigMessage.Folders.Devices flags
 const (
 	FlagShareTrusted  uint32 = 1 << 0
 	FlagShareReadOnly        = 1 << 1
@@ -66,11 +78,11 @@ type pongMessage struct{ EmptyMessage }
 
 type Model interface {
 	// An index was received from the peer device
-	Index(deviceID DeviceID, folder string, files []FileInfo)
+	Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
 	// An index update was received from the peer device
-	IndexUpdate(deviceID DeviceID, folder string, files []FileInfo)
+	IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
 	// A request was made by the peer device
-	Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error)
+	Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
 	// A cluster configuration message was received
 	ClusterConfig(deviceID DeviceID, config ClusterConfigMessage)
 	// The peer device closed the connection
@@ -80,9 +92,9 @@ type Model interface {
 type Connection interface {
 	ID() DeviceID
 	Name() string
-	Index(folder string, files []FileInfo) error
-	IndexUpdate(folder string, files []FileInfo) error
-	Request(folder string, name string, offset int64, size int) ([]byte, error)
+	Index(folder string, files []FileInfo, flags uint32, options []Option) error
+	IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error
+	Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
 	ClusterConfig(config ClusterConfigMessage)
 	Statistics() Statistics
 }
@@ -169,7 +181,7 @@ func (c *rawConnection) Name() string {
 }
 
 // Index writes the list of file information to the connected peer device
-func (c *rawConnection) Index(folder string, idx []FileInfo) error {
+func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error {
 	select {
 	case <-c.closed:
 		return ErrClosed
@@ -177,15 +189,17 @@ func (c *rawConnection) Index(folder string, idx []FileInfo) error {
 	}
 	c.idxMut.Lock()
 	c.send(-1, messageTypeIndex, IndexMessage{
-		Folder: folder,
-		Files:  idx,
+		Folder:  folder,
+		Files:   idx,
+		Flags:   flags,
+		Options: options,
 	})
 	c.idxMut.Unlock()
 	return nil
 }
 
 // IndexUpdate writes the list of file information to the connected peer device as an update
-func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error {
+func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error {
 	select {
 	case <-c.closed:
 		return ErrClosed
@@ -193,15 +207,17 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error {
 	}
 	c.idxMut.Lock()
 	c.send(-1, messageTypeIndexUpdate, IndexMessage{
-		Folder: folder,
-		Files:  idx,
+		Folder:  folder,
+		Files:   idx,
+		Flags:   flags,
+		Options: options,
 	})
 	c.idxMut.Unlock()
 	return nil
 }
 
 // Request returns the bytes for the specified block after fetching them from the connected peer.
-func (c *rawConnection) Request(folder string, name string, offset int64, size int) ([]byte, error) {
+func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	var id int
 	select {
 	case id = <-c.nextID:
@@ -218,10 +234,13 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
 	c.awaitingMut.Unlock()
 
 	ok := c.send(id, messageTypeRequest, RequestMessage{
-		Folder: folder,
-		Name:   name,
-		Offset: offset,
-		Size:   int32(size),
+		Folder:  folder,
+		Name:    name,
+		Offset:  offset,
+		Size:    int32(size),
+		Hash:    hash,
+		Flags:   flags,
+		Options: options,
 	})
 	if !ok {
 		return nil, ErrClosed
@@ -280,11 +299,6 @@ func (c *rawConnection) readerLoop() (err error) {
 
 		switch msg := msg.(type) {
 		case IndexMessage:
-			if msg.Flags != 0 {
-				// We don't currently support or expect any flags.
-				return fmt.Errorf("protocol error: unknown flags 0x%x in Index(Update) message", msg.Flags)
-			}
-
 			switch hdr.msgType {
 			case messageTypeIndex:
 				if c.state < stateCCRcvd {
@@ -301,10 +315,6 @@ func (c *rawConnection) readerLoop() (err error) {
 			}
 
 		case RequestMessage:
-			if msg.Flags != 0 {
-				// We don't currently support or expect any flags.
-				return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", msg.Flags)
-			}
 			if c.state < stateIdxRcvd {
 				return fmt.Errorf("protocol error: request message in state %d", c.state)
 			}
@@ -460,16 +470,16 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
 
 func (c *rawConnection) handleIndex(im IndexMessage) {
 	if debug {
-		l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files))
+		l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
 	}
-	c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files))
+	c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
 }
 
 func (c *rawConnection) handleIndexUpdate(im IndexMessage) {
 	if debug {
-		l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files))
+		l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
 	}
-	c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files))
+	c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
 }
 
 func filterIndexMessageFiles(fs []FileInfo) []FileInfo {
@@ -499,10 +509,11 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo {
 }
 
 func (c *rawConnection) handleRequest(msgID int, req RequestMessage) {
-	data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size))
+	data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options)
 
 	c.send(msgID, messageTypeResponse, ResponseMessage{
 		Data: data,
+		Code: errorToCode(err),
 	})
 }
 
@@ -510,7 +521,7 @@ func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) {
 	c.awaitingMut.Lock()
 	if rc := c.awaiting[msgID]; rc != nil {
 		c.awaiting[msgID] = nil
-		rc <- asyncResult{resp.Data, nil}
+		rc <- asyncResult{resp.Data, codeToError(resp.Code)}
 		close(rc)
 	}
 	c.awaitingMut.Unlock()

Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go

@@ -229,10 +229,10 @@ func TestClose(t *testing.T) {
 		t.Error("Ping should not return true")
 	}
 
-	c0.Index("default", nil)
-	c0.Index("default", nil)
+	c0.Index("default", nil, 0, nil)
+	c0.Index("default", nil, 0, nil)
 
-	if _, err := c0.Request("default", "foo", 0, 0); err == nil {
+	if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil {
 		t.Error("Request should return an error")
 	}
 }

Godeps/_workspace/src/github.com/syncthing/protocol/vector.go

@@ -0,0 +1,105 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+// The Vector type represents a version vector. The zero value is a usable
+// version vector. The vector has slice semantics and some operations on it
+// are "append-like" in that they may return the same vector modified, or a
+// new allocated Vector with the modified contents.
+type Vector []Counter
+
+// Counter represents a single counter in the version vector.
+type Counter struct {
+	ID    uint64
+	Value uint64
+}
+
+// Update returns a Vector with the index for the specific ID incremented by
+// one. If it is possible, the vector v is updated and returned. If it is not,
+// a copy will be created, updated and returned.
+func (v Vector) Update(ID uint64) Vector {
+	for i := range v {
+		if v[i].ID == ID {
+			// Update an existing index
+			v[i].Value++
+			return v
+		} else if v[i].ID > ID {
+			// Insert a new index
+			nv := make(Vector, len(v)+1)
+			copy(nv, v[:i])
+			nv[i].ID = ID
+			nv[i].Value = 1
+			copy(nv[i+1:], v[i:])
+			return nv
+		}
+	}
+	// Append a new new index
+	return append(v, Counter{ID, 1})
+}
+
+// Merge returns the vector containing the maximum indexes from a and b. If it
+// is possible, the vector a is updated and returned. If it is not, a copy
+// will be created, updated and returned.
+func (a Vector) Merge(b Vector) Vector {
+	var ai, bi int
+	for bi < len(b) {
+		if ai == len(a) {
+			// We've reach the end of a, all that remains are appends
+			return append(a, b[bi:]...)
+		}
+
+		if a[ai].ID > b[bi].ID {
+			// The index from b should be inserted here
+			n := make(Vector, len(a)+1)
+			copy(n, a[:ai])
+			n[ai] = b[bi]
+			copy(n[ai+1:], a[ai:])
+			a = n
+		}
+
+		if a[ai].ID == b[bi].ID {
+			if v := b[bi].Value; v > a[ai].Value {
+				a[ai].Value = v
+			}
+		}
+
+		if bi < len(b) && a[ai].ID == b[bi].ID {
+			bi++
+		}
+		ai++
+	}
+
+	return a
+}
+
+// Copy returns an identical vector that is not shared with v.
+func (v Vector) Copy() Vector {
+	nv := make(Vector, len(v))
+	copy(nv, v)
+	return nv
+}
+
+// Equal returns true when the two vectors are equivalent.
+func (a Vector) Equal(b Vector) bool {
+	return a.Compare(b) == Equal
+}
+
+// LesserEqual returns true when the two vectors are equivalent or a is Lesser
+// than b.
+func (a Vector) LesserEqual(b Vector) bool {
+	comp := a.Compare(b)
+	return comp == Lesser || comp == Equal
+}
+
+// LesserEqual returns true when the two vectors are equivalent or a is Greater
+// than b.
+func (a Vector) GreaterEqual(b Vector) bool {
+	comp := a.Compare(b)
+	return comp == Greater || comp == Equal
+}
+
+// Concurrent returns true when the two vectors are concrurrent.
+func (a Vector) Concurrent(b Vector) bool {
+	comp := a.Compare(b)
+	return comp == ConcurrentGreater || comp == ConcurrentLesser
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+// Ordering represents the relationship between two Vectors.
+type Ordering int
+
+const (
+	Equal Ordering = iota
+	Greater
+	Lesser
+	ConcurrentLesser
+	ConcurrentGreater
+)
+
+// There's really no such thing as "concurrent lesser" and "concurrent
+// greater" in version vectors, just "concurrent". But it's useful to be able
+// to get a strict ordering between versions for stable sorts and so on, so we
+// return both variants. The convenience method Concurrent() can be used to
+// check for either case.
+
+// Compare returns the Ordering that describes a's relation to b.
+func (a Vector) Compare(b Vector) Ordering {
+	var ai, bi int     // index into a and b
+	var av, bv Counter // value at current index
+
+	result := Equal
+
+	for ai < len(a) || bi < len(b) {
+		var aMissing, bMissing bool
+
+		if ai < len(a) {
+			av = a[ai]
+		} else {
+			av = Counter{}
+			aMissing = true
+		}
+
+		if bi < len(b) {
+			bv = b[bi]
+		} else {
+			bv = Counter{}
+			bMissing = true
+		}
+
+		switch {
+		case av.ID == bv.ID:
+			// We have a counter value for each side
+			if av.Value > bv.Value {
+				if result == Lesser {
+					return ConcurrentLesser
+				}
+				result = Greater
+			} else if av.Value < bv.Value {
+				if result == Greater {
+					return ConcurrentGreater
+				}
+				result = Lesser
+			}
+
+		case !aMissing && av.ID < bv.ID || bMissing:
+			// Value is missing on the b side
+			if av.Value > 0 {
+				if result == Lesser {
+					return ConcurrentLesser
+				}
+				result = Greater
+			}
+
+		case !bMissing && bv.ID < av.ID || aMissing:
+			// Value is missing on the a side
+			if bv.Value > 0 {
+				if result == Greater {
+					return ConcurrentGreater
+				}
+				result = Lesser
+			}
+		}
+
+		if ai < len(a) && (av.ID <= bv.ID || bMissing) {
+			ai++
+		}
+		if bi < len(b) && (bv.ID <= av.ID || aMissing) {
+			bi++
+		}
+	}
+
+	return result
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare_test.go

@@ -0,0 +1,249 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+import (
+	"math"
+	"testing"
+)
+
+func TestCompare(t *testing.T) {
+	testcases := []struct {
+		a, b Vector
+		r    Ordering
+	}{
+		// Empty vectors are identical
+		{Vector{}, Vector{}, Equal},
+		{Vector{}, nil, Equal},
+		{nil, Vector{}, Equal},
+		{nil, Vector{Counter{42, 0}}, Equal},
+		{Vector{}, Vector{Counter{42, 0}}, Equal},
+		{Vector{Counter{42, 0}}, nil, Equal},
+		{Vector{Counter{42, 0}}, Vector{}, Equal},
+
+		// Zero is the implied value for a missing Counter
+		{
+			Vector{Counter{42, 0}},
+			Vector{Counter{77, 0}},
+			Equal,
+		},
+
+		// Equal vectors are equal
+		{
+			Vector{Counter{42, 33}},
+			Vector{Counter{42, 33}},
+			Equal,
+		},
+		{
+			Vector{Counter{42, 33}, Counter{77, 24}},
+			Vector{Counter{42, 33}, Counter{77, 24}},
+			Equal,
+		},
+
+		// These a-vectors are all greater than the b-vector
+		{
+			Vector{Counter{42, 1}},
+			nil,
+			Greater,
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{},
+			Greater,
+		},
+		{
+			Vector{Counter{0, 1}},
+			Vector{Counter{0, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{Counter{42, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{math.MaxUint64, 1}},
+			Vector{Counter{math.MaxUint64, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{0, math.MaxUint64}},
+			Vector{Counter{0, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, math.MaxUint64}},
+			Vector{Counter{42, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{math.MaxUint64, math.MaxUint64}},
+			Vector{Counter{math.MaxUint64, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{0, math.MaxUint64}},
+			Vector{Counter{0, math.MaxUint64 - 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, math.MaxUint64}},
+			Vector{Counter{42, math.MaxUint64 - 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{math.MaxUint64, math.MaxUint64}},
+			Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, 2}},
+			Vector{Counter{42, 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 2}},
+			Vector{Counter{22, 22}, Counter{42, 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, 2}, Counter{77, 3}},
+			Vector{Counter{42, 1}, Counter{77, 3}},
+			Greater,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Greater,
+		},
+		{
+			Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Greater,
+		},
+
+		// These a-vectors are all lesser than the b-vector
+		{nil, Vector{Counter{42, 1}}, Lesser},
+		{Vector{}, Vector{Counter{42, 1}}, Lesser},
+		{
+			Vector{Counter{42, 0}},
+			Vector{Counter{42, 1}},
+			Lesser,
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{Counter{42, 2}},
+			Lesser,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 1}},
+			Vector{Counter{22, 22}, Counter{42, 2}},
+			Lesser,
+		},
+		{
+			Vector{Counter{42, 1}, Counter{77, 3}},
+			Vector{Counter{42, 2}, Counter{77, 3}},
+			Lesser,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
+			Lesser,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
+			Lesser,
+		},
+
+		// These are all in conflict
+		{
+			Vector{Counter{42, 2}},
+			Vector{Counter{43, 1}},
+			ConcurrentGreater,
+		},
+		{
+			Vector{Counter{43, 1}},
+			Vector{Counter{42, 2}},
+			ConcurrentLesser,
+		},
+		{
+			Vector{Counter{22, 23}, Counter{42, 1}},
+			Vector{Counter{22, 22}, Counter{42, 2}},
+			ConcurrentGreater,
+		},
+		{
+			Vector{Counter{22, 21}, Counter{42, 2}},
+			Vector{Counter{22, 22}, Counter{42, 1}},
+			ConcurrentLesser,
+		},
+		{
+			Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}},
+			Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}},
+			ConcurrentLesser,
+		},
+	}
+
+	for i, tc := range testcases {
+		// Test real Compare
+		if r := tc.a.Compare(tc.b); r != tc.r {
+			t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r)
+		}
+
+		// Test convenience functions
+		switch tc.r {
+		case Greater:
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if !tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v not >= %+v", tc.a, tc.b)
+			}
+			if tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v <= %+v", tc.a, tc.b)
+			}
+		case Lesser:
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v >= %+v", tc.a, tc.b)
+			}
+			if !tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+		case Equal:
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if !tc.a.Equal(tc.b) {
+				t.Errorf("%+v not == %+v", tc.a, tc.b)
+			}
+			if !tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+			if !tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+		case ConcurrentLesser, ConcurrentGreater:
+			if !tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v not concurrent %+v", tc.a, tc.b)
+			}
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v >= %+v", tc.a, tc.b)
+			}
+			if tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v <= %+v", tc.a, tc.b)
+			}
+		}
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_test.go

@@ -0,0 +1,122 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+import "testing"
+
+func TestUpdate(t *testing.T) {
+	var v Vector
+
+	// Append
+
+	v = v.Update(42)
+	expected := Vector{Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Insert at front
+
+	v = v.Update(36)
+	expected = Vector{Counter{36, 1}, Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Insert in moddle
+
+	v = v.Update(37)
+	expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Update existing
+
+	v = v.Update(37)
+	expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+}
+
+func TestCopy(t *testing.T) {
+	v0 := Vector{Counter{42, 1}}
+	v1 := v0.Copy()
+	v1.Update(42)
+	if v0.Compare(v1) != Lesser {
+		t.Errorf("Copy error, %+v should be ancestor of %+v", v0, v1)
+	}
+}
+
+func TestMerge(t *testing.T) {
+	testcases := []struct {
+		a, b, m Vector
+	}{
+		// No-ops
+		{
+			Vector{},
+			Vector{},
+			Vector{},
+		},
+		{
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+
+		// Appends
+		{
+			Vector{},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+		{
+			Vector{Counter{22, 1}},
+			Vector{Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+		{
+			Vector{Counter{22, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+
+		// Insert
+		{
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{Counter{22, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+
+		// Update
+		{
+			Vector{Counter{22, 1}, Counter{42, 2}},
+			Vector{Counter{22, 2}, Counter{42, 1}},
+			Vector{Counter{22, 2}, Counter{42, 2}},
+		},
+
+		// All of the above
+		{
+			Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}},
+			Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}},
+			Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}},
+		},
+	}
+
+	for i, tc := range testcases {
+		if m := tc.a.Merge(tc.b); m.Compare(tc.m) != Equal {
+			t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m)
+		}
+	}
+
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+// This stuff is hacked up manually because genxdr doesn't support 'type
+// Vector []Counter' declarations and it was tricky when I tried to add it...
+
+type xdrWriter interface {
+	WriteUint32(uint32) (int, error)
+	WriteUint64(uint64) (int, error)
+}
+type xdrReader interface {
+	ReadUint32() uint32
+	ReadUint64() uint64
+}
+
+// EncodeXDRInto encodes the vector as an XDR object into the given XDR
+// encoder.
+func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) {
+	w.WriteUint32(uint32(len(v)))
+	for i := range v {
+		w.WriteUint64(v[i].ID)
+		w.WriteUint64(v[i].Value)
+	}
+	return 4 + 16*len(v), nil
+}
+
+// DecodeXDRFrom decodes the XDR objects from the given reader into itself.
+func (v *Vector) DecodeXDRFrom(r xdrReader) error {
+	l := int(r.ReadUint32())
+	n := make(Vector, l)
+	for i := range n {
+		n[i].ID = r.ReadUint64()
+		n[i].Value = r.ReadUint64()
+	}
+	*v = n
+	return nil
+}

Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go

@@ -20,7 +20,7 @@ func (c wireFormatConnection) Name() string {
 	return c.next.Name()
 }
 
-func (c wireFormatConnection) Index(folder string, fs []FileInfo) error {
+func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error {
 	var myFs = make([]FileInfo, len(fs))
 	copy(myFs, fs)
 
@@ -28,10 +28,10 @@ func (c wireFormatConnection) Index(folder string, fs []FileInfo) error {
 		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
 	}
 
-	return c.next.Index(folder, myFs)
+	return c.next.Index(folder, myFs, flags, options)
 }
 
-func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
+func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error {
 	var myFs = make([]FileInfo, len(fs))
 	copy(myFs, fs)
 
@@ -39,12 +39,12 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
 		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
 	}
 
-	return c.next.IndexUpdate(folder, myFs)
+	return c.next.IndexUpdate(folder, myFs, flags, options)
 }
 
-func (c wireFormatConnection) Request(folder, name string, offset int64, size int) ([]byte, error) {
+func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	name = norm.NFC.String(filepath.ToSlash(name))
-	return c.next.Request(folder, name, offset, size)
+	return c.next.Request(folder, name, offset, size, hash, flags, options)
 }
 
 func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -153,7 +153,7 @@ type Options struct {
 	BlockCacher Cacher
 
 	// BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
-	// Use -1 for zero, this has same effect with specifying NoCacher to BlockCacher.
+	// Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher.
 	//
 	// The default value is 8MiB.
 	BlockCacheCapacity int
@@ -308,7 +308,7 @@ type Options struct {
 	OpenFilesCacher Cacher
 
 	// OpenFilesCacheCapacity defines the capacity of the open files caching.
-	// Use -1 for zero, this has same effect with specifying NoCacher to OpenFilesCacher.
+	// Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher.
 	//
 	// The default value is 500.
 	OpenFilesCacheCapacity int
@@ -355,9 +355,9 @@ func (o *Options) GetBlockCacher() Cacher {
 }
 
 func (o *Options) GetBlockCacheCapacity() int {
-	if o == nil || o.BlockCacheCapacity <= 0 {
+	if o == nil || o.BlockCacheCapacity == 0 {
 		return DefaultBlockCacheCapacity
-	} else if o.BlockCacheCapacity == -1 {
+	} else if o.BlockCacheCapacity < 0 {
 		return 0
 	}
 	return o.BlockCacheCapacity
@@ -497,7 +497,7 @@ func (o *Options) GetMaxMemCompationLevel() int {
 	if o != nil {
 		if o.MaxMemCompationLevel > 0 {
 			level = o.MaxMemCompationLevel
-		} else if o.MaxMemCompationLevel == -1 {
+		} else if o.MaxMemCompationLevel < 0 {
 			level = 0
 		}
 	}
@@ -525,9 +525,9 @@ func (o *Options) GetOpenFilesCacher() Cacher {
 }
 
 func (o *Options) GetOpenFilesCacheCapacity() int {
-	if o == nil || o.OpenFilesCacheCapacity <= 0 {
+	if o == nil || o.OpenFilesCacheCapacity == 0 {
 		return DefaultOpenFilesCacheCapacity
-	} else if o.OpenFilesCacheCapacity == -1 {
+	} else if o.OpenFilesCacheCapacity < 0 {
 		return 0
 	}
 	return o.OpenFilesCacheCapacity

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go

@@ -7,10 +7,15 @@ package snappy
 import (
 	"encoding/binary"
 	"errors"
+	"io"
 )
 
-// ErrCorrupt reports that the input is invalid.
-var ErrCorrupt = errors.New("snappy: corrupt input")
+var (
+	// ErrCorrupt reports that the input is invalid.
+	ErrCorrupt = errors.New("snappy: corrupt input")
+	// ErrUnsupported reports that the input isn't supported.
+	ErrUnsupported = errors.New("snappy: unsupported input")
+)
 
 // DecodedLen returns the length of the decoded block.
 func DecodedLen(src []byte) (int, error) {
@@ -122,3 +127,166 @@ func Decode(dst, src []byte) ([]byte, error) {
 	}
 	return dst[:d], nil
 }
+
+// NewReader returns a new Reader that decompresses from r, using the framing
+// format described at
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		r:       r,
+		decoded: make([]byte, maxUncompressedChunkLen),
+		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
+	}
+}
+
+// Reader is an io.Reader than can read Snappy-compressed bytes.
+type Reader struct {
+	r       io.Reader
+	err     error
+	decoded []byte
+	buf     []byte
+	// decoded[i:j] contains decoded bytes that have not yet been passed on.
+	i, j       int
+	readHeader bool
+}
+
+// Reset discards any buffered data, resets all state, and switches the Snappy
+// reader to read from r. This permits reusing a Reader rather than allocating
+// a new one.
+func (r *Reader) Reset(reader io.Reader) {
+	r.r = reader
+	r.err = nil
+	r.i = 0
+	r.j = 0
+	r.readHeader = false
+}
+
+func (r *Reader) readFull(p []byte) (ok bool) {
+	if _, r.err = io.ReadFull(r.r, p); r.err != nil {
+		if r.err == io.ErrUnexpectedEOF {
+			r.err = ErrCorrupt
+		}
+		return false
+	}
+	return true
+}
+
+// Read satisfies the io.Reader interface.
+func (r *Reader) Read(p []byte) (int, error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+	for {
+		if r.i < r.j {
+			n := copy(p, r.decoded[r.i:r.j])
+			r.i += n
+			return n, nil
+		}
+		if !r.readFull(r.buf[:4]) {
+			return 0, r.err
+		}
+		chunkType := r.buf[0]
+		if !r.readHeader {
+			if chunkType != chunkTypeStreamIdentifier {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.readHeader = true
+		}
+		chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
+		if chunkLen > len(r.buf) {
+			r.err = ErrUnsupported
+			return 0, r.err
+		}
+
+		// The chunk types are specified at
+		// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+		switch chunkType {
+		case chunkTypeCompressedData:
+			// Section 4.2. Compressed data (chunk type 0x00).
+			if chunkLen < checksumSize {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			buf := r.buf[:chunkLen]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+			buf = buf[checksumSize:]
+
+			n, err := DecodedLen(buf)
+			if err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			if n > len(r.decoded) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if _, err := Decode(r.decoded, buf); err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			if crc(r.decoded[:n]) != checksum {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeUncompressedData:
+			// Section 4.3. Uncompressed data (chunk type 0x01).
+			if chunkLen < checksumSize {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			buf := r.buf[:checksumSize]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+			// Read directly into r.decoded instead of via r.buf.
+			n := chunkLen - checksumSize
+			if !r.readFull(r.decoded[:n]) {
+				return 0, r.err
+			}
+			if crc(r.decoded[:n]) != checksum {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeStreamIdentifier:
+			// Section 4.1. Stream identifier (chunk type 0xff).
+			if chunkLen != len(magicBody) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if !r.readFull(r.buf[:len(magicBody)]) {
+				return 0, r.err
+			}
+			for i := 0; i < len(magicBody); i++ {
+				if r.buf[i] != magicBody[i] {
+					r.err = ErrCorrupt
+					return 0, r.err
+				}
+			}
+			continue
+		}
+
+		if chunkType <= 0x7f {
+			// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
+			r.err = ErrUnsupported
+			return 0, r.err
+
+		} else {
+			// Section 4.4 Padding (chunk type 0xfe).
+			// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
+			if !r.readFull(r.buf[:chunkLen]) {
+				return 0, r.err
+			}
+		}
+	}
+}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go

@@ -6,6 +6,7 @@ package snappy
 
 import (
 	"encoding/binary"
+	"io"
 )
 
 // We limit how far copy back-references can go, the same as the C++ code.
@@ -172,3 +173,86 @@ func MaxEncodedLen(srcLen int) int {
 	// This last factor dominates the blowup, so the final estimate is:
 	return 32 + srcLen + srcLen/6
 }
+
+// NewWriter returns a new Writer that compresses to w, using the framing
+// format described at
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w:   w,
+		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
+	}
+}
+
+// Writer is an io.Writer than can write Snappy-compressed bytes.
+type Writer struct {
+	w           io.Writer
+	err         error
+	enc         []byte
+	buf         [checksumSize + chunkHeaderSize]byte
+	wroteHeader bool
+}
+
+// Reset discards the writer's state and switches the Snappy writer to write to
+// w. This permits reusing a Writer rather than allocating a new one.
+func (w *Writer) Reset(writer io.Writer) {
+	w.w = writer
+	w.err = nil
+	w.wroteHeader = false
+}
+
+// Write satisfies the io.Writer interface.
+func (w *Writer) Write(p []byte) (n int, errRet error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+	if !w.wroteHeader {
+		copy(w.enc, magicChunk)
+		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
+			w.err = err
+			return n, err
+		}
+		w.wroteHeader = true
+	}
+	for len(p) > 0 {
+		var uncompressed []byte
+		if len(p) > maxUncompressedChunkLen {
+			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
+		} else {
+			uncompressed, p = p, nil
+		}
+		checksum := crc(uncompressed)
+
+		// Compress the buffer, discarding the result if the improvement
+		// isn't at least 12.5%.
+		chunkType := uint8(chunkTypeCompressedData)
+		chunkBody, err := Encode(w.enc, uncompressed)
+		if err != nil {
+			w.err = err
+			return n, err
+		}
+		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
+			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
+		}
+
+		chunkLen := 4 + len(chunkBody)
+		w.buf[0] = chunkType
+		w.buf[1] = uint8(chunkLen >> 0)
+		w.buf[2] = uint8(chunkLen >> 8)
+		w.buf[3] = uint8(chunkLen >> 16)
+		w.buf[4] = uint8(checksum >> 0)
+		w.buf[5] = uint8(checksum >> 8)
+		w.buf[6] = uint8(checksum >> 16)
+		w.buf[7] = uint8(checksum >> 24)
+		if _, err = w.w.Write(w.buf[:]); err != nil {
+			w.err = err
+			return n, err
+		}
+		if _, err = w.w.Write(chunkBody); err != nil {
+			w.err = err
+			return n, err
+		}
+		n += len(uncompressed)
+	}
+	return n, nil
+}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go

@@ -8,6 +8,10 @@
 // The C++ snappy implementation is at http://code.google.com/p/snappy/
 package snappy
 
+import (
+	"hash/crc32"
+)
+
 /*
 Each encoded block begins with the varint-encoded length of the decoded data,
 followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
@@ -36,3 +40,29 @@ const (
 	tagCopy2   = 0x02
 	tagCopy4   = 0x03
 )
+
+const (
+	checksumSize    = 4
+	chunkHeaderSize = 4
+	magicChunk      = "\xff\x06\x00\x00" + magicBody
+	magicBody       = "sNaPpY"
+	// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says
+	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
+	maxUncompressedChunkLen = 65536
+)
+
+const (
+	chunkTypeCompressedData   = 0x00
+	chunkTypeUncompressedData = 0x01
+	chunkTypePadding          = 0xfe
+	chunkTypeStreamIdentifier = 0xff
+)
+
+var crcTable = crc32.MakeTable(crc32.Castagnoli)
+
+// crc implements the checksum specified in section 3 of
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func crc(b []byte) uint32 {
+	c := crc32.Update(0, crcTable, b)
+	return uint32(c>>15|c<<17) + 0xa282ead8
+}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go

@@ -18,7 +18,10 @@ import (
 	"testing"
 )
 
-var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
+var (
+	download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
+	testdata = flag.String("testdata", "testdata", "Directory containing the test data")
+)
 
 func roundtrip(b, ebuf, dbuf []byte) error {
 	e, err := Encode(ebuf, b)
@@ -55,11 +58,11 @@ func TestSmallCopy(t *testing.T) {
 }
 
 func TestSmallRand(t *testing.T) {
-	rand.Seed(27354294)
+	rng := rand.New(rand.NewSource(27354294))
 	for n := 1; n < 20000; n += 23 {
 		b := make([]byte, n)
-		for i, _ := range b {
-			b[i] = uint8(rand.Uint32())
+		for i := range b {
+			b[i] = uint8(rng.Uint32())
 		}
 		if err := roundtrip(b, nil, nil); err != nil {
 			t.Fatal(err)
@@ -70,7 +73,7 @@ func TestSmallRand(t *testing.T) {
 func TestSmallRegular(t *testing.T) {
 	for n := 1; n < 20000; n += 23 {
 		b := make([]byte, n)
-		for i, _ := range b {
+		for i := range b {
 			b[i] = uint8(i%10 + 'a')
 		}
 		if err := roundtrip(b, nil, nil); err != nil {
@@ -79,6 +82,120 @@ func TestSmallRegular(t *testing.T) {
 	}
 }
 
+func cmp(a, b []byte) error {
+	if len(a) != len(b) {
+		return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
+		}
+	}
+	return nil
+}
+
+func TestFramingFormat(t *testing.T) {
+	// src is comprised of alternating 1e5-sized sequences of random
+	// (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
+	// because it is larger than maxUncompressedChunkLen (64k).
+	src := make([]byte, 1e6)
+	rng := rand.New(rand.NewSource(1))
+	for i := 0; i < 10; i++ {
+		if i%2 == 0 {
+			for j := 0; j < 1e5; j++ {
+				src[1e5*i+j] = uint8(rng.Intn(256))
+			}
+		} else {
+			for j := 0; j < 1e5; j++ {
+				src[1e5*i+j] = uint8(i)
+			}
+		}
+	}
+
+	buf := new(bytes.Buffer)
+	if _, err := NewWriter(buf).Write(src); err != nil {
+		t.Fatalf("Write: encoding: %v", err)
+	}
+	dst, err := ioutil.ReadAll(NewReader(buf))
+	if err != nil {
+		t.Fatalf("ReadAll: decoding: %v", err)
+	}
+	if err := cmp(dst, src); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestReaderReset(t *testing.T) {
+	gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
+	buf := new(bytes.Buffer)
+	if _, err := NewWriter(buf).Write(gold); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	encoded, invalid, partial := buf.String(), "invalid", "partial"
+	r := NewReader(nil)
+	for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
+		if s == partial {
+			r.Reset(strings.NewReader(encoded))
+			if _, err := r.Read(make([]byte, 101)); err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+			continue
+		}
+		r.Reset(strings.NewReader(s))
+		got, err := ioutil.ReadAll(r)
+		switch s {
+		case encoded:
+			if err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+			if err := cmp(got, gold); err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+		case invalid:
+			if err == nil {
+				t.Errorf("#%d: got nil error, want non-nil", i)
+				continue
+			}
+		}
+	}
+}
+
+func TestWriterReset(t *testing.T) {
+	gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
+	var gots, wants [][]byte
+	const n = 20
+	w, failed := NewWriter(nil), false
+	for i := 0; i <= n; i++ {
+		buf := new(bytes.Buffer)
+		w.Reset(buf)
+		want := gold[:len(gold)*i/n]
+		if _, err := w.Write(want); err != nil {
+			t.Errorf("#%d: Write: %v", i, err)
+			failed = true
+			continue
+		}
+		got, err := ioutil.ReadAll(NewReader(buf))
+		if err != nil {
+			t.Errorf("#%d: ReadAll: %v", i, err)
+			failed = true
+			continue
+		}
+		gots = append(gots, got)
+		wants = append(wants, want)
+	}
+	if failed {
+		return
+	}
+	for i := range gots {
+		if err := cmp(gots[i], wants[i]); err != nil {
+			t.Errorf("#%d: %v", i, err)
+		}
+	}
+}
+
 func benchDecode(b *testing.B, src []byte) {
 	encoded, err := Encode(nil, src)
 	if err != nil {
@@ -102,7 +219,7 @@ func benchEncode(b *testing.B, src []byte) {
 	}
 }
 
-func readFile(b *testing.B, filename string) []byte {
+func readFile(b testing.TB, filename string) []byte {
 	src, err := ioutil.ReadFile(filename)
 	if err != nil {
 		b.Fatalf("failed reading %s: %s", filename, err)
@@ -144,7 +261,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
 func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
 
 // testFiles' values are copied directly from
-// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc.
+// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
 // The label field is unused in snappy-go.
 var testFiles = []struct {
 	label    string
@@ -152,29 +269,36 @@ var testFiles = []struct {
 }{
 	{"html", "html"},
 	{"urls", "urls.10K"},
-	{"jpg", "house.jpg"},
-	{"pdf", "mapreduce-osdi-1.pdf"},
+	{"jpg", "fireworks.jpeg"},
+	{"jpg_200", "fireworks.jpeg"},
+	{"pdf", "paper-100k.pdf"},
 	{"html4", "html_x_4"},
-	{"cp", "cp.html"},
-	{"c", "fields.c"},
-	{"lsp", "grammar.lsp"},
-	{"xls", "kennedy.xls"},
 	{"txt1", "alice29.txt"},
 	{"txt2", "asyoulik.txt"},
 	{"txt3", "lcet10.txt"},
 	{"txt4", "plrabn12.txt"},
-	{"bin", "ptt5"},
-	{"sum", "sum"},
-	{"man", "xargs.1"},
 	{"pb", "geo.protodata"},
 	{"gaviota", "kppkn.gtb"},
 }
 
 // The test data files are present at this canonical URL.
-const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/"
+const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
 
 func downloadTestdata(basename string) (errRet error) {
-	filename := filepath.Join("testdata", basename)
+	filename := filepath.Join(*testdata, basename)
+	if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
+		return nil
+	}
+
+	if !*download {
+		return fmt.Errorf("test data not found; skipping benchmark without the -download flag")
+	}
+	// Download the official snappy C++ implementation reference test data
+	// files for benchmarking.
+	if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("failed to create testdata: %s", err)
+	}
+
 	f, err := os.Create(filename)
 	if err != nil {
 		return fmt.Errorf("failed to create %s: %s", filename, err)
@@ -185,36 +309,27 @@ func downloadTestdata(basename string) (errRet error) {
 			os.Remove(filename)
 		}
 	}()
-	resp, err := http.Get(baseURL + basename)
+	url := baseURL + basename
+	resp, err := http.Get(url)
 	if err != nil {
-		return fmt.Errorf("failed to download %s: %s", baseURL+basename, err)
+		return fmt.Errorf("failed to download %s: %s", url, err)
 	}
 	defer resp.Body.Close()
+	if s := resp.StatusCode; s != http.StatusOK {
+		return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
+	}
 	_, err = io.Copy(f, resp.Body)
 	if err != nil {
-		return fmt.Errorf("failed to write %s: %s", filename, err)
+		return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
 	}
 	return nil
 }
 
 func benchFile(b *testing.B, n int, decode bool) {
-	filename := filepath.Join("testdata", testFiles[n].filename)
-	if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
-		if !*download {
-			b.Fatal("test data not found; skipping benchmark without the -download flag")
-		}
-		// Download the official snappy C++ implementation reference test data
-		// files for benchmarking.
-		if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
-			b.Fatalf("failed to create testdata: %s", err)
-		}
-		for _, tf := range testFiles {
-			if err := downloadTestdata(tf.filename); err != nil {
-				b.Fatalf("failed to download testdata: %s", err)
-			}
-		}
+	if err := downloadTestdata(testFiles[n].filename); err != nil {
+		b.Fatalf("failed to download testdata: %s", err)
 	}
-	data := readFile(b, filename)
+	data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
 	if decode {
 		benchDecode(b, data)
 	} else {
@@ -235,12 +350,6 @@ func Benchmark_UFlat8(b *testing.B)  { benchFile(b, 8, true) }
 func Benchmark_UFlat9(b *testing.B)  { benchFile(b, 9, true) }
 func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
 func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
-func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) }
-func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) }
-func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) }
-func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) }
-func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) }
-func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) }
 func Benchmark_ZFlat0(b *testing.B)  { benchFile(b, 0, false) }
 func Benchmark_ZFlat1(b *testing.B)  { benchFile(b, 1, false) }
 func Benchmark_ZFlat2(b *testing.B)  { benchFile(b, 2, false) }
@@ -253,9 +362,3 @@ func Benchmark_ZFlat8(b *testing.B)  { benchFile(b, 8, false) }
 func Benchmark_ZFlat9(b *testing.B)  { benchFile(b, 9, false) }
 func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
 func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }
-func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) }
-func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) }
-func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) }
-func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) }
-func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) }
-func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }

Godeps/_workspace/src/github.com/thejerf/suture/.travis.yml

@@ -0,0 +1,7 @@
+language: go
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - 1.4
+  - tip

Godeps/_workspace/src/github.com/thejerf/suture/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2014 Barracuda Networks, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

Godeps/_workspace/src/github.com/thejerf/suture/README.md

@@ -0,0 +1,45 @@
+Suture
+======
+
+[![Build Status](https://travis-ci.org/thejerf/suture.png?branch=master)](https://travis-ci.org/thejerf/suture)
+
+Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" ->
+"sutree" -> "suture" -> holds your code together when it's trying to die.
+
+This is intended to be a production-quality library going into code that I
+will be very early on the phone tree to support when it goes down. However,
+it has not been deployed into something quite that serious yet. (I will
+update this statement when that changes.)
+
+It is intended to deal gracefully with the real failure cases that can
+occur with supervision trees (such as burning all your CPU time endlessly
+restarting dead services), while also making no unnecessary demands on the
+"service" code, and providing hooks to perform adequate logging with in a
+production environment.
+
+[A blog post describing the design decisions](http://www.jerf.org/iri/post/2930)
+is available.
+
+This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/suture),
+including an example, usage, and everything else you might expect from a
+README.md on GitHub. (DRY.)
+
+This is not currently tagged with particular git tags for Go as this is
+currently considered to be alpha code. As I move this into production and
+feel more confident about it, I'll give it relevant tags.
+
+Code Signing
+------------
+
+Starting with the commit after ac7cf8591b, I will be signing this repository
+with the ["jerf" keybase account](https://keybase.io/jerf).
+
+Aspiration
+----------
+
+One of the big wins the Erlang community has with their pervasive OTP
+support is that it makes it easy for them to distribute libraries that
+easily fit into the OTP paradigm. It ought to someday be considered a good
+idea to distribute libraries that provide some sort of supervisor tree
+functionality out of the box. It is possible to provide this functionality
+without explicitly depending on the Suture library.

Godeps/_workspace/src/github.com/thejerf/suture/pre-commit

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+GOLINTOUT=$(golint *go)
+
+if [ ! -z "$GOLINTOUT" -o "$?" != 0 ]; then
+    echo golint failed:
+    echo $GOLINTOUT
+    exit 1
+fi
+
+go test

Godeps/_workspace/src/github.com/thejerf/suture/suture.go

@@ -0,0 +1,650 @@
+/*
+
+Package suture provides Erlang-like supervisor trees.
+
+This implements Erlang-esque supervisor trees, as adapted for Go. This is
+intended to be an industrial-strength implementation, but it has not yet
+been deployed in a hostile environment. (It's headed there, though.)
+
+Supervisor Tree -> SuTree -> suture -> holds your code together when it's
+trying to fall apart.
+
+Why use Suture?
+
+ * You want to write bullet-resistant services that will remain available
+   despite unforeseen failure.
+ * You need the code to be smart enough not to consume 100% of the CPU
+   restarting things.
+ * You want to easily compose multiple such services in one program.
+ * You want the Erlang programmers to stop lording their supervision
+   trees over you.
+
+Suture has 100% test coverage, and is golint clean. This doesn't prove it
+free of bugs, but it shows I care.
+
+A blog post describing the design decisions is available at
+http://www.jerf.org/iri/post/2930 .
+
+Using Suture
+
+To idiomatically use Suture, create a Supervisor which is your top level
+"application" supervisor. This will often occur in your program's "main"
+function.
+
+Create "Service"s, which implement the Service interface. .Add() them
+to your Supervisor. Supervisors are also services, so you can create a
+tree structure here, depending on the exact combination of restarts
+you want to create.
+
+Finally, as what is probably the last line of your main() function, call
+.Serve() on your top level supervisor. This will start all the services
+you've defined.
+
+See the Example for an example, using a simple service that serves out
+incrementing integers.
+
+*/
+package suture
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	notRunning = iota
+	normal
+	paused
+)
+
+type supervisorID uint32
+type serviceID uint32
+
+var currentSupervisorID uint32
+
+// ErrWrongSupervisor is returned by the (*Supervisor).Remove method
+// if you pass a ServiceToken from the wrong Supervisor.
+var ErrWrongSupervisor = errors.New("wrong supervisor for this service token, no service removed")
+
+// ServiceToken is an opaque identifier that can be used to terminate a service that
+// has been Add()ed to a Supervisor.
+type ServiceToken struct {
+	id uint64
+}
+
+/*
+Supervisor is the core type of the module that represents a Supervisor.
+
+Supervisors should be constructed either by New or NewSimple.
+
+Once constructed, a Supervisor should be started in one of three ways:
+
+ 1. Calling .Serve().
+ 2. Calling .ServeBackground().
+ 3. Adding it to an existing Supervisor.
+
+Calling Serve will cause the supervisor to run until it is shut down by
+an external user calling Stop() on it. If that never happens, it simply
+runs forever. I suggest creating your services in Supervisors, then making
+a Serve() call on your top-level Supervisor be the last line of your main
+func.
+
+Calling ServeBackground will CORRECTLY start the supervisor running in a
+new goroutine. You do not want to just:
+
+  go supervisor.Serve()
+
+because that will briefly create a race condition as it starts up, if you
+try to .Add() services immediately afterward.
+
+*/
+type Supervisor struct {
+	Name string
+	id   supervisorID
+
+	failureDecay     float64
+	failureThreshold float64
+	failureBackoff   time.Duration
+	timeout          time.Duration
+	log              func(string)
+	services         map[serviceID]Service
+	lastFail         time.Time
+	failures         float64
+	restartQueue     []serviceID
+	state            uint8
+	serviceCounter   serviceID
+	control          chan supervisorMessage
+	resumeTimer      <-chan time.Time
+
+	// The testing uses the ability to grab these individual logging functions
+	// and get inside of suture's handling at a deep level.
+	// If you ever come up with some need to get into these, submit a pull
+	// request to make them public and some smidge of justification, and
+	// I'll happily do it.
+	logBadStop func(Service)
+	logFailure func(service Service, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte)
+	logBackoff func(*Supervisor, bool)
+
+	// avoid a dependency on github.com/thejerf/abtime by just implementing
+	// a minimal chunk.
+	getNow    func() time.Time
+	getResume func(time.Duration) <-chan time.Time
+}
+
+// Spec is used to pass arguments to the New function to create a
+// supervisor. See the New function for full documentation.
+type Spec struct {
+	Log              func(string)
+	FailureDecay     float64
+	FailureThreshold float64
+	FailureBackoff   time.Duration
+	Timeout          time.Duration
+}
+
+/*
+
+New is the full constructor function for a supervisor.
+
+The name is a friendly human name for the supervisor, used in logging. Suture
+does not care if this is unique, but it is good for your sanity if it is.
+
+If not set, the following values are used:
+
+ * Log:               A function is created that uses log.Print.
+ * FailureDecay:      30 seconds
+ * FailureThreshold:  5 failures
+ * FailureBackoff:    15 seconds
+ * Timeout:           10 seconds
+
+The Log function will be called when errors occur. Suture will log the
+following:
+
+ * When a service has failed, with a descriptive message about the
+   current backoff status, and whether it was immediately restarted
+ * When the supervisor has gone into its backoff mode, and when it
+   exits it
+ * When a service fails to stop
+
+The failureRate, failureThreshold, and failureBackoff controls how failures
+are handled, in order to avoid the supervisor failure case where the
+program does nothing but restarting failed services. If you do not
+care how failures behave, the default values should be fine for the
+vast majority of services, but if you want the details:
+
+The supervisor tracks the number of failures that have occurred, with an
+exponential decay on the count. Every FailureDecay seconds, the number of
+failures that have occurred is cut in half. (This is done smoothly with an
+exponential function.) When a failure occurs, the number of failures
+is incremented by one. When the number of failures passes the
+FailureThreshold, the entire service waits for FailureBackoff seconds
+before attempting any further restarts, at which point it resets its
+failure count to zero.
+
+Timeout is how long Suture will wait for a service to properly terminate.
+
+*/
+func New(name string, spec Spec) (s *Supervisor) {
+	s = new(Supervisor)
+
+	s.Name = name
+	s.id = supervisorID(atomic.AddUint32(&currentSupervisorID, 1))
+
+	if spec.Log == nil {
+		s.log = func(msg string) {
+			log.Print(fmt.Sprintf("Supervisor %s: %s", s.Name, msg))
+		}
+	} else {
+		s.log = spec.Log
+	}
+
+	if spec.FailureDecay == 0 {
+		s.failureDecay = 30
+	} else {
+		s.failureDecay = spec.FailureDecay
+	}
+	if spec.FailureThreshold == 0 {
+		s.failureThreshold = 5
+	} else {
+		s.failureThreshold = spec.FailureThreshold
+	}
+	if spec.FailureBackoff == 0 {
+		s.failureBackoff = time.Second * 15
+	} else {
+		s.failureBackoff = spec.FailureBackoff
+	}
+	if spec.Timeout == 0 {
+		s.timeout = time.Second * 10
+	} else {
+		s.timeout = spec.Timeout
+	}
+
+	// overriding these allows for testing the threshold behavior
+	s.getNow = time.Now
+	s.getResume = time.After
+
+	s.control = make(chan supervisorMessage)
+	s.services = make(map[serviceID]Service)
+	s.restartQueue = make([]serviceID, 0, 1)
+	s.resumeTimer = make(chan time.Time)
+
+	// set up the default logging handlers
+	s.logBadStop = func(service Service) {
+		s.log(fmt.Sprintf("Service %s failed to terminate in a timely manner", serviceName(service)))
+	}
+	s.logFailure = func(service Service, failures float64, threshold float64, restarting bool, err interface{}, st []byte) {
+		var errString string
+
+		e, canError := err.(error)
+		if canError {
+			errString = e.Error()
+		} else {
+			errString = fmt.Sprintf("%#v", err)
+		}
+
+		s.log(fmt.Sprintf("Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", serviceName(service), failures, threshold, restarting, errString, string(st)))
+	}
+	s.logBackoff = func(s *Supervisor, entering bool) {
+		if entering {
+			s.log("Entering the backoff state.")
+		} else {
+			s.log("Exiting backoff state.")
+		}
+	}
+
+	return
+}
+
+func serviceName(service Service) (serviceName string) {
+	stringer, canStringer := service.(fmt.Stringer)
+	if canStringer {
+		serviceName = stringer.String()
+	} else {
+		serviceName = fmt.Sprintf("%#v", service)
+	}
+	return
+}
+
+// NewSimple is a convenience function to create a service with just a name
+// and the sensible defaults.
+func NewSimple(name string) *Supervisor {
+	return New(name, Spec{})
+}
+
+/*
+Service is the interface that describes a service to a Supervisor.
+
+Serve Method
+
+The Serve method is called by a Supervisor to start the service.
+The service should execute within the goroutine that this is
+called in. If this function either returns or panics, the Supervisor
+will call it again.
+
+A Serve method SHOULD do as much cleanup of the state as possible,
+to prevent any corruption in the previous state from crashing the
+service again.
+
+Stop Method
+
+This method is used by the supervisor to stop the service. Calling this
+directly on a Service given to a Supervisor will simply result in the
+Service being restarted; use the Supervisor's .Remove(ServiceToken) method
+to stop a service. A supervisor will call .Stop() only once. Thus, it may
+be as destructive as it likes to get the service to stop.
+
+Once Stop has been called on a Service, the Service SHOULD NOT be
+reused in any other supervisor! Because of the impossibility of
+guaranteeing that the service has actually stopped in Go, you can't
+prove that you won't be starting two goroutines using the exact
+same memory to store state, causing completely unpredictable behavior.
+
+Stop should not return until the service has actually stopped.
+"Stopped" here is defined as "the service will stop servicing any
+further requests in the future". For instance, a common implementation
+is to receive a message on a dedicated "stop" channel and immediately
+returning. Once the stop command has been processed, the service is
+stopped.
+
+Another common Stop implementation is to forcibly close an open socket
+or other resource, which will cause detectable errors to manifest in the
+service code. Bear in mind that to perfectly correctly use this
+approach requires a bit more work to handle the chance of a Stop
+command coming in before the resource has been created.
+
+If a service does not Stop within the supervisor's timeout duration, a log
+entry will be made with a descriptive string to that effect. This does
+not guarantee that the service is hung; it may still get around to being
+properly stopped in the future. Until the service is fully stopped,
+both the service and the spawned goroutine trying to stop it will be
+"leaked".
+
+Stringer Interface
+
+It is not mandatory to implement the fmt.Stringer interface on your
+service, but if your Service does happen to implement that, the log
+messages that describe your service will use that when naming the
+service. Otherwise, you'll see the GoString of your service object,
+obtained via fmt.Sprintf("%#v", service).
+
+*/
+type Service interface {
+	Serve()
+	Stop()
+}
+
+/*
+Add adds a service to this supervisor.
+
+If the supervisor is currently running, the service will be started
+immediately. If the supervisor is not currently running, the service
+will be started when the supervisor is.
+
+The returned ServiceID may be passed to the Remove method of the Supervisor
+to terminate the service.
+*/
+func (s *Supervisor) Add(service Service) ServiceToken {
+	if s == nil {
+		panic("can't add service to nil *suture.Supervisor")
+	}
+
+	if s.state == notRunning {
+		id := s.serviceCounter
+		s.serviceCounter++
+
+		s.services[id] = service
+		s.restartQueue = append(s.restartQueue, id)
+
+		return ServiceToken{uint64(s.id)<<32 | uint64(id)}
+	}
+
+	response := make(chan serviceID)
+	s.control <- addService{service, response}
+	return ServiceToken{uint64(s.id)<<32 | uint64(<-response)}
+}
+
+// ServeBackground starts running a supervisor in its own goroutine. This
+// method does not return until it is safe to use .Add() on the Supervisor.
+func (s *Supervisor) ServeBackground() {
+	go s.Serve()
+	s.sync()
+}
+
+/*
+Serve starts the supervisor. You should call this on the top-level supervisor,
+but nothing else.
+*/
+func (s *Supervisor) Serve() {
+	if s == nil {
+		panic("Can't serve with a nil *suture.Supervisor")
+	}
+	if s.id == 0 {
+		panic("Can't call Serve on an incorrectly-constructed *suture.Supervisor")
+	}
+
+	defer func() {
+		s.state = notRunning
+	}()
+
+	if s.state != notRunning {
+		// FIXME: Don't explain why I don't need a semaphore, just use one
+		// This doesn't use a semaphore because it's just a sanity check.
+		panic("Running a supervisor while it is already running?")
+	}
+
+	s.state = normal
+
+	// for all the services I currently know about, start them
+	for _, id := range s.restartQueue {
+		service, present := s.services[id]
+		if present {
+			s.runService(service, id)
+		}
+	}
+	s.restartQueue = make([]serviceID, 0, 1)
+
+	for {
+		select {
+		case m := <-s.control:
+			switch msg := m.(type) {
+			case serviceFailed:
+				s.handleFailedService(msg.id, msg.err, msg.stacktrace)
+			case serviceEnded:
+				service, monitored := s.services[msg.id]
+				if monitored {
+					s.handleFailedService(msg.id, fmt.Sprintf("%s returned unexpectedly", service), []byte("[unknown stack trace]"))
+				}
+			case addService:
+				id := s.serviceCounter
+				s.serviceCounter++
+
+				s.services[id] = msg.service
+				s.runService(msg.service, id)
+
+				msg.response <- id
+			case removeService:
+				s.removeService(msg.id)
+			case stopSupervisor:
+				for id := range s.services {
+					s.removeService(id)
+				}
+				return
+			case listServices:
+				services := []Service{}
+				for _, service := range s.services {
+					services = append(services, service)
+				}
+				msg.c <- services
+			case syncSupervisor:
+				// this does nothing on purpose; its sole purpose is to
+				// introduce a sync point via the channel receive
+			case panicSupervisor:
+				// used only by tests
+				panic("Panicking as requested!")
+			}
+		case _ = <-s.resumeTimer:
+			// We're resuming normal operation after a pause due to
+			// excessive thrashing
+			// FIXME: Ought to permit some spacing of these functions, rather
+			// than simply hammering through them
+			s.state = normal
+			s.failures = 0
+			s.logBackoff(s, false)
+			for _, id := range s.restartQueue {
+				service, present := s.services[id]
+				if present {
+					s.runService(service, id)
+				}
+			}
+			s.restartQueue = make([]serviceID, 0, 1)
+		}
+	}
+}
+
+func (s *Supervisor) handleFailedService(id serviceID, err interface{}, stacktrace []byte) {
+	now := s.getNow()
+
+	if s.lastFail.IsZero() {
+		s.lastFail = now
+		s.failures = 1.0
+	} else {
+		sinceLastFail := now.Sub(s.lastFail).Seconds()
+		intervals := sinceLastFail / s.failureDecay
+		s.failures = s.failures*math.Pow(.5, intervals) + 1
+	}
+
+	if s.failures > s.failureThreshold {
+		s.state = paused
+		s.logBackoff(s, true)
+		s.resumeTimer = s.getResume(s.failureBackoff)
+	}
+
+	s.lastFail = now
+
+	failedService, monitored := s.services[id]
+
+	// It is possible for a service to be no longer monitored
+	// by the time we get here. In that case, just ignore it.
+	if monitored {
+		if s.state == normal {
+			s.runService(failedService, id)
+			s.logFailure(failedService, s.failures, s.failureThreshold, true, err, stacktrace)
+		} else {
+			// FIXME: When restarting, check that the service still
+			// exists (it may have been stopped in the meantime)
+			s.restartQueue = append(s.restartQueue, id)
+			s.logFailure(failedService, s.failures, s.failureThreshold, false, err, stacktrace)
+		}
+	}
+}
+
+func (s *Supervisor) runService(service Service, id serviceID) {
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				buf := make([]byte, 65535, 65535)
+				written := runtime.Stack(buf, false)
+				buf = buf[:written]
+				s.fail(id, r, buf)
+			}
+		}()
+
+		service.Serve()
+
+		s.serviceEnded(id)
+	}()
+}
+
+func (s *Supervisor) removeService(id serviceID) {
+	service, present := s.services[id]
+	if present {
+		delete(s.services, id)
+		go func() {
+			successChan := make(chan bool)
+			go func() {
+				service.Stop()
+				successChan <- true
+			}()
+
+			failChan := s.getResume(s.timeout)
+
+			select {
+			case <-successChan:
+				// Life is good!
+			case <-failChan:
+				s.logBadStop(service)
+			}
+		}()
+	}
+}
+
+// String implements the fmt.Stringer interface.
+func (s *Supervisor) String() string {
+	return s.Name
+}
+
+// sum type pattern for type-safe message passing; see
+// http://www.jerf.org/iri/post/2917
+
+type supervisorMessage interface {
+	isSupervisorMessage()
+}
+
+/*
+Remove will remove the given service from the Supervisor, and attempt to Stop() it.
+The ServiceID token comes from the Add() call.
+*/
+func (s *Supervisor) Remove(id ServiceToken) error {
+	sID := supervisorID(id.id >> 32)
+	if sID != s.id {
+		return ErrWrongSupervisor
+	}
+	s.control <- removeService{serviceID(id.id & 0xffffffff)}
+	return nil
+}
+
+/*
+
+Services returns a []Service containing a snapshot of the services this
+Supervisor is managing.
+
+*/
+func (s *Supervisor) Services() []Service {
+	ls := listServices{make(chan []Service)}
+	s.control <- ls
+	return <-ls.c
+}
+
+type listServices struct {
+	c chan []Service
+}
+
+func (ls listServices) isSupervisorMessage() {}
+
+type removeService struct {
+	id serviceID
+}
+
+func (rs removeService) isSupervisorMessage() {}
+
+func (s *Supervisor) sync() {
+	s.control <- syncSupervisor{}
+}
+
+type syncSupervisor struct {
+}
+
+func (ss syncSupervisor) isSupervisorMessage() {}
+
+func (s *Supervisor) fail(id serviceID, err interface{}, stacktrace []byte) {
+	s.control <- serviceFailed{id, err, stacktrace}
+}
+
+type serviceFailed struct {
+	id         serviceID
+	err        interface{}
+	stacktrace []byte
+}
+
+func (sf serviceFailed) isSupervisorMessage() {}
+
+func (s *Supervisor) serviceEnded(id serviceID) {
+	s.control <- serviceEnded{id}
+}
+
+type serviceEnded struct {
+	id serviceID
+}
+
+func (s serviceEnded) isSupervisorMessage() {}
+
+// added by the Add() method
+type addService struct {
+	service  Service
+	response chan serviceID
+}
+
+func (as addService) isSupervisorMessage() {}
+
+// Stop stops the Supervisor.
+func (s *Supervisor) Stop() {
+	s.control <- stopSupervisor{}
+}
+
+type stopSupervisor struct {
+}
+
+func (ss stopSupervisor) isSupervisorMessage() {}
+
+func (s *Supervisor) panic() {
+	s.control <- panicSupervisor{}
+}
+
+type panicSupervisor struct {
+}
+
+func (ps panicSupervisor) isSupervisorMessage() {}

Godeps/_workspace/src/github.com/thejerf/suture/suture_simple_test.go

@@ -0,0 +1,49 @@
+package suture
+
+import "fmt"
+
+type Incrementor struct {
+	current int
+	next    chan int
+	stop    chan bool
+}
+
+func (i *Incrementor) Stop() {
+	fmt.Println("Stopping the service")
+	i.stop <- true
+}
+
+func (i *Incrementor) Serve() {
+	for {
+		select {
+		case i.next <- i.current:
+			i.current += 1
+		case <-i.stop:
+			// We sync here just to guarantee the output of "Stopping the service",
+			// so this passes the test reliably.
+			// Most services would simply "return" here.
+			i.stop <- true
+			return
+		}
+	}
+}
+
+func ExampleNew_simple() {
+	supervisor := NewSimple("Supervisor")
+	service := &Incrementor{0, make(chan int), make(chan bool)}
+	supervisor.Add(service)
+
+	go supervisor.ServeBackground()
+
+	fmt.Println("Got:", <-service.next)
+	fmt.Println("Got:", <-service.next)
+	supervisor.Stop()
+
+	// We sync here just to guarantee the output of "Stopping the service"
+	<-service.stop
+
+	// Output:
+	// Got: 0
+	// Got: 1
+	// Stopping the service
+}

Godeps/_workspace/src/github.com/thejerf/suture/suture_test.go

@@ -0,0 +1,592 @@
+package suture
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+const (
+	Happy = iota
+	Fail
+	Panic
+	Hang
+	UseStopChan
+)
+
+var everMultistarted = false
+
+// Test that supervisors work perfectly when everything is hunky dory.
+func TestTheHappyCase(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A")
+	if s.String() != "A" {
+		t.Fatal("Can't get name from a supervisor")
+	}
+	service := NewService("B")
+
+	s.Add(service)
+
+	go s.Serve()
+
+	<-service.started
+
+	// If we stop the service, it just gets restarted
+	service.Stop()
+	<-service.started
+
+	// And it is shut down when we stop the supervisor
+	service.take <- UseStopChan
+	s.Stop()
+	<-service.stop
+}
+
+// Test that adding to a running supervisor does indeed start the service.
+func TestAddingToRunningSupervisor(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A1")
+
+	s.ServeBackground()
+	defer s.Stop()
+
+	service := NewService("B1")
+	s.Add(service)
+
+	<-service.started
+
+	services := s.Services()
+	if !reflect.DeepEqual([]Service{service}, services) {
+		t.Fatal("Can't get list of services as expected.")
+	}
+}
+
+// Test what happens when services fail.
+func TestFailures(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A2")
+	s.failureThreshold = 3.5
+
+	go s.Serve()
+	defer func() {
+		// to avoid deadlocks during shutdown, we have to not try to send
+		// things out on channels while we're shutting down (this undoes the
+		// logFailure overide about 25 lines down)
+		s.logFailure = func(Service, float64, float64, bool, interface{}, []byte) {}
+		s.Stop()
+	}()
+	s.sync()
+
+	service1 := NewService("B2")
+	service2 := NewService("C2")
+
+	s.Add(service1)
+	<-service1.started
+	s.Add(service2)
+	<-service2.started
+
+	nowFeeder := NewNowFeeder()
+	pastVal := time.Unix(1000000, 0)
+	nowFeeder.appendTimes(pastVal)
+	s.getNow = nowFeeder.getter
+
+	resumeChan := make(chan time.Time)
+	s.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+
+	failNotify := make(chan bool)
+	// use this to synchronize on here
+	s.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
+		failNotify <- r
+	}
+
+	// All that setup was for this: Service1, please return now.
+	service1.take <- Fail
+	restarted := <-failNotify
+	<-service1.started
+
+	if !restarted || s.failures != 1 || s.lastFail != pastVal {
+		t.Fatal("Did not fail in the expected manner")
+	}
+	// Getting past this means the service was restarted.
+	service1.take <- Happy
+
+	// Service2, your turn.
+	service2.take <- Fail
+	nowFeeder.appendTimes(pastVal)
+	restarted = <-failNotify
+	<-service2.started
+	if !restarted || s.failures != 2 || s.lastFail != pastVal {
+		t.Fatal("Did not fail in the expected manner")
+	}
+	// And you're back. (That is, the correct service was restarted.)
+	service2.take <- Happy
+
+	// Now, one failureDecay later, is everything working correctly?
+	oneDecayLater := time.Unix(1000030, 0)
+	nowFeeder.appendTimes(oneDecayLater)
+	service2.take <- Fail
+	restarted = <-failNotify
+	<-service2.started
+	// playing a bit fast and loose here with floating point, but...
+	// we get 2 by taking the current failure value of 2, decaying it
+	// by one interval, which cuts it in half to 1, then adding 1 again,
+	// all of which "should" be precise
+	if !restarted || s.failures != 2 || s.lastFail != oneDecayLater {
+		t.Fatal("Did not decay properly", s.lastFail, oneDecayLater)
+	}
+
+	// For a change of pace, service1 would you be so kind as to panic?
+	nowFeeder.appendTimes(oneDecayLater)
+	service1.take <- Panic
+	restarted = <-failNotify
+	<-service1.started
+	if !restarted || s.failures != 3 || s.lastFail != oneDecayLater {
+		t.Fatal("Did not correctly recover from a panic")
+	}
+
+	nowFeeder.appendTimes(oneDecayLater)
+	backingoff := make(chan bool)
+	s.logBackoff = func(s *Supervisor, backingOff bool) {
+		backingoff <- backingOff
+	}
+
+	// And with this failure, we trigger the backoff code.
+	service1.take <- Fail
+	backoff := <-backingoff
+	restarted = <-failNotify
+
+	if !backoff || restarted || s.failures != 4 {
+		t.Fatal("Broke past the threshold but did not log correctly", s.failures)
+	}
+	if service1.existing != 0 {
+		t.Fatal("service1 still exists according to itself?")
+	}
+
+	// service2 is still running, because we don't shut anything down in a
+	// backoff, we just stop restarting.
+	service2.take <- Happy
+
+	var correct bool
+	timer := time.NewTimer(time.Millisecond * 10)
+	// verify the service has not been restarted
+	// hard to get around race conditions here without simply using a timer...
+	select {
+	case service1.take <- Happy:
+		correct = false
+	case <-timer.C:
+		correct = true
+	}
+	if !correct {
+		t.Fatal("Restarted the service during the backoff interval")
+	}
+
+	// tell the supervisor the restart interval has passed
+	resumeChan <- time.Time{}
+	backoff = <-backingoff
+	<-service1.started
+	s.sync()
+	if s.failures != 0 {
+		t.Fatal("Did not reset failure count after coming back from timeout.")
+	}
+
+	nowFeeder.appendTimes(oneDecayLater)
+	service1.take <- Fail
+	restarted = <-failNotify
+	<-service1.started
+	if !restarted || backoff {
+		t.Fatal("For some reason, got that we were backing off again.", restarted, backoff)
+	}
+}
+
+func TestRunningAlreadyRunning(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A3")
+	go s.Serve()
+	defer s.Stop()
+
+	// ensure the supervisor has made it to its main loop
+	s.sync()
+	var errored bool
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				errored = true
+			}
+		}()
+
+		s.Serve()
+	}()
+	if !errored {
+		t.Fatal("Supervisor failed to prevent itself from double-running.")
+	}
+}
+
+func TestFullConstruction(t *testing.T) {
+	t.Parallel()
+
+	s := New("Moo", Spec{
+		Log:              func(string) {},
+		FailureDecay:     1,
+		FailureThreshold: 2,
+		FailureBackoff:   3,
+		Timeout:          time.Second * 29,
+	})
+	if s.String() != "Moo" || s.failureDecay != 1 || s.failureThreshold != 2 || s.failureBackoff != 3 || s.timeout != time.Second*29 {
+		t.Fatal("Full construction failed somehow")
+	}
+}
+
+// This is mostly for coverage testing.
+func TestDefaultLogging(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A4")
+
+	service := NewService("B4")
+	s.Add(service)
+
+	s.failureThreshold = .5
+	s.failureBackoff = time.Millisecond * 25
+	go s.Serve()
+	s.sync()
+
+	<-service.started
+
+	resumeChan := make(chan time.Time)
+	s.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+
+	service.take <- UseStopChan
+	service.take <- Fail
+	<-service.stop
+	resumeChan <- time.Time{}
+
+	<-service.started
+
+	service.take <- Happy
+
+	serviceName(&BarelyService{})
+
+	s.logBadStop(service)
+	s.logFailure(service, 1, 1, true, errors.New("test error"), []byte{})
+
+	s.Stop()
+}
+
+func TestNestedSupervisors(t *testing.T) {
+	t.Parallel()
+
+	super1 := NewSimple("Top5")
+	super2 := NewSimple("Nested5")
+	service := NewService("Service5")
+
+	super1.Add(super2)
+	super2.Add(service)
+
+	go super1.Serve()
+	super1.sync()
+
+	<-service.started
+	service.take <- Happy
+
+	super1.Stop()
+}
+
+func TestStoppingSupervisorStopsServices(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("Top6")
+	service := NewService("Service 6")
+
+	s.Add(service)
+
+	go s.Serve()
+	s.sync()
+
+	<-service.started
+
+	service.take <- UseStopChan
+
+	s.Stop()
+	<-service.stop
+}
+
+func TestStoppingStillWorksWithHungServices(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("Top7")
+	service := NewService("Service WillHang7")
+
+	s.Add(service)
+
+	go s.Serve()
+
+	<-service.started
+
+	service.take <- UseStopChan
+	service.take <- Hang
+
+	resumeChan := make(chan time.Time)
+	s.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+	failNotify := make(chan struct{})
+	s.logBadStop = func(s Service) {
+		failNotify <- struct{}{}
+	}
+
+	s.Stop()
+
+	resumeChan <- time.Time{}
+	<-failNotify
+	service.release <- true
+	<-service.stop
+}
+
+func TestRemoveService(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("Top")
+	service := NewService("ServiceToRemove8")
+
+	id := s.Add(service)
+
+	go s.Serve()
+
+	<-service.started
+	service.take <- UseStopChan
+
+	err := s.Remove(id)
+	if err != nil {
+		t.Fatal("Removing service somehow failed")
+	}
+	<-service.stop
+
+	err = s.Remove(ServiceToken{1<<36 + 1})
+	if err != ErrWrongSupervisor {
+		t.Fatal("Did not detect that the ServiceToken was wrong")
+	}
+}
+
+func TestFailureToConstruct(t *testing.T) {
+	t.Parallel()
+
+	var s *Supervisor
+
+	panics(func() {
+		s.Serve()
+	})
+
+	s = new(Supervisor)
+	panics(func() {
+		s.Serve()
+	})
+}
+
+func TestFailingSupervisors(t *testing.T) {
+	t.Parallel()
+
+	// This is a bit of a complicated test, so let me explain what
+	// all this is doing:
+	// 1. Set up a top-level supervisor with a hair-trigger backoff.
+	// 2. Add a supervisor to that.
+	// 3. To that supervisor, add a service.
+	// 4. Panic the supervisor in the middle, sending the top-level into
+	//    backoff.
+	// 5. Kill the lower level service too.
+	// 6. Verify that when the top-level service comes out of backoff,
+	//    the service ends up restarted as expected.
+
+	// Ultimately, we can't have more than a best-effort recovery here.
+	// A panic'ed supervisor can't really be trusted to have consistent state,
+	// and without *that*, we can't trust it to do anything sensible with
+	// the children it may have been running. So unlike Erlang, we can't
+	// can't really expect to be able to safely restart them or anything.
+	// Really, the "correct" answer is that the Supervisor must never panic,
+	// but in the event that it does, this verifies that it at least tries
+	// to get on with life.
+
+	// This also tests that if a Supervisor itself panics, and one of its
+	// monitored services goes down in the meantime, that the monitored
+	// service also gets correctly restarted when the supervisor does.
+
+	s1 := NewSimple("Top9")
+	s2 := NewSimple("Nested9")
+	service := NewService("Service9")
+
+	s1.Add(s2)
+	s2.Add(service)
+
+	go s1.Serve()
+	<-service.started
+
+	s1.failureThreshold = .5
+
+	// let us control precisely when s1 comes back
+	resumeChan := make(chan time.Time)
+	s1.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+	failNotify := make(chan string)
+	// use this to synchronize on here
+	s1.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
+		failNotify <- fmt.Sprintf("%s", s)
+	}
+
+	s2.panic()
+
+	failing := <-failNotify
+	// that's enough sync to guarantee this:
+	if failing != "Nested9" || s1.state != paused {
+		t.Fatal("Top-level supervisor did not go into backoff as expected")
+	}
+
+	service.take <- Fail
+
+	resumeChan <- time.Time{}
+	<-service.started
+}
+
+func TestNilSupervisorAdd(t *testing.T) {
+	t.Parallel()
+
+	var s *Supervisor
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatal("did not panic as expected on nil add")
+		}
+	}()
+
+	s.Add(s)
+}
+
+// http://golangtutorials.blogspot.com/2011/10/gotest-unit-testing-and-benchmarking-go.html
+// claims test function are run in the same order as the source file...
+// I'm not sure if this is part of the contract, though. Especially in the
+// face of "t.Parallel()"...
+func TestEverMultistarted(t *testing.T) {
+	if everMultistarted {
+		t.Fatal("Seem to have multistarted a service at some point, bummer.")
+	}
+}
+
+// A test service that can be induced to fail, panic, or hang on demand.
+func NewService(name string) *FailableService {
+	return &FailableService{name, make(chan bool), make(chan int),
+		make(chan bool, 1), make(chan bool), make(chan bool), 0}
+}
+
+type FailableService struct {
+	name     string
+	started  chan bool
+	take     chan int
+	shutdown chan bool
+	release  chan bool
+	stop     chan bool
+	existing int
+}
+
+func (s *FailableService) Serve() {
+	if s.existing != 0 {
+		everMultistarted = true
+		panic("Multi-started the same service! " + s.name)
+	}
+	s.existing += 1
+
+	s.started <- true
+
+	useStopChan := false
+
+	for {
+		select {
+		case val := <-s.take:
+			switch val {
+			case Happy:
+				// Do nothing on purpose. Life is good!
+			case Fail:
+				s.existing -= 1
+				if useStopChan {
+					s.stop <- true
+				}
+				return
+			case Panic:
+				s.existing -= 1
+				panic("Panic!")
+			case Hang:
+				// or more specifically, "hang until I release you"
+				<-s.release
+			case UseStopChan:
+				useStopChan = true
+			}
+		case <-s.shutdown:
+			s.existing -= 1
+			if useStopChan {
+				s.stop <- true
+			}
+			return
+		}
+	}
+}
+
+func (s *FailableService) String() string {
+	return s.name
+}
+
+func (s *FailableService) Stop() {
+	s.shutdown <- true
+}
+
+type NowFeeder struct {
+	values []time.Time
+	getter func() time.Time
+	m      sync.Mutex
+}
+
+// This is used to test serviceName; it's a service without a Stringer.
+type BarelyService struct{}
+
+func (bs *BarelyService) Serve() {}
+func (bs *BarelyService) Stop()  {}
+
+func NewNowFeeder() (nf *NowFeeder) {
+	nf = new(NowFeeder)
+	nf.getter = func() time.Time {
+		nf.m.Lock()
+		defer nf.m.Unlock()
+		if len(nf.values) > 0 {
+			ret := nf.values[0]
+			nf.values = nf.values[1:]
+			return ret
+		}
+		panic("Ran out of values for NowFeeder")
+	}
+	return
+}
+
+func (nf *NowFeeder) appendTimes(t ...time.Time) {
+	nf.m.Lock()
+	defer nf.m.Unlock()
+	nf.values = append(nf.values, t...)
+}
+
+func panics(doesItPanic func()) (panics bool) {
+	defer func() {
+		if r := recover(); r != nil {
+			panics = true
+		}
+	}()
+
+	doesItPanic()
+
+	return
+}

Godeps/_workspace/src/golang.org/x/text/unicode/norm/Makefile

@@ -1,23 +0,0 @@
-# Copyright 2011 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-maketables: maketables.go triegen.go
-	go build $^
-
-normregtest: normregtest.go
-	go build $^
-
-tables:	maketables
-	./maketables > tables.go
-	gofmt -w tables.go
-
-# Downloads from www.unicode.org, so not part
-# of standard test scripts.
-test: testtables regtest
-
-testtables: maketables
-	./maketables -test > data_test.go && go test -tags=test
-
-regtest: normregtest
-	./normregtest

Godeps/_workspace/src/golang.org/x/text/unicode/norm/maketables.go

@@ -16,20 +16,17 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"net/http"
-	"os"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
-	"unicode"
 
+	"golang.org/x/text/internal/gen"
 	"golang.org/x/text/internal/triegen"
 	"golang.org/x/text/internal/ucd"
 )
 
 func main() {
-	flag.Parse()
+	gen.Init()
 	loadUnicodeData()
 	compactCCC()
 	loadCompositionExclusions()
@@ -46,24 +43,18 @@ func main() {
 	}
 }
 
-var url = flag.String("url",
-	"http://www.unicode.org/Public/"+unicode.Version+"/ucd/",
-	"URL of Unicode database directory")
-var tablelist = flag.String("tables",
-	"all",
-	"comma-separated list of which tables to generate; "+
-		"can be 'decomp', 'recomp', 'info' and 'all'")
-var test = flag.Bool("test",
-	false,
-	"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
-var verbose = flag.Bool("verbose",
-	false,
-	"write data to stdout as it is parsed")
-var localFiles = flag.Bool("local",
-	false,
-	"data files have been copied to the current directory; for debugging only")
-
-var logger = log.New(os.Stderr, "", log.Lshortfile)
+var (
+	tablelist = flag.String("tables",
+		"all",
+		"comma-separated list of which tables to generate; "+
+			"can be 'decomp', 'recomp', 'info' and 'all'")
+	test = flag.Bool("test",
+		false,
+		"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
+	verbose = flag.Bool("verbose",
+		false,
+		"write data to stdout as it is parsed")
+)
 
 const MaxChar = 0x10FFFF // anything above this shouldn't exist
 
@@ -189,27 +180,6 @@ func (f FormInfo) String() string {
 
 type Decomposition []rune
 
-func openReader(file string) (input io.ReadCloser) {
-	if *localFiles {
-		f, err := os.Open(file)
-		if err != nil {
-			logger.Fatal(err)
-		}
-		input = f
-	} else {
-		path := *url + file
-		resp, err := http.Get(path)
-		if err != nil {
-			logger.Fatal(err)
-		}
-		if resp.StatusCode != 200 {
-			logger.Fatal("bad GET status for "+file, resp.Status)
-		}
-		input = resp.Body
-	}
-	return
-}
-
 func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 	decomp := strings.Split(s, " ")
 	if len(decomp) > 0 && skipfirst {
@@ -226,7 +196,7 @@ func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 }
 
 func loadUnicodeData() {
-	f := openReader("UnicodeData.txt")
+	f := gen.OpenUCDFile("UnicodeData.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
@@ -242,7 +212,7 @@ func loadUnicodeData() {
 			if len(decmap) > 0 {
 				exp, err = parseDecomposition(decmap, true)
 				if err != nil {
-					logger.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
+					log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
 				}
 				isCompat = true
 			}
@@ -261,7 +231,7 @@ func loadUnicodeData() {
 		}
 	}
 	if err := p.Err(); err != nil {
-		logger.Fatal(err)
+		log.Fatal(err)
 	}
 }
 
@@ -296,18 +266,18 @@ func compactCCC() {
 // 0958    # ...
 // See http://unicode.org/reports/tr44/ for full explanation
 func loadCompositionExclusions() {
-	f := openReader("CompositionExclusions.txt")
+	f := gen.OpenUCDFile("CompositionExclusions.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
 		c := &chars[p.Rune(0)]
 		if c.excludeInComp {
-			logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
+			log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
 		}
 		c.excludeInComp = true
 	}
 	if e := p.Err(); e != nil {
-		logger.Fatal(e)
+		log.Fatal(e)
 	}
 }
 
@@ -542,19 +512,19 @@ func computeNonStarterCounts() {
 	}
 }
 
-func printBytes(b []byte, name string) {
-	fmt.Printf("// %s: %d bytes\n", name, len(b))
-	fmt.Printf("var %s = [...]byte {", name)
+func printBytes(w io.Writer, b []byte, name string) {
+	fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
+	fmt.Fprintf(w, "var %s = [...]byte {", name)
 	for i, c := range b {
 		switch {
 		case i%64 == 0:
-			fmt.Printf("\n// Bytes %x - %x\n", i, i+63)
+			fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
 		case i%8 == 0:
-			fmt.Printf("\n")
+			fmt.Fprintf(w, "\n")
 		}
-		fmt.Printf("0x%.2X, ", c)
+		fmt.Fprintf(w, "0x%.2X, ", c)
 	}
-	fmt.Print("\n}\n\n")
+	fmt.Fprint(w, "\n}\n\n")
 }
 
 // See forminfo.go for format.
@@ -610,13 +580,13 @@ func (m *decompSet) insert(key int, s string) {
 	m[key][s] = true
 }
 
-func printCharInfoTables() int {
+func printCharInfoTables(w io.Writer) int {
 	mkstr := func(r rune, f *FormInfo) (int, string) {
 		d := f.expandedDecomp
 		s := string([]rune(d))
 		if max := 1 << 6; len(s) >= max {
 			const msg = "%U: too many bytes in decomposition: %d >= %d"
-			logger.Fatalf(msg, r, len(s), max)
+			log.Fatalf(msg, r, len(s), max)
 		}
 		head := uint8(len(s))
 		if f.quickCheck[MComposed] != QCYes {
@@ -631,11 +601,11 @@ func printCharInfoTables() int {
 		tccc := ccc(d[len(d)-1])
 		cc := ccc(r)
 		if cc != 0 && lccc == 0 && tccc == 0 {
-			logger.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
+			log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
 		}
 		if tccc < lccc && lccc != 0 {
 			const msg = "%U: lccc (%d) must be <= tcc (%d)"
-			logger.Fatalf(msg, r, lccc, tccc)
+			log.Fatalf(msg, r, lccc, tccc)
 		}
 		index := normalDecomp
 		nTrail := chars[r].nTrailingNonStarters
@@ -652,13 +622,13 @@ func printCharInfoTables() int {
 			if lccc > 0 {
 				s += string([]byte{lccc})
 				if index == firstCCC {
-					logger.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
+					log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
 				}
 				index = firstLeadingCCC
 			}
 			if cc != lccc {
 				if cc != 0 {
-					logger.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
+					log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
 				}
 				index = firstCCCZeroExcept
 			}
@@ -680,7 +650,7 @@ func printCharInfoTables() int {
 				continue
 			}
 			if f.combinesBackward {
-				logger.Fatalf("%U: combinesBackward and decompose", c.codePoint)
+				log.Fatalf("%U: combinesBackward and decompose", c.codePoint)
 			}
 			index, s := mkstr(c.codePoint, &f)
 			decompSet.insert(index, s)
@@ -691,7 +661,7 @@ func printCharInfoTables() int {
 	size := 0
 	positionMap := make(map[string]uint16)
 	decompositions.WriteString("\000")
-	fmt.Println("const (")
+	fmt.Fprintln(w, "const (")
 	for i, m := range decompSet {
 		sa := []string{}
 		for s := range m {
@@ -704,13 +674,13 @@ func printCharInfoTables() int {
 			positionMap[s] = uint16(p)
 		}
 		if cname[i] != "" {
-			fmt.Printf("%s = 0x%X\n", cname[i], decompositions.Len())
+			fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
 		}
 	}
-	fmt.Println("maxDecomp = 0x8000")
-	fmt.Println(")")
+	fmt.Fprintln(w, "maxDecomp = 0x8000")
+	fmt.Fprintln(w, ")")
 	b := decompositions.Bytes()
-	printBytes(b, "decomps")
+	printBytes(w, b, "decomps")
 	size += len(b)
 
 	varnames := []string{"nfc", "nfkc"}
@@ -726,7 +696,7 @@ func printCharInfoTables() int {
 				if c.ccc != ccc(d[0]) {
 					// We assume the lead ccc of a decomposition !=0 in this case.
 					if ccc(d[0]) == 0 {
-						logger.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
+						log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
 					}
 				}
 			} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
@@ -737,9 +707,9 @@ func printCharInfoTables() int {
 				trie.Insert(c.codePoint, uint64(0x8000|v))
 			}
 		}
-		sz, err := trie.Gen(os.Stdout, triegen.Compact(&normCompacter{name: varnames[i]}))
+		sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
 		if err != nil {
-			logger.Fatal(err)
+			log.Fatal(err)
 		}
 		size += sz
 	}
@@ -755,30 +725,9 @@ func contains(sa []string, s string) bool {
 	return false
 }
 
-// Extract the version number from the URL.
-func version() string {
-	// From http://www.unicode.org/standard/versions/#Version_Numbering:
-	// for the later Unicode versions, data files are located in
-	// versioned directories.
-	fields := strings.Split(*url, "/")
-	for _, f := range fields {
-		if match, _ := regexp.MatchString(`[0-9]\.[0-9]\.[0-9]`, f); match {
-			return f
-		}
-	}
-	logger.Fatal("unknown version")
-	return "Unknown"
-}
-
-const fileHeader = `// Generated by running
-//	maketables --tables=%s --url=%s
-// DO NOT EDIT
-
-package norm
-
-`
-
 func makeTables() {
+	w := &bytes.Buffer{}
+
 	size := 0
 	if *tablelist == "" {
 		return
@@ -787,7 +736,6 @@ func makeTables() {
 	if *tablelist == "all" {
 		list = []string{"recomp", "info"}
 	}
-	fmt.Printf(fileHeader, *tablelist, *url)
 
 	// Compute maximum decomposition size.
 	max := 0
@@ -797,30 +745,30 @@ func makeTables() {
 		}
 	}
 
-	fmt.Println("const (")
-	fmt.Println("\t// Version is the Unicode edition from which the tables are derived.")
-	fmt.Printf("\tVersion = %q\n", version())
-	fmt.Println()
-	fmt.Println("\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
-	fmt.Println("\t// may need to write atomically for any Form. Making a destination buffer at")
-	fmt.Println("\t// least this size ensures that Transform can always make progress and that")
-	fmt.Println("\t// the user does not need to grow the buffer on an ErrShortDst.")
-	fmt.Printf("\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
-	fmt.Println(")\n")
+	fmt.Fprintln(w, "const (")
+	fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
+	fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
+	fmt.Fprintln(w)
+	fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
+	fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
+	fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
+	fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
+	fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
+	fmt.Fprintln(w, ")\n")
 
 	// Print the CCC remap table.
 	size += len(cccMap)
-	fmt.Printf("var ccc = [%d]uint8{", len(cccMap))
+	fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
 	for i := 0; i < len(cccMap); i++ {
 		if i%8 == 0 {
-			fmt.Println()
+			fmt.Fprintln(w)
 		}
-		fmt.Printf("%3d, ", cccMap[uint8(i)])
+		fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
 	}
-	fmt.Println("\n}\n")
+	fmt.Fprintln(w, "\n}\n")
 
 	if contains(list, "info") {
-		size += printCharInfoTables()
+		size += printCharInfoTables(w)
 	}
 
 	if contains(list, "recomp") {
@@ -842,20 +790,21 @@ func makeTables() {
 		}
 		sz := nrentries * 8
 		size += sz
-		fmt.Printf("// recompMap: %d bytes (entries only)\n", sz)
-		fmt.Println("var recompMap = map[uint32]rune{")
+		fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
+		fmt.Fprintln(w, "var recompMap = map[uint32]rune{")
 		for i, c := range chars {
 			f := c.forms[FCanonical]
 			d := f.decomp
 			if !f.isOneWay && len(d) > 0 {
 				key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
-				fmt.Printf("0x%.8X: 0x%.4X,\n", key, i)
+				fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i)
 			}
 		}
-		fmt.Printf("}\n\n")
+		fmt.Fprintf(w, "}\n\n")
 	}
 
-	fmt.Printf("// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
+	fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
+	gen.WriteGoFile("tables.go", "norm", w.Bytes())
 }
 
 func printChars() {
@@ -901,7 +850,7 @@ func verifyComputed() {
 		nfc := c.forms[FCanonical]
 		nfkc := c.forms[FCompatibility]
 		if nfc.combinesBackward != nfkc.combinesBackward {
-			logger.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
+			log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
 		}
 	}
 }
@@ -913,7 +862,7 @@ func verifyComputed() {
 // 0374          ; NFD_QC; N # ...
 // See http://unicode.org/reports/tr44/ for full explanation
 func testDerived() {
-	f := openReader("DerivedNormalizationProps.txt")
+	f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
@@ -946,12 +895,12 @@ func testDerived() {
 			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
 		}
 		if got := c.forms[ftype].quickCheck[mode]; got != qr {
-			logger.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
+			log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
 		}
 		c.forms[ftype].verified[mode] = true
 	}
 	if err := p.Err(); err != nil {
-		logger.Fatal(err)
+		log.Fatal(err)
 	}
 	// Any unspecified value must be QCYes. Verify this.
 	for i, c := range chars {
@@ -959,20 +908,14 @@ func testDerived() {
 			for k, qr := range fd.quickCheck {
 				if !fd.verified[k] && qr != QCYes {
 					m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
-					logger.Printf(m, i, j, k, qr, c.name)
+					log.Printf(m, i, j, k, qr, c.name)
 				}
 			}
 		}
 	}
 }
 
-var testHeader = `// Generated by running
-//   maketables --test --url=%s
-// +build test
-
-package norm
-
-const (
+var testHeader = `const (
 	Yes = iota
 	No
 	Maybe
@@ -1010,8 +953,10 @@ func printTestdata() {
 		nTrail uint8
 		f      string
 	}
+
 	last := lastInfo{}
-	fmt.Printf(testHeader, *url)
+	w := &bytes.Buffer{}
+	fmt.Fprintf(w, testHeader)
 	for r, c := range chars {
 		f := c.forms[FCanonical]
 		qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
@@ -1025,9 +970,10 @@ func printTestdata() {
 		}
 		current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
 		if last != current {
-			fmt.Printf("\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
+			fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
 			last = current
 		}
 	}
-	fmt.Println("}")
+	fmt.Fprintln(w, "}")
+	gen.WriteGoFile("data_test.go", "norm", w.Bytes())
 }

Godeps/_workspace/src/golang.org/x/text/unicode/norm/normalize.go

@@ -2,6 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:generate go run maketables.go triegen.go
+//go:generate go run maketables.go triegen.go -test
+
 // Package norm contains types and functions for normalizing Unicode strings.
 package norm
 

Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go

@@ -1,6 +1,4 @@
-// Generated by running
-//	maketables --tables=all --url=http://www.unicode.org/Public/7.0.0/ucd/
-// DO NOT EDIT
+// This file was generated by go generate; DO NOT EDIT
 
 package norm
 

Godeps/_workspace/src/golang.org/x/text/unicode/norm/ucd_test.go

@@ -2,52 +2,37 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build ignore
-
-package main
+package norm
 
 import (
 	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
-	"log"
-	"net/http"
-	"os"
-	"path"
 	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
+	"testing"
 	"time"
-	"unicode"
 	"unicode/utf8"
 
-	"golang.org/x/text/unicode/norm"
+	"golang.org/x/text/internal/gen"
 )
 
-func main() {
-	flag.Parse()
-	loadTestData()
-	CharacterByCharacterTests()
-	StandardTests()
-	PerformanceTest()
-	if errorCount == 0 {
-		fmt.Println("PASS")
+var long = flag.Bool("long", false,
+	"run time-consuming tests, such as tests that fetch data online")
+
+var once sync.Once
+
+func skipShort(t *testing.T) {
+	if !gen.IsLocal() && !*long {
+		t.Skip("skipping test to prevent downloading; to run use -long or use -local to specify a local source")
 	}
+	once.Do(func() { loadTestData(t) })
 }
 
-const file = "NormalizationTest.txt"
-
-var url = flag.String("url",
-	"http://www.unicode.org/Public/"+unicode.Version+"/ucd/"+file,
-	"URL of Unicode database directory")
-var localFiles = flag.Bool("local",
-	false,
-	"data files have been copied to the current directory; for debugging only")
-
-var logger = log.New(os.Stderr, "", log.Lshortfile)
-
 // This regression test runs the test set in NormalizationTest.txt
 // (taken from http://www.unicode.org/Public/<unicode.Version>/ucd/).
 //
@@ -124,22 +109,8 @@ var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.
 var counter int
 
 // Load the data form NormalizationTest.txt
-func loadTestData() {
-	if *localFiles {
-		pwd, _ := os.Getwd()
-		*url = "file://" + path.Join(pwd, file)
-	}
-	t := &http.Transport{}
-	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
-	c := &http.Client{Transport: t}
-	resp, err := c.Get(*url)
-	if err != nil {
-		logger.Fatal(err)
-	}
-	if resp.StatusCode != 200 {
-		logger.Fatal("bad GET status for "+file, resp.Status)
-	}
-	f := resp.Body
+func loadTestData(t *testing.T) {
+	f := gen.OpenUCDFile("NormalizationTest.txt")
 	defer f.Close()
 	scanner := bufio.NewScanner(f)
 	for scanner.Scan() {
@@ -150,11 +121,11 @@ func loadTestData() {
 		m := partRe.FindStringSubmatch(line)
 		if m != nil {
 			if len(m) < 3 {
-				logger.Fatal("Failed to parse Part: ", line)
+				t.Fatal("Failed to parse Part: ", line)
 			}
 			i, err := strconv.Atoi(m[1])
 			if err != nil {
-				logger.Fatal(err)
+				t.Fatal(err)
 			}
 			name := m[2]
 			part = append(part, Part{name: name[:len(name)-1], number: i})
@@ -162,7 +133,7 @@ func loadTestData() {
 		}
 		m = testRe.FindStringSubmatch(line)
 		if m == nil || len(m) < 7 {
-			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
+			t.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
 		}
 		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
 		counter++
@@ -170,7 +141,7 @@ func loadTestData() {
 			for _, split := range strings.Split(m[j], " ") {
 				r, err := strconv.ParseUint(split, 16, 64)
 				if err != nil {
-					logger.Fatal(err)
+					t.Fatal(err)
 				}
 				if test.r == 0 {
 					// save for CharacterByCharacterTests
@@ -185,50 +156,38 @@ func loadTestData() {
 		part.tests = append(part.tests, test)
 	}
 	if scanner.Err() != nil {
-		logger.Fatal(scanner.Err())
+		t.Fatal(scanner.Err())
 	}
 }
 
-var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}
-
-var errorCount int
-
-func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
+func cmpResult(t *testing.T, tc *Test, name string, f Form, gold, test, result string) {
 	if gold != result {
-		errorCount++
-		if errorCount > 20 {
-			return
-		}
-		logger.Printf("%s:%s: %s(%+q)=%+q; want %+q: %s",
-			t.Name(), name, fstr[f], test, result, gold, t.name)
+		t.Errorf("%s:%s: %s(%+q)=%+q; want %+q: %s",
+			tc.Name(), name, fstr[f], test, result, gold, tc.name)
 	}
 }
 
-func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
+func cmpIsNormal(t *testing.T, tc *Test, name string, f Form, test string, result, want bool) {
 	if result != want {
-		errorCount++
-		if errorCount > 20 {
-			return
-		}
-		logger.Printf("%s:%s: %s(%+q)=%v; want %v", t.Name(), name, fstr[f], test, result, want)
+		t.Errorf("%s:%s: %s(%+q)=%v; want %v", tc.Name(), name, fstr[f], test, result, want)
 	}
 }
 
-func doTest(t *Test, f norm.Form, gold, test string) {
+func doTest(t *testing.T, tc *Test, f Form, gold, test string) {
 	testb := []byte(test)
 	result := f.Bytes(testb)
-	cmpResult(t, "Bytes", f, gold, test, string(result))
+	cmpResult(t, tc, "Bytes", f, gold, test, string(result))
 
 	sresult := f.String(test)
-	cmpResult(t, "String", f, gold, test, sresult)
+	cmpResult(t, tc, "String", f, gold, test, sresult)
 
 	acc := []byte{}
-	i := norm.Iter{}
+	i := Iter{}
 	i.InitString(f, test)
 	for !i.Done() {
 		acc = append(acc, i.Next()...)
 	}
-	cmpResult(t, "Iter.Next", f, gold, test, string(acc))
+	cmpResult(t, tc, "Iter.Next", f, gold, test, string(acc))
 
 	buf := make([]byte, 128)
 	acc = nil
@@ -237,32 +196,33 @@ func doTest(t *Test, f norm.Form, gold, test string) {
 		acc = append(acc, buf[:nDst]...)
 		p += nSrc
 	}
-	cmpResult(t, "Transform", f, gold, test, string(acc))
+	cmpResult(t, tc, "Transform", f, gold, test, string(acc))
 
 	for i := range test {
 		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
-		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
+		cmpResult(t, tc, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
 	}
-	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
-	cmpIsNormal(t, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
+	cmpIsNormal(t, tc, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
+	cmpIsNormal(t, tc, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
 }
 
-func doConformanceTests(t *Test, partn int) {
+func doConformanceTests(t *testing.T, tc *Test, partn int) {
 	for i := 0; i <= 2; i++ {
-		doTest(t, norm.NFC, t.cols[1], t.cols[i])
-		doTest(t, norm.NFD, t.cols[2], t.cols[i])
-		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
-		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
+		doTest(t, tc, NFC, tc.cols[1], tc.cols[i])
+		doTest(t, tc, NFD, tc.cols[2], tc.cols[i])
+		doTest(t, tc, NFKC, tc.cols[3], tc.cols[i])
+		doTest(t, tc, NFKD, tc.cols[4], tc.cols[i])
 	}
 	for i := 3; i <= 4; i++ {
-		doTest(t, norm.NFC, t.cols[3], t.cols[i])
-		doTest(t, norm.NFD, t.cols[4], t.cols[i])
-		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
-		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
+		doTest(t, tc, NFC, tc.cols[3], tc.cols[i])
+		doTest(t, tc, NFD, tc.cols[4], tc.cols[i])
+		doTest(t, tc, NFKC, tc.cols[3], tc.cols[i])
+		doTest(t, tc, NFKD, tc.cols[4], tc.cols[i])
 	}
 }
 
-func CharacterByCharacterTests() {
+func TestCharacterByCharacter(t *testing.T) {
+	skipShort(t)
 	tests := part[1].tests
 	var last rune = 0
 	for i := 0; i <= len(tests); i++ { // last one is special case
@@ -274,37 +234,39 @@ func CharacterByCharacterTests() {
 		}
 		for last++; last < r; last++ {
 			// Check all characters that were not explicitly listed in the test.
-			t := &Test{partnr: 1, number: -1}
+			tc := &Test{partnr: 1, number: -1}
 			char := string(last)
-			doTest(t, norm.NFC, char, char)
-			doTest(t, norm.NFD, char, char)
-			doTest(t, norm.NFKC, char, char)
-			doTest(t, norm.NFKD, char, char)
+			doTest(t, tc, NFC, char, char)
+			doTest(t, tc, NFD, char, char)
+			doTest(t, tc, NFKC, char, char)
+			doTest(t, tc, NFKD, char, char)
 		}
 		if i < len(tests) {
-			doConformanceTests(&tests[i], 1)
+			doConformanceTests(t, &tests[i], 1)
 		}
 	}
 }
 
-func StandardTests() {
+func TestStandardTests(t *testing.T) {
+	skipShort(t)
 	for _, j := range []int{0, 2, 3} {
 		for _, test := range part[j].tests {
-			doConformanceTests(&test, j)
+			doConformanceTests(t, &test, j)
 		}
 	}
 }
 
-// PerformanceTest verifies that normalization is O(n). If any of the
+// TestPerformance verifies that normalization is O(n). If any of the
 // code does not properly check for maxCombiningChars, normalization
 // may exhibit O(n**2) behavior.
-func PerformanceTest() {
+func TestPerformance(t *testing.T) {
+	skipShort(t)
 	runtime.GOMAXPROCS(2)
 	success := make(chan bool, 1)
 	go func() {
 		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
 		buf = append(buf, "\u035B"...)
-		norm.NFC.Append(nil, buf...)
+		NFC.Append(nil, buf...)
 		success <- true
 	}()
 	timeout := time.After(1 * time.Second)
@@ -312,7 +274,6 @@ func PerformanceTest() {
 	case <-success:
 		// test completed before the timeout
 	case <-timeout:
-		errorCount++
-		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
+		t.Errorf(`unexpectedly long time to complete PerformanceTest`)
 	}
 }

build.sh

@@ -118,7 +118,7 @@ case "${1:-default}" in
 			-e "STTRACE=$STTRACE" \
 			syncthing/build:latest \
 			sh -c './build.sh clean \
-				&& go vet ./cmd/... ./internal/... \
+				&& go tool vet -composites=false cmd/*/*.go internal/*/*.go \
 				&& ( golint ./cmd/... ; golint ./internal/... ) | egrep -v "comment on exported|should have comment" \
 				&& ./build.sh all \
 				&& STTRACE=all ./build.sh test-cov'
@@ -134,7 +134,7 @@ case "${1:-default}" in
 				&& go run build.go -race \
 				&& export GOPATH=$(pwd)/Godeps/_workspace:$GOPATH \
 				&& cd test \
-				&& go test -tags integration -v -timeout 60m -short \
+				&& go test -tags integration -v -timeout 90m -short \
 				&& git clean -fxd .'
 		;;
 

changelog.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"log"
+	"os/exec"
+	"regexp"
+)
+
+var (
+	subjectIssues = regexp.MustCompile(`^([^(]+)\s+\((?:fixes|ref) ([^)]+)\)$`)
+	issueNumbers  = regexp.MustCompile(`(#\d+)`)
+)
+
+func main() {
+	flag.Parse()
+
+	// Display changelog since the version given on the command line, or
+	// figure out the last release if there were no arguments.
+	var prevRel string
+	if flag.NArg() > 0 {
+		prevRel = flag.Arg(0)
+	} else {
+		bs, err := runError("git", "describe", "--abbrev=0", "HEAD^")
+		if err != nil {
+			log.Fatal(err)
+		}
+		prevRel = string(bs)
+	}
+
+	// Get the git log with subject and author nickname
+	bs, err := runError("git", "log", "--reverse", "--pretty=format:%s|%aN", prevRel+"..")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Split into lines
+	for _, line := range bytes.Split(bs, []byte{'\n'}) {
+		// Split into subject and author
+		fields := bytes.Split(line, []byte{'|'})
+		subj := fields[0]
+		author := fields[1]
+
+		// Check if subject contains a "(fixes ...)" or "(ref ...)""
+		if m := subjectIssues.FindSubmatch(subj); len(m) > 0 {
+			// Find all issue numbers
+			issues := issueNumbers.FindAll(m[2], -1)
+
+			// Format a changelog entry
+			fmt.Printf("* %s (%s, @%s)\n", m[1], bytes.Join(issues, []byte(", ")), author)
+		}
+	}
+}
+
+func runError(cmd string, args ...string) ([]byte, error) {
+	ecmd := exec.Command(cmd, args...)
+	bs, err := ecmd.CombinedOutput()
+	if err != nil {
+		return nil, err
+	}
+	return bytes.TrimSpace(bs), nil
+}

changelog.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-since="$1"
-if [[ -z $since ]] ; then
-	since="$(git describe --abbrev=0 HEAD^).."
-fi
-
-git log --reverse --pretty=format:'* %s, @%aN)' "$since" | egrep 'fixes #\d|ref #\d' | sed 's/)[,. ]*,/,/' | sed 's/fixes #/#/g' | sed 's/ref #/#/g'
-
-git diff "$since" -- AUTHORS
-

cmd/stevents/main.go

@@ -27,7 +27,7 @@ func main() {
 	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
 
-	target := flag.String("target", "localhost:8080", "Target Syncthing instance")
+	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 

cmd/syncthing/gui.go

@@ -49,6 +49,11 @@ var (
 	eventSub     *events.BufferedSubscription
 )
 
+var (
+	lastEventRequest    time.Time
+	lastEventRequestMut sync.Mutex
+)
+
 func init() {
 	l.AddHandler(logger.LevelWarn, showGuiError)
 	sub := events.Default.Subscribe(events.AllEvents)
@@ -58,7 +63,7 @@ func init() {
 func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) error {
 	var err error
 
-	cert, err := loadCert(confDir, "https-")
+	cert, err := tls.LoadX509KeyPair(locations[locHttpsCertFile], locations[locHttpsKeyFile])
 	if err != nil {
 		l.Infoln("Loading HTTPS certificate:", err)
 		l.Infoln("Creating new HTTPS certificate")
@@ -71,8 +76,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 			name = tlsDefaultCommonName
 		}
 
-		newCertificate(confDir, "https-", name)
-		cert, err = loadCert(confDir, "https-")
+		cert, err = newCertificate(locations[locHttpsCertFile], locations[locHttpsKeyFile], name)
 	}
 	if err != nil {
 		return err
@@ -179,6 +183,9 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 		ReadTimeout: 10 * time.Second,
 	}
 
+	csrv := &folderSummarySvc{model: m}
+	go csrv.Serve()
+
 	go func() {
 		err := srv.Serve(listener)
 		if err != nil {
@@ -293,8 +300,14 @@ func restGetCompletion(m *model.Model, w http.ResponseWriter, r *http.Request) {
 }
 
 func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
-	var qs = r.URL.Query()
-	var folder = qs.Get("folder")
+	qs := r.URL.Query()
+	folder := qs.Get("folder")
+	res := folderSummary(m, folder)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(res)
+}
+
+func folderSummary(m *model.Model, folder string) map[string]interface{} {
 	var res = make(map[string]interface{})
 
 	res["invalid"] = cfg.Folders()[folder].Invalid
@@ -322,8 +335,7 @@ func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	json.NewEncoder(w).Encode(res)
+	return res
 }
 
 func restPostOverride(m *model.Model, w http.ResponseWriter, r *http.Request) {
@@ -598,6 +610,10 @@ func restGetEvents(w http.ResponseWriter, r *http.Request) {
 	since, _ := strconv.Atoi(sinceStr)
 	limit, _ := strconv.Atoi(limitStr)
 
+	lastEventRequestMut.Lock()
+	lastEventRequest = time.Now()
+	lastEventRequestMut.Unlock()
+
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 
 	// Flush before blocking, to indicate that we've received the request
@@ -618,7 +634,7 @@ func restGetUpgrade(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, upgrade.ErrUpgradeUnsupported.Error(), 500)
 		return
 	}
-	rel, err := upgrade.LatestRelease(strings.Contains(Version, "-beta"))
+	rel, err := upgrade.LatestGithubRelease(Version)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return
@@ -660,7 +676,7 @@ func restGetLang(w http.ResponseWriter, r *http.Request) {
 }
 
 func restPostUpgrade(w http.ResponseWriter, r *http.Request) {
-	rel, err := upgrade.LatestRelease(strings.Contains(Version, "-beta"))
+	rel, err := upgrade.LatestGithubRelease(Version)
 	if err != nil {
 		l.Warnln("getting latest release:", err)
 		http.Error(w, err.Error(), 500)
@@ -685,8 +701,8 @@ func restPostScan(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")
 	if folder != "" {
-		sub := qs.Get("sub")
-		err := m.ScanFolderSub(folder, sub)
+		subs := qs["sub"]
+		err := m.ScanFolderSubs(folder, subs)
 		if err != nil {
 			http.Error(w, err.Error(), 500)
 		}
@@ -828,6 +844,8 @@ func mimeTypeForFile(file string) string {
 		return "application/x-font-ttf"
 	case ".woff":
 		return "application/x-font-woff"
+	case ".svg":
+		return "image/svg+xml"
 	default:
 		return mime.TypeByExtension(ext)
 	}

cmd/syncthing/gui_csrf.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -92,7 +91,7 @@ func newCsrfToken() string {
 }
 
 func saveCsrfTokens() {
-	name := filepath.Join(confDir, "csrftokens.txt")
+	name := locations[locCsrfTokens]
 	tmp := fmt.Sprintf("%s.tmp.%d", name, time.Now().UnixNano())
 
 	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
@@ -117,8 +116,7 @@ func saveCsrfTokens() {
 }
 
 func loadCsrfTokens() {
-	name := filepath.Join(confDir, "csrftokens.txt")
-	f, err := os.Open(name)
+	f, err := os.Open(locations[locCsrfTokens])
 	if err != nil {
 		return
 	}

cmd/syncthing/locations.go

@@ -0,0 +1,109 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/osutil"
+)
+
+type locationEnum string
+
+// Use strings as keys to make printout and serialization of the locations map
+// more meaningful.
+const (
+	locConfigFile    locationEnum = "config"
+	locCertFile                   = "certFile"
+	locKeyFile                    = "keyFile"
+	locHttpsCertFile              = "httpsCertFile"
+	locHttpsKeyFile               = "httpsKeyFile"
+	locDatabase                   = "database"
+	locLogFile                    = "logFile"
+	locCsrfTokens                 = "csrfTokens"
+	locPanicLog                   = "panicLog"
+	locDefFolder                  = "defFolder"
+)
+
+// Platform dependent directories
+var baseDirs = map[string]string{
+	"config": defaultConfigDir(), // Overridden by -home flag
+	"home":   homeDir(),          // User's home directory, *not* -home flag
+}
+
+// Use the variables from baseDirs here
+var locations = map[locationEnum]string{
+	locConfigFile:    "${config}/config.xml",
+	locCertFile:      "${config}/cert.pem",
+	locKeyFile:       "${config}/key.pem",
+	locHttpsCertFile: "${config}/https-cert.pem",
+	locHttpsKeyFile:  "${config}/https-key.pem",
+	locDatabase:      "${config}/index-v0.11.0.db",
+	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
+	locCsrfTokens:    "${config}/csrftokens.txt",
+	locPanicLog:      "${config}/panic-20060102-150405.log", // passed through time.Format()
+	locDefFolder:     "${home}/Sync",
+}
+
+// expandLocations replaces the variables in the location map with actual
+// directory locations.
+func expandLocations() error {
+	for key, dir := range locations {
+		for varName, value := range baseDirs {
+			dir = strings.Replace(dir, "${"+varName+"}", value, -1)
+		}
+		var err error
+		dir, err = osutil.ExpandTilde(dir)
+		if err != nil {
+			return err
+		}
+		locations[key] = dir
+	}
+	return nil
+}
+
+// defaultConfigDir returns the default configuration directory, as figured
+// out by various the environment variables present on each platform, or dies
+// trying.
+func defaultConfigDir() string {
+	switch runtime.GOOS {
+	case "windows":
+		if p := os.Getenv("LocalAppData"); p != "" {
+			return filepath.Join(p, "Syncthing")
+		}
+		return filepath.Join(os.Getenv("AppData"), "Syncthing")
+
+	case "darwin":
+		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		if err != nil {
+			l.Fatalln(err)
+		}
+		return dir
+
+	default:
+		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
+			return filepath.Join(xdgCfg, "syncthing")
+		}
+		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		if err != nil {
+			l.Fatalln(err)
+		}
+		return dir
+	}
+}
+
+// homeDir returns the user's home directory, or dies trying.
+func homeDir() string {
+	home, err := osutil.ExpandTilde("~")
+	if err != nil {
+		l.Fatalln(err)
+	}
+	return home
+}

cmd/syncthing/main.go

@@ -19,7 +19,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
-	"runtime/debug"
 	"runtime/pprof"
 	"strconv"
 	"strings"
@@ -51,7 +50,6 @@ var (
 	BuildHost   = "unknown"
 	BuildUser   = "unknown"
 	IsRelease   bool
-	IsBeta      bool
 	LongVersion string
 )
 
@@ -63,7 +61,10 @@ const (
 	exitUpgrading          = 4
 )
 
-const bepProtocolName = "bep/1.0"
+const (
+	bepProtocolName   = "bep/1.0"
+	pingEventInterval = time.Minute
+)
 
 var l = logger.DefaultLogger
 
@@ -80,9 +81,6 @@ func init() {
 	exp := regexp.MustCompile(`^v\d+\.\d+\.\d+(-beta[\d\.]+)?$`)
 	IsRelease = exp.MatchString(Version)
 
-	// Check for a beta build
-	IsBeta = strings.Contains(Version, "-beta")
-
 	stamp, _ := strconv.Atoi(BuildStamp)
 	BuildDate = time.Unix(int64(stamp), 0)
 
@@ -170,7 +168,11 @@ are mostly useful for developers. Use with care.
  STNOUPGRADE     Disable automatic upgrades.
 
  GOMAXPROCS      Set the maximum number of CPU cores to use. Defaults to all
-                 available CPU cores.`
+                 available CPU cores.
+
+ GOGC            Percentage of heap growth at which to trigger GC. Default is
+                 100. Lower numbers keep peak memory usage down, at the price
+                 of CPU usage (ie. performance).`
 )
 
 // Command line and environment options
@@ -197,17 +199,11 @@ var (
 )
 
 func main() {
-	defConfDir, err := getDefaultConfDir()
-	if err != nil {
-		l.Fatalln("home:", err)
-	}
-
 	if runtime.GOOS == "windows" {
 		// On Windows, we use a log file by default. Setting the -logfile flag
-		// to the empty string disables this behavior.
+		// to "-" disables this behavior.
 
-		logFile = filepath.Join(defConfDir, "syncthing.log")
-		flag.StringVar(&logFile, "logfile", logFile, "Log file name (blank for stdout)")
+		flag.StringVar(&logFile, "logfile", "", "Log file name (use \"-\" for stdout)")
 
 		// We also add an option to hide the console window
 		flag.BoolVar(&noConsole, "no-console", false, "Hide console window")
@@ -227,23 +223,30 @@ func main() {
 	flag.BoolVar(&showVersion, "version", false, "Show version")
 	flag.StringVar(&upgradeTo, "upgrade-to", upgradeTo, "Force upgrade directly from specified URL")
 
-	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, defConfDir))
+	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, baseDirs["config"]))
 	flag.Parse()
 
 	if noConsole {
 		osutil.HideConsole()
 	}
 
-	if confDir == "" {
+	if confDir != "" {
 		// Not set as default above because the string can be really long.
-		confDir = defConfDir
+		baseDirs["config"] = confDir
 	}
 
-	if confDir != defConfDir && filepath.Dir(logFile) == defConfDir {
-		// The user changed the config dir with -home, but not the log file
-		// location. In this case we assume they meant for the logfile to
-		// still live in it's default location *relative to the config dir*.
-		logFile = filepath.Join(confDir, "syncthing.log")
+	if runtime.GOOS == "windows" {
+		if logFile == "" {
+			// Use the default log file location
+			logFile = locations[locLogFile]
+		} else if logFile == "-" {
+			// Don't use a logFile
+			logFile = ""
+		}
+	}
+
+	if err := expandLocations(); err != nil {
+		l.Fatalln(err)
 	}
 
 	if showVersion {
@@ -270,13 +273,13 @@ func main() {
 			}
 		}
 
-		cert, err := loadCert(dir, "")
+		certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 		if err == nil {
 			l.Warnln("Key exists; will not overwrite.")
 			l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
 		} else {
-			newCertificate(dir, "", tlsDefaultCommonName)
-			cert, err = loadCert(dir, "")
+			cert, err = newCertificate(certFile, keyFile, tlsDefaultCommonName)
 			myID = protocol.NewDeviceID(cert.Certificate[0])
 			if err != nil {
 				l.Fatalln("load cert:", err)
@@ -302,17 +305,12 @@ func main() {
 		return
 	}
 
-	confDir, err := osutil.ExpandTilde(confDir)
-	if err != nil {
-		l.Fatalln("home:", err)
-	}
-
-	if info, err := os.Stat(confDir); err == nil && !info.IsDir() {
-		l.Fatalln("Config directory", confDir, "is not a directory")
+	if info, err := os.Stat(baseDirs["config"]); err == nil && !info.IsDir() {
+		l.Fatalln("Config directory", baseDirs["config"], "is not a directory")
 	}
 
 	// Ensure that our home directory exists.
-	ensureDir(confDir, 0700)
+	ensureDir(baseDirs["config"], 0700)
 
 	if upgradeTo != "" {
 		err := upgrade.ToURL(upgradeTo)
@@ -324,7 +322,7 @@ func main() {
 	}
 
 	if doUpgrade || doUpgradeCheck {
-		rel, err := upgrade.LatestRelease(IsBeta)
+		rel, err := upgrade.LatestGithubRelease(Version)
 		if err != nil {
 			l.Fatalln("Upgrade:", err) // exits 1
 		}
@@ -338,7 +336,7 @@ func main() {
 
 		if doUpgrade {
 			// Use leveldb database locks to protect against concurrent upgrades
-			_, err = leveldb.OpenFile(filepath.Join(confDir, "index"), &opt.Options{OpenFilesCacheCapacity: 100})
+			_, err = leveldb.OpenFile(locations[locDatabase], &opt.Options{OpenFilesCacheCapacity: 100})
 			if err != nil {
 				l.Fatalln("Cannot upgrade, database seems to be locked. Is another copy of Syncthing already running?")
 			}
@@ -368,21 +366,16 @@ func main() {
 func syncthingMain() {
 	var err error
 
-	if len(os.Getenv("GOGC")) == 0 {
-		debug.SetGCPercent(25)
-	}
-
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
 		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
-	events.Default.Log(events.Starting, map[string]string{"home": confDir})
+	events.Default.Log(events.Starting, map[string]string{"home": baseDirs["config"]})
 
 	// Ensure that that we have a certificate and key.
-	cert, err = loadCert(confDir, "")
+	cert, err = tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
 	if err != nil {
-		newCertificate(confDir, "", tlsDefaultCommonName)
-		cert, err = loadCert(confDir, "")
+		cert, err = newCertificate(locations[locCertFile], locations[locKeyFile], tlsDefaultCommonName)
 		if err != nil {
 			l.Fatalln("load cert:", err)
 		}
@@ -400,7 +393,7 @@ func syncthingMain() {
 
 	// Prepare to be able to save configuration
 
-	cfgFile := filepath.Join(confDir, "config.xml")
+	cfgFile := locations[locConfigFile]
 
 	var myName string
 
@@ -495,7 +488,7 @@ func syncthingMain() {
 		l.Infoln("Local networks:", strings.Join(networks, ", "))
 	}
 
-	dbFile := filepath.Join(confDir, "index")
+	dbFile := locations[locDatabase]
 	dbOpts := &opt.Options{OpenFilesCacheCapacity: 100}
 	ldb, err := leveldb.OpenFile(dbFile, dbOpts)
 	if err != nil && errors.IsCorrupted(err) {
@@ -514,9 +507,7 @@ func syncthingMain() {
 		}
 	}
 
-	m := model.NewModel(cfg, myName, "syncthing", Version, ldb)
-
-	sanityCheckFolders(cfg, m)
+	m := model.NewModel(cfg, myID, myName, "syncthing", Version, ldb)
 
 	// GUI
 
@@ -526,14 +517,12 @@ func syncthingMain() {
 	// start needing a bunch of files which are nowhere to be found. This
 	// needs to be changed when we correctly do persistent indexes.
 	for _, folderCfg := range cfg.Folders() {
-		if folderCfg.Invalid != "" {
-			continue
-		}
+		m.AddFolder(folderCfg)
 		for _, device := range folderCfg.DeviceIDs() {
 			if device == myID {
 				continue
 			}
-			m.Index(device, folderCfg.ID, nil)
+			m.Index(device, folderCfg.ID, nil, 0, nil)
 		}
 	}
 
@@ -557,10 +546,6 @@ func syncthingMain() {
 	go listenConnect(myID, m, tlsCfg)
 
 	for _, folder := range cfg.Folders() {
-		if folder.Invalid != "" {
-			continue
-		}
-
 		// Routine to pull blocks from other devices to synchronize the local
 		// folder. Does not run when we are in read only (publish only) mode.
 		if folder.ReadOnly {
@@ -626,7 +611,7 @@ func syncthingMain() {
 	}
 
 	events.Default.Log(events.StartupComplete, nil)
-	go generateEvents()
+	go generatePingEvents()
 
 	code := <-stop
 
@@ -677,66 +662,12 @@ func setupGUI(cfg *config.Wrapper, m *model.Model) {
 	}
 }
 
-func sanityCheckFolders(cfg *config.Wrapper, m *model.Model) {
-nextFolder:
-	for id, folder := range cfg.Folders() {
-		if folder.Invalid != "" {
-			continue
-		}
-		m.AddFolder(folder)
-
-		fi, err := os.Stat(folder.Path)
-		if m.CurrentLocalVersion(id) > 0 {
-			// Safety check. If the cached index contains files but the
-			// folder doesn't exist, we have a problem. We would assume
-			// that all files have been deleted which might not be the case,
-			// so mark it as invalid instead.
-			if err != nil || !fi.IsDir() {
-				l.Warnf("Stopping folder %q - path does not exist, but has files in index", folder.ID)
-				cfg.InvalidateFolder(id, "folder path missing")
-				continue nextFolder
-			} else if !folder.HasMarker() {
-				l.Warnf("Stopping folder %q - path exists, but folder marker missing, check for mount issues", folder.ID)
-				cfg.InvalidateFolder(id, "folder marker missing")
-				continue nextFolder
-			}
-		} else if os.IsNotExist(err) {
-			// If we don't have any files in the index, and the directory
-			// doesn't exist, try creating it.
-			err = os.MkdirAll(folder.Path, 0700)
-			if err != nil {
-				l.Warnf("Stopping folder %q - %v", folder.ID, err)
-				cfg.InvalidateFolder(id, err.Error())
-				continue nextFolder
-			}
-			err = folder.CreateMarker()
-		} else if !folder.HasMarker() {
-			// If we don't have any files in the index, and the path does exist
-			// but the marker is not there, create it.
-			err = folder.CreateMarker()
-		}
-
-		if err != nil {
-			// If there was another error or we could not create the
-			// path, the folder is invalid.
-			l.Warnf("Stopping folder %q - %v", folder.ID, err)
-			cfg.InvalidateFolder(id, err.Error())
-			continue nextFolder
-		}
-	}
-}
-
 func defaultConfig(myName string) config.Configuration {
-	defaultFolder, err := osutil.ExpandTilde("~/Sync")
-	if err != nil {
-		l.Fatalln("home:", err)
-	}
-
 	newCfg := config.New(myID)
 	newCfg.Folders = []config.FolderConfiguration{
 		{
 			ID:              "default",
-			Path:            defaultFolder,
+			Path:            locations[locDefFolder],
 			RescanIntervalS: 60,
 			Devices:         []config.FolderDeviceConfiguration{{DeviceID: myID}},
 		},
@@ -749,7 +680,7 @@ func defaultConfig(myName string) config.Configuration {
 		},
 	}
 
-	port, err := getFreePort("127.0.0.1", 8080)
+	port, err := getFreePort("127.0.0.1", 8384)
 	if err != nil {
 		l.Fatalln("get free port (GUI):", err)
 	}
@@ -763,9 +694,9 @@ func defaultConfig(myName string) config.Configuration {
 	return newCfg
 }
 
-func generateEvents() {
+func generatePingEvents() {
 	for {
-		time.Sleep(300 * time.Second)
+		time.Sleep(pingEventInterval)
 		events.Default.Log(events.Ping, nil)
 	}
 }
@@ -873,12 +804,7 @@ func renewUPnP(port int) {
 }
 
 func resetFolders() {
-	confDir, err := osutil.ExpandTilde(confDir)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	cfgFile := filepath.Join(confDir, "config.xml")
+	cfgFile := locations[locConfigFile]
 	cfg, err := config.Load(cfgFile, myID)
 	if err != nil {
 		log.Fatal(err)
@@ -888,14 +814,13 @@ func resetFolders() {
 	for _, folder := range cfg.Folders() {
 		if _, err := os.Stat(folder.Path); err == nil {
 			base := filepath.Base(folder.Path)
-			dir := filepath.Dir(filepath.Join(folder.Path, ".."))
+			dir := filepath.Dir(folder.Path)
 			l.Infof("Reset: Moving %s -> %s", folder.Path, filepath.Join(dir, base+suffix))
 			os.Rename(folder.Path, filepath.Join(dir, base+suffix))
 		}
 	}
 
-	idx := filepath.Join(confDir, "index")
-	os.RemoveAll(idx)
+	os.RemoveAll(locations[locDatabase])
 }
 
 func restart() {
@@ -941,25 +866,6 @@ func ensureDir(dir string, mode int) {
 	}
 }
 
-func getDefaultConfDir() (string, error) {
-	switch runtime.GOOS {
-	case "windows":
-		if p := os.Getenv("LocalAppData"); p != "" {
-			return filepath.Join(p, "Syncthing"), nil
-		}
-		return filepath.Join(os.Getenv("AppData"), "Syncthing"), nil
-
-	case "darwin":
-		return osutil.ExpandTilde("~/Library/Application Support/Syncthing")
-
-	default:
-		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
-			return filepath.Join(xdgCfg, "syncthing"), nil
-		}
-		return osutil.ExpandTilde("~/.config/syncthing")
-	}
-}
-
 // getFreePort returns a free TCP port fort listening on. The ports given are
 // tried in succession and the first to succeed is returned. If none succeed,
 // a random high port is returned.
@@ -1057,7 +963,7 @@ func autoUpgrade() {
 		case <-timer.C:
 		}
 
-		rel, err := upgrade.LatestRelease(IsBeta)
+		rel, err := upgrade.LatestGithubRelease(Version)
 		if err == upgrade.ErrUpgradeUnsupported {
 			events.Default.Unsubscribe(sub)
 			return

cmd/syncthing/main_test.go

@@ -19,7 +19,7 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
-func TestSanityCheck(t *testing.T) {
+func TestFolderErrors(t *testing.T) {
 	fcfg := config.FolderConfiguration{
 		ID:   "folder",
 		Path: "testdata/testfolder",
@@ -28,7 +28,8 @@ func TestSanityCheck(t *testing.T) {
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	for _, file := range []string{".stfolder", "testfolder", "testfolder/.stfolder"} {
+	for _, file := range []string{".stfolder", "testfolder/.stfolder", "testfolder"} {
+		os.Remove("testdata/" + file)
 		_, err := os.Stat("testdata/" + file)
 		if err == nil {
 			t.Error("Found unexpected file")
@@ -39,10 +40,10 @@ func TestSanityCheck(t *testing.T) {
 
 	// Case 1 - new folder, directory and marker created
 
-	m := model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m := model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "" {
+	if err := m.CheckFolderHealth("folder"); err != nil {
 		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
 	}
 
@@ -66,10 +67,10 @@ func TestSanityCheck(t *testing.T) {
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	m = model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "" {
+	if err := m.CheckFolderHealth("folder"); err != nil {
 		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
 	}
 
@@ -80,31 +81,62 @@ func TestSanityCheck(t *testing.T) {
 
 	os.Remove("testdata/.stfolder")
 
-	// Case 3 - marker missing
+	// Case 3 - Folder marker missing
 
 	set := db.NewFileSet("folder", ldb)
 	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
 		{Name: "dummyfile"},
 	})
 
-	m = model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "folder marker missing" {
-		t.Error("Incorrect error")
+	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "Folder marker missing" {
+		t.Error("Incorrect error: Folder marker missing !=", m.CheckFolderHealth("folder"))
 	}
 
-	// Case 4 - path missing
+	// Case 3.1 - recover after folder marker missing
+
+	if err = fcfg.CreateMarker(); err != nil {
+		t.Error(err)
+	}
+
+	if err := m.CheckFolderHealth("folder"); err != nil {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	// Case 4 - Folder path missing
+
+	os.Remove("testdata/testfolder/.stfolder")
+	os.Remove("testdata/testfolder/")
 
 	fcfg.Path = "testdata/testfolder"
-	cfg = config.Wrap("/tmp/test", config.Configuration{
+	cfg = config.Wrap("testdata/subfolder", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	m = model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "folder path missing" {
-		t.Error("Incorrect error")
+	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "Folder path missing" {
+		t.Error("Incorrect error: Folder path missing !=", m.CheckFolderHealth("folder"))
+	}
+
+	// Case 4.1 - recover after folder path missing
+
+	os.Mkdir("testdata/testfolder", 0700)
+
+	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "Folder marker missing" {
+		t.Error("Incorrect error: Folder marker missing !=", m.CheckFolderHealth("folder"))
+	}
+
+	// Case 4.2 - recover after missing marker
+
+	if err = fcfg.CreateMarker(); err != nil {
+		t.Error(err)
+	}
+
+	if err := m.CheckFolderHealth("folder"); err != nil {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
 	}
 }

cmd/syncthing/monitor.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
@@ -164,7 +163,7 @@ func copyStderr(stderr io.ReadCloser, dst io.Writer) {
 			dst.Write([]byte(line))
 
 			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
-				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
+				panicFd, err = os.Create(time.Now().Format(locations[locPanicLog]))
 				if err != nil {
 					l.Warnln("Create panic log:", err)
 					continue

cmd/syncthing/summarysvc.go

@@ -0,0 +1,190 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"sync"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/model"
+	"github.com/thejerf/suture"
+)
+
+// The folderSummarySvc adds summary information events (FolderSummary and
+// FolderCompletion) into the event stream at certain intervals.
+type folderSummarySvc struct {
+	model     *model.Model
+	srv       suture.Service
+	stop      chan struct{}
+	immediate chan string
+
+	// For keeping track of folders to recalculate for
+	foldersMut sync.Mutex
+	folders    map[string]struct{}
+}
+
+func (c *folderSummarySvc) Serve() {
+	srv := suture.NewSimple("folderSummarySvc")
+	srv.Add(serviceFunc(c.listenForUpdates))
+	srv.Add(serviceFunc(c.calculateSummaries))
+
+	c.immediate = make(chan string)
+	c.stop = make(chan struct{})
+	c.folders = make(map[string]struct{})
+	c.srv = srv
+
+	srv.Serve()
+}
+
+func (c *folderSummarySvc) Stop() {
+	// c.srv.Stop() is mostly a no-op here, but we need to call it anyway so
+	// c.srv doesn't try to restart the serviceFuncs when they exit after we
+	// close the stop channel.
+	c.srv.Stop()
+	close(c.stop)
+}
+
+// listenForUpdates subscribes to the event bus and makes note of folders that
+// need their data recalculated.
+func (c *folderSummarySvc) listenForUpdates() {
+	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged)
+	defer events.Default.Unsubscribe(sub)
+
+	for {
+		// This loop needs to be fast so we don't miss too many events.
+
+		select {
+		case ev := <-sub.C():
+			// Whenever the local or remote index is updated for a given
+			// folder we make a note of it.
+
+			data := ev.Data.(map[string]interface{})
+			folder := data["folder"].(string)
+
+			if ev.Type == events.StateChanged && data["to"].(string) == "idle" && data["from"].(string) == "syncing" {
+				// The folder changed to idle from syncing. We should do an
+				// immediate refresh to update the GUI. The send to
+				// c.immediate must be nonblocking so that we can continue
+				// handling events.
+
+				select {
+				case c.immediate <- folder:
+					c.foldersMut.Lock()
+					delete(c.folders, folder)
+					c.foldersMut.Unlock()
+
+				default:
+				}
+			} else {
+				// This folder needs to be refreshed whenever we do the next
+				// refresh.
+
+				c.foldersMut.Lock()
+				c.folders[folder] = struct{}{}
+				c.foldersMut.Unlock()
+			}
+
+		case <-c.stop:
+			return
+		}
+	}
+}
+
+// calculateSummaries periodically recalculates folder summaries and
+// completion percentage, and sends the results on the event bus.
+func (c *folderSummarySvc) calculateSummaries() {
+	const pumpInterval = 2 * time.Second
+	pump := time.NewTimer(pumpInterval)
+
+	for {
+		select {
+		case <-pump.C:
+			t0 := time.Now()
+			for _, folder := range c.foldersToHandle() {
+				c.sendSummary(folder)
+			}
+
+			// We don't want to spend all our time calculating summaries. Lets
+			// set an arbitrary limit at not spending more than about 30% of
+			// our time here...
+			wait := 2*time.Since(t0) + pumpInterval
+			pump.Reset(wait)
+
+		case folder := <-c.immediate:
+			c.sendSummary(folder)
+
+		case <-c.stop:
+			return
+		}
+	}
+}
+
+// foldersToHandle returns the list of folders needing a summary update, and
+// clears the list.
+func (c *folderSummarySvc) foldersToHandle() []string {
+	// We only recalculate sumamries if someone is listening to events
+	// (a request to /rest/events has been made within the last
+	// pingEventInterval).
+
+	lastEventRequestMut.Lock()
+	// XXX: Reaching out to a global var here is very ugly :( Should
+	// we make the gui stuff a proper object with methods on it that
+	// we can query about this kind of thing?
+	last := lastEventRequest
+	lastEventRequestMut.Unlock()
+	if time.Since(last) < pingEventInterval {
+		return nil
+	}
+
+	c.foldersMut.Lock()
+	res := make([]string, 0, len(c.folders))
+	for folder := range c.folders {
+		res = append(res, folder)
+		delete(c.folders, folder)
+	}
+	c.foldersMut.Unlock()
+	return res
+}
+
+// sendSummary send the summary events for a single folder
+func (c *folderSummarySvc) sendSummary(folder string) {
+	// The folder summary contains how many bytes, files etc
+	// are in the folder and how in sync we are.
+	data := folderSummary(c.model, folder)
+	events.Default.Log(events.FolderSummary, map[string]interface{}{
+		"folder":  folder,
+		"summary": data,
+	})
+
+	for _, devCfg := range cfg.Folders()[folder].Devices {
+		if devCfg.DeviceID.Equals(myID) {
+			// We already know about ourselves.
+			continue
+		}
+		if !c.model.ConnectedTo(devCfg.DeviceID) {
+			// We're not interested in disconnected devices.
+			continue
+		}
+
+		// Get completion percentage of this folder for the
+		// remote device.
+		comp := c.model.Completion(devCfg.DeviceID, folder)
+		events.Default.Log(events.FolderCompletion, map[string]interface{}{
+			"folder":     folder,
+			"device":     devCfg.DeviceID.String(),
+			"completion": comp,
+		})
+	}
+}
+
+// serviceFunc wraps a function to create a suture.Service without stop
+// functionality.
+type serviceFunc func()
+
+func (f serviceFunc) Serve() { f() }
+func (f serviceFunc) Stop()  {}

cmd/syncthing/tls.go

@@ -19,7 +19,6 @@ import (
 	mr "math/rand"
 	"net"
 	"os"
-	"path/filepath"
 	"time"
 )
 
@@ -28,13 +27,7 @@ const (
 	tlsDefaultCommonName = "syncthing"
 )
 
-func loadCert(dir string, prefix string) (tls.Certificate, error) {
-	cf := filepath.Join(dir, prefix+"cert.pem")
-	kf := filepath.Join(dir, prefix+"key.pem")
-	return tls.LoadX509KeyPair(cf, kf)
-}
-
-func newCertificate(dir, prefix, name string) {
+func newCertificate(certFile, keyFile, name string) (tls.Certificate, error) {
 	l.Infof("Generating RSA key and certificate for %s...", name)
 
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
@@ -63,7 +56,7 @@ func newCertificate(dir, prefix, name string) {
 		l.Fatalln("create cert:", err)
 	}
 
-	certOut, err := os.Create(filepath.Join(dir, prefix+"cert.pem"))
+	certOut, err := os.Create(certFile)
 	if err != nil {
 		l.Fatalln("save cert:", err)
 	}
@@ -76,7 +69,7 @@ func newCertificate(dir, prefix, name string) {
 		l.Fatalln("save cert:", err)
 	}
 
-	keyOut, err := os.OpenFile(filepath.Join(dir, prefix+"key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		l.Fatalln("save key:", err)
 	}
@@ -88,6 +81,8 @@ func newCertificate(dir, prefix, name string) {
 	if err != nil {
 		l.Fatalln("save key:", err)
 	}
+
+	return tls.LoadX509KeyPair(certFile, keyFile)
 }
 
 type DowngradingListener struct {

etc/linux-systemd/system/syncthing@.service

@@ -8,8 +8,7 @@ User=%i
 Environment=STNORESTART=yes
 ExecStart=/usr/bin/syncthing -no-browser -logflags=0
 Restart=on-failure
-RestartPreventExitStatus=1
-SuccessExitStatus=2
+SuccessExitStatus=2 3 4
 RestartForceExitStatus=3 4
 
 [Install]

etc/linux-systemd/user/syncthing.service

@@ -7,8 +7,7 @@ After=network.target
 Environment=STNORESTART=yes
 ExecStart=/usr/bin/syncthing -no-browser -logflags=0
 Restart=on-failure
-RestartPreventExitStatus=1
-SuccessExitStatus=2
+SuccessExitStatus=2 3 4
 RestartForceExitStatus=3 4
 
 [Install]

gui/assets/lang/lang-en.json

@@ -1,4 +1,5 @@
 {
+   "A external command handles the versioning. It has to remove the file from the synced folder.": "A external command handles the versioning. It has to remove the file from the synced folder.",
    "API Key": "API Key",
    "About": "About",
    "Add": "Add",
@@ -16,6 +17,7 @@
    "CPU Utilization": "CPU Utilization",
    "Changelog": "Changelog",
    "Close": "Close",
+   "Command": "Command",
    "Comment, when used at the start of a line": "Comment, when used at the start of a line",
    "Compression": "Compression",
    "Compression is recommended in most setups.": "Compression is recommended in most setups.",
@@ -42,6 +44,7 @@
    "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.",
    "Enter ignore patterns, one per line.": "Enter ignore patterns, one per line.",
    "Error": "Error",
+   "External File Versioning": "External File Versioning",
    "File Versioning": "File Versioning",
    "File permission bits are ignored when looking for changes. Use on FAT filesystems.": "File permission bits are ignored when looking for changes. Use on FAT filesystems.",
    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.": "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.",
@@ -144,6 +147,7 @@
    "The device ID to enter here can be found in the \"Edit \u003e Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "The device ID to enter here can be found in the \"Edit \u003e Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).",
    "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.",
    "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.",
+   "The first command line parameter is the folder path and the second parameter is the relative path in the folder.": "The first command line parameter is the folder path and the second parameter is the relative path in the folder.",
    "The folder ID cannot be blank.": "The folder ID cannot be blank.",
    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.",
    "The folder ID must be unique.": "The folder ID must be unique.",
@@ -153,6 +157,7 @@
    "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "The maximum time to keep a version (in days, set to 0 to keep versions forever).",
    "The number of old versions to keep, per file.": "The number of old versions to keep, per file.",
    "The number of versions must be a number and cannot be blank.": "The number of versions must be a number and cannot be blank.",
+   "The path cannot be blank.": "The path cannot be blank.",
    "The rescan interval must be a non-negative number of seconds.": "The rescan interval must be a non-negative number of seconds.",
    "The rescan interval must be at least 5 seconds.": "The rescan interval must be at least 5 seconds.",
    "Unknown": "Unknown",

gui/index.html

@@ -973,8 +973,8 @@
           <li>Jens Diemer</li>
           <li>Jochen Voss</li>
           <li>Johan Vromans</li>
-          <li>Kamada Ken'ichi</li>
           <li>Karol Różycki</li>
+          <li>Ken'ichi Kamada</li>
           <li>Lode Hoste</li>
           <li>Marc Laporte</li>
           <li>Marc Pujol</li>

gui/scripts/syncthing/core/controllers/eventController.js

@@ -1,3 +1,5 @@
+var debugEvents = false;
+
 angular.module('syncthing.core')
     .controller('EventController', function ($scope, $http) {
         'use strict';
@@ -20,7 +22,9 @@ angular.module('syncthing.core')
 
             if (lastID > 0) {
                 data.forEach(function (event) {
-                    console.log("event", event.id, event.type, event.data);
+                    if (debugEvents) {
+                        console.log("event", event.id, event.type, event.data);
+                    }
                     $scope.$emit(event.type, event);
                 });
             }

gui/scripts/syncthing/core/controllers/syncthingController.js

@@ -1,4 +1,7 @@
 angular.module('syncthing.core')
+    .config(function($locationProvider) {
+        $locationProvider.html5Mode(true).hashPrefix('!');
+    })
     .controller('SyncthingController', function ($scope, $http, $location, LocaleService) {
         'use strict';
 
@@ -137,19 +140,11 @@ angular.module('syncthing.core')
 
         $scope.$on('LocalIndexUpdated', function (event, arg) {
             var data = arg.data;
-            refreshFolder(data.folder);
             refreshFolderStats();
-
-            // Update completion status for all devices that we share this folder with.
-            $scope.folders[data.folder].devices.forEach(function (deviceCfg) {
-                refreshCompletion(deviceCfg.deviceID, data.folder);
-            });
         });
 
         $scope.$on('RemoteIndexUpdated', function (event, arg) {
-            var data = arg.data;
-            refreshFolder(data.folder);
-            refreshCompletion(data.device, data.folder);
+            // Nothing
         });
 
         $scope.$on('DeviceDisconnected', function (event, arg) {
@@ -212,7 +207,6 @@ angular.module('syncthing.core')
             var stats = arg.data;
             var progress = {};
             for (var folder in stats) {
-                refreshFolder(folder);
                 progress[folder] = {};
                 for (var file in stats[folder]) {
                     var s = stats[folder][file];
@@ -238,7 +232,6 @@ angular.module('syncthing.core')
             }
             for (var folder in $scope.progress) {
                 if (!(folder in progress)) {
-                    refreshFolder(folder);
                     if ($scope.neededFolder == folder) {
                         refreshNeed(folder);
                     }
@@ -255,6 +248,30 @@ angular.module('syncthing.core')
             console.log("DownloadProgress", $scope.progress);
         });
 
+        $scope.$on('FolderSummary', function (event, arg) {
+            var data = arg.data;
+            $scope.model[data.folder] = data.summary;
+        });
+
+        $scope.$on('FolderCompletion', function (event, arg) {
+            var data = arg.data;
+            if (!$scope.completion[data.device]) {
+                $scope.completion[data.device] = {};
+            }
+            $scope.completion[data.device][data.folder] = data.completion;
+
+            var tot = 0,
+                cnt = 0;
+            for (var cmp in $scope.completion[data.device]) {
+                if (cmp === "_total") {
+                    continue;
+                }
+                tot += $scope.completion[data.device][cmp];
+                cnt += 1;
+            }
+            $scope.completion[data.device]._total = tot / cnt;
+        });
+
         $scope.emitHTTPError = function (data, status, headers, config) {
             $scope.$emit('HTTPError', {data: data, status: status, headers: headers, config: config});
         };
@@ -322,31 +339,25 @@ angular.module('syncthing.core')
                 return;
             }
 
-            var key = "refreshCompletion" + device + folder;
-            if (!debouncedFuncs[key]) {
-                debouncedFuncs[key] = debounce(function () {
-                    $http.get(urlbase + '/completion?device=' + device + '&folder=' + encodeURIComponent(folder)).success(function (data) {
-                        if (!$scope.completion[device]) {
-                            $scope.completion[device] = {};
-                        }
-                        $scope.completion[device][folder] = data.completion;
+            $http.get(urlbase + '/completion?device=' + device + '&folder=' + encodeURIComponent(folder)).success(function (data) {
+                if (!$scope.completion[device]) {
+                    $scope.completion[device] = {};
+                }
+                $scope.completion[device][folder] = data.completion;
 
-                        var tot = 0,
-                            cnt = 0;
-                        for (var cmp in $scope.completion[device]) {
-                            if (cmp === "_total") {
-                                continue;
-                            }
-                            tot += $scope.completion[device][cmp];
-                            cnt += 1;
-                        }
-                        $scope.completion[device]._total = tot / cnt;
+                var tot = 0,
+                    cnt = 0;
+                for (var cmp in $scope.completion[device]) {
+                    if (cmp === "_total") {
+                        continue;
+                    }
+                    tot += $scope.completion[device][cmp];
+                    cnt += 1;
+                }
+                $scope.completion[device]._total = tot / cnt;
 
-                        console.log("refreshCompletion", device, folder, $scope.completion[device]);
-                    }).error($scope.emitHTTPError);
-                }, 1000, true);
-            }
-            debouncedFuncs[key]();
+                console.log("refreshCompletion", device, folder, $scope.completion[device]);
+            }).error($scope.emitHTTPError);
         }
 
         function refreshConnectionStats() {
@@ -409,7 +420,7 @@ angular.module('syncthing.core')
                 }
                 console.log("refreshDeviceStats", data);
             }).error($scope.emitHTTPError);
-        }, 500);
+        }, 2500);
 
         var refreshFolderStats = debounce(function () {
             $http.get(urlbase + "/stats/folder").success(function (data) {
@@ -421,7 +432,7 @@ angular.module('syncthing.core')
                 }
                 console.log("refreshfolderStats", data);
             }).error($scope.emitHTTPError);
-        }, 500);
+        }, 2500);
 
         $scope.refresh = function () {
             refreshSystem();

internal/config/config.go

@@ -187,7 +187,7 @@ type OptionsConfiguration struct {
 
 type GUIConfiguration struct {
 	Enabled  bool   `xml:"enabled,attr" json:"enabled" default:"true"`
-	Address  string `xml:"address" json:"address" default:"127.0.0.1:8080"`
+	Address  string `xml:"address" json:"address" default:"127.0.0.1:8384"`
 	User     string `xml:"user,omitempty" json:"user"`
 	Password string `xml:"password,omitempty" json:"password"`
 	UseTLS   bool   `xml:"tls,attr" json:"useTLS"`
@@ -370,6 +370,11 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		}
 	}
 
+	// Very short reconnection intervals are annoying
+	if cfg.Options.ReconnectIntervalS < 5 {
+		cfg.Options.ReconnectIntervalS = 5
+	}
+
 	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
 	cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
 

internal/config/wrapper.go

@@ -220,8 +220,8 @@ func (w *Wrapper) SetGUI(gui GUIConfiguration) {
 	w.replaces <- w.cfg
 }
 
-// InvalidateFolder sets the invalid marker on the given folder.
-func (w *Wrapper) InvalidateFolder(id string, err string) {
+// Sets the folder error state. Emits ConfigSaved to cause a GUI refresh.
+func (w *Wrapper) SetFolderError(id string, err error) {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 
@@ -229,8 +229,15 @@ func (w *Wrapper) InvalidateFolder(id string, err string) {
 
 	for i := range w.cfg.Folders {
 		if w.cfg.Folders[i].ID == id {
-			w.cfg.Folders[i].Invalid = err
-			w.replaces <- w.cfg
+			errstr := ""
+			if err != nil {
+				errstr = err.Error()
+			}
+			if errstr != w.cfg.Folders[i].Invalid {
+				w.cfg.Folders[i].Invalid = errstr
+				events.Default.Log(events.ConfigSaved, w.cfg)
+				w.replaces <- w.cfg
+			}
 			return
 		}
 	}

internal/db/leveldb.go

@@ -17,7 +17,6 @@ import (
 	"sync"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/lamport"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -49,7 +48,7 @@ const (
 )
 
 type fileVersion struct {
-	version int64
+	version protocol.Vector
 	device  []byte
 }
 
@@ -242,8 +241,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 			}
 			var ef FileInfoTruncated
 			ef.UnmarshalXDR(dbi.Value())
-			if fs[fsi].Version > ef.Version ||
-				(fs[fsi].Version == ef.Version && fs[fsi].Flags != ef.Flags) {
+			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Flags != ef.Flags {
 				if debugDB {
 					l.Debugln("generic replace; differs - insert")
 				}
@@ -315,7 +313,7 @@ func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) i
 	})
 }
 
-func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 {
+func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, myID uint64) int64 {
 	return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
 		var tf FileInfoTruncated
 		err := tf.UnmarshalXDR(dbi.Value())
@@ -329,7 +327,7 @@ func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.F
 			ts := clock(tf.LocalVersion)
 			f := protocol.FileInfo{
 				Name:         tf.Name,
-				Version:      lamport.Default.Tick(tf.Version),
+				Version:      tf.Version.Update(myID),
 				LocalVersion: ts,
 				Flags:        tf.Flags | protocol.FlagDeleted,
 				Modified:     tf.Modified,
@@ -394,7 +392,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 		}
 		// Flags might change without the version being bumped when we set the
 		// invalid flag on an existing file.
-		if ef.Version != f.Version || ef.Flags != f.Flags {
+		if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
 			if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
@@ -454,7 +452,7 @@ func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) in
 // ldbUpdateGlobal adds this device+version to the version list for the given
 // file. If the device is already present in the list, the version is updated.
 // If the file does not have an entry in the global list, it is created.
-func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device, file []byte, version int64) bool {
+func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device, file []byte, version protocol.Vector) bool {
 	if debugDB {
 		l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file, version)
 	}
@@ -465,10 +463,8 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device, file []byte, v
 	}
 
 	var fl versionList
-	nv := fileVersion{
-		device:  device,
-		version: version,
-	}
+
+	// Remove the device from the current version list
 	if svl != nil {
 		err = fl.UnmarshalXDR(svl)
 		if err != nil {
@@ -477,7 +473,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device, file []byte, v
 
 		for i := range fl.versions {
 			if bytes.Compare(fl.versions[i].device, device) == 0 {
-				if fl.versions[i].version == version {
+				if fl.versions[i].version.Equal(version) {
 					// No need to do anything
 					return false
 				}
@@ -487,8 +483,15 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device, file []byte, v
 		}
 	}
 
+	nv := fileVersion{
+		device:  device,
+		version: version,
+	}
 	for i := range fl.versions {
-		if fl.versions[i].version <= version {
+		// We compare  against ConcurrentLesser as well here because we need
+		// to enforce a consistent ordering of versions even in the case of
+		// conflicts.
+		if comp := fl.versions[i].version.Compare(version); comp == protocol.Equal || comp == protocol.Lesser || comp == protocol.ConcurrentLesser {
 			t := append(fl.versions, fileVersion{})
 			copy(t[i+1:], t[i:])
 			t[i] = nv
@@ -776,7 +779,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
 
 	var devices []protocol.DeviceID
 	for _, v := range vl.versions {
-		if v.version != vl.versions[0].version {
+		if !v.version.Equal(vl.versions[0].version) {
 			break
 		}
 		n := protocol.DeviceIDFromBytes(v.device)
@@ -808,7 +811,7 @@ func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterat
 	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
 	defer dbi.Release()
 
-outer:
+nextFile:
 	for dbi.Next() {
 		var vl versionList
 		err := vl.UnmarshalXDR(dbi.Value())
@@ -822,12 +825,15 @@ outer:
 
 		have := false // If we have the file, any version
 		need := false // If we have a lower version of the file
-		var haveVersion int64
+		var haveVersion protocol.Vector
 		for _, v := range vl.versions {
 			if bytes.Compare(v.device, device) == 0 {
 				have = true
 				haveVersion = v.version
-				need = v.version < vl.versions[0].version
+				// XXX: This marks Concurrent (i.e. conflicting) changes as
+				// needs. Maybe we should do that, but it needs special
+				// handling in the puller.
+				need = !v.version.GreaterEqual(vl.versions[0].version)
 				break
 			}
 		}
@@ -835,11 +841,12 @@ outer:
 		if need || !have {
 			name := globalKeyName(dbi.Key())
 			needVersion := vl.versions[0].version
-		inner:
+
+		nextVersion:
 			for i := range vl.versions {
-				if vl.versions[i].version != needVersion {
+				if !vl.versions[i].version.Equal(needVersion) {
 					// We haven't found a valid copy of the file with the needed version.
-					continue outer
+					continue nextFile
 				}
 				fk := deviceKey(folder, vl.versions[i].device, name)
 				if debugDB {
@@ -866,12 +873,12 @@ outer:
 
 				if gf.IsInvalid() {
 					// The file is marked invalid for whatever reason, don't use it.
-					continue inner
+					continue nextVersion
 				}
 
 				if gf.IsDeleted() && !have {
 					// We don't need deleted files that we don't have
-					continue outer
+					continue nextFile
 				}
 
 				if debugDB {
@@ -883,7 +890,7 @@ outer:
 				}
 
 				// This file is handled, no need to look further in the version list
-				continue outer
+				continue nextFile
 			}
 		}
 	}

internal/db/leveldb_xdr.go

@@ -18,9 +18,9 @@ fileVersion Structure:
  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                       version (64 bits)                       +
-|                                                               |
+/                                                               /
+\                       Vector Structure                        \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                       Length of device                        |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -31,7 +31,7 @@ fileVersion Structure:
 
 
 struct fileVersion {
-	hyper version;
+	Vector version;
 	opaque device<>;
 }
 
@@ -39,7 +39,7 @@ struct fileVersion {
 
 func (o fileVersion) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o fileVersion) MarshalXDR() ([]byte, error) {
@@ -57,29 +57,32 @@ func (o fileVersion) MustMarshalXDR() []byte {
 func (o fileVersion) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o fileVersion) encodeXDR(xw *xdr.Writer) (int, error) {
-	xw.WriteUint64(uint64(o.version))
+func (o fileVersion) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	_, err := o.version.EncodeXDRInto(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
 	xw.WriteBytes(o.device)
 	return xw.Tot(), xw.Error()
 }
 
 func (o *fileVersion) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *fileVersion) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *fileVersion) decodeXDR(xr *xdr.Reader) error {
-	o.version = int64(xr.ReadUint64())
+func (o *fileVersion) DecodeXDRFrom(xr *xdr.Reader) error {
+	(&o.version).DecodeXDRFrom(xr)
 	o.device = xr.ReadBytes()
 	return xr.Error()
 }
@@ -107,7 +110,7 @@ struct versionList {
 
 func (o versionList) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o versionList) MarshalXDR() ([]byte, error) {
@@ -125,14 +128,14 @@ func (o versionList) MustMarshalXDR() []byte {
 func (o versionList) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o versionList) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o versionList) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(uint32(len(o.versions)))
 	for i := range o.versions {
-		_, err := o.versions[i].encodeXDR(xw)
+		_, err := o.versions[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -142,20 +145,20 @@ func (o versionList) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *versionList) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *versionList) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *versionList) decodeXDR(xr *xdr.Reader) error {
+func (o *versionList) DecodeXDRFrom(xr *xdr.Reader) error {
 	_versionsSize := int(xr.ReadUint32())
 	o.versions = make([]fileVersion, _versionsSize)
 	for i := range o.versions {
-		(&o.versions[i]).decodeXDR(xr)
+		(&o.versions[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }

internal/db/set.go

@@ -16,7 +16,6 @@ import (
 	"sync"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/lamport"
 	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -61,7 +60,6 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
 		if f.LocalVersion > s.localVersion[deviceID] {
 			s.localVersion[deviceID] = f.LocalVersion
 		}
-		lamport.Default.Tick(f.Version)
 		return true
 	})
 	if debug {
@@ -90,14 +88,14 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
 	}
 }
 
-func (s *FileSet) ReplaceWithDelete(device protocol.DeviceID, fs []protocol.FileInfo) {
+func (s *FileSet) ReplaceWithDelete(device protocol.DeviceID, fs []protocol.FileInfo, myID uint64) {
 	if debug {
 		l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.folder, device, len(fs))
 	}
 	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
-	if lv := ldbReplaceWithDelete(s.db, []byte(s.folder), device[:], fs); lv > s.localVersion[device] {
+	if lv := ldbReplaceWithDelete(s.db, []byte(s.folder), device[:], fs, myID); lv > s.localVersion[device] {
 		s.localVersion[device] = lv
 	}
 	if device == protocol.LocalDeviceID {
@@ -118,7 +116,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 		updates := make([]protocol.FileInfo, 0, len(fs))
 		for _, newFile := range fs {
 			existingFile, ok := ldbGet(s.db, []byte(s.folder), device[:], []byte(newFile.Name))
-			if !ok || existingFile.Version <= newFile.Version {
+			if !ok || !existingFile.Version.Equal(newFile.Version) {
 				discards = append(discards, existingFile)
 				updates = append(updates, newFile)
 			}

internal/db/set_test.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/internal/db"
-	"github.com/syncthing/syncthing/internal/lamport"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
@@ -27,6 +26,8 @@ func init() {
 	remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
 }
 
+const myID = 1
+
 func genBlocks(n int) []protocol.BlockInfo {
 	b := make([]protocol.BlockInfo, n)
 	for i := range b {
@@ -95,7 +96,6 @@ func (l fileList) String() string {
 }
 
 func TestGlobalSet(t *testing.T) {
-	lamport.Default = lamport.Clock{}
 
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
 	if err != nil {
@@ -105,34 +105,34 @@ func TestGlobalSet(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local0 := fileList{
-		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "z", Version: 1000, Blocks: genBlocks(8)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
 	}
 	local1 := fileList{
-		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
 	}
 	localTot := fileList{
 		local0[0],
 		local0[1],
 		local0[2],
 		local0[3],
-		protocol.FileInfo{Name: "z", Version: 1001, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
 	}
 
 	remote0 := fileList{
-		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(5)},
 	}
 	remote1 := fileList{
-		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(6)},
-		protocol.FileInfo{Name: "e", Version: 1000, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(6)},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(7)},
 	}
 	remoteTot := fileList{
 		remote0[0],
@@ -160,8 +160,8 @@ func TestGlobalSet(t *testing.T) {
 		local0[3],
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local0)
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local1)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local0, myID)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local1, myID)
 	m.Replace(remoteDevice0, remote0)
 	m.Update(remoteDevice0, remote1)
 
@@ -256,8 +256,6 @@ func TestGlobalSet(t *testing.T) {
 }
 
 func TestNeedWithInvalid(t *testing.T) {
-	lamport.Default = lamport.Clock{}
-
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
 	if err != nil {
 		t.Fatal(err)
@@ -266,26 +264,26 @@ func TestNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
 	}
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "e", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
 	}
 
 	expectedNeed := fileList{
-		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
 	}
 
-	s.ReplaceWithDelete(protocol.LocalDeviceID, localHave)
+	s.ReplaceWithDelete(protocol.LocalDeviceID, localHave, myID)
 	s.Replace(remoteDevice0, remote0Have)
 	s.Replace(remoteDevice1, remote1Have)
 
@@ -298,8 +296,6 @@ func TestNeedWithInvalid(t *testing.T) {
 }
 
 func TestUpdateToInvalid(t *testing.T) {
-	lamport.Default = lamport.Clock{}
-
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
 	if err != nil {
 		t.Fatal(err)
@@ -308,13 +304,13 @@ func TestUpdateToInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
 	}
 
-	s.ReplaceWithDelete(protocol.LocalDeviceID, localHave)
+	s.ReplaceWithDelete(protocol.LocalDeviceID, localHave, myID)
 
 	have := fileList(haveList(s, protocol.LocalDeviceID))
 	sort.Sort(have)
@@ -323,7 +319,7 @@ func TestUpdateToInvalid(t *testing.T) {
 		t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
 	}
 
-	localHave[1] = protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagInvalid}
+	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagInvalid}
 	s.Update(protocol.LocalDeviceID, localHave[1:2])
 
 	have = fileList(haveList(s, protocol.LocalDeviceID))
@@ -335,8 +331,6 @@ func TestUpdateToInvalid(t *testing.T) {
 }
 
 func TestInvalidAvailability(t *testing.T) {
-	lamport.Default = lamport.Clock{}
-
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
 	if err != nil {
 		t.Fatal(err)
@@ -345,16 +339,16 @@ func TestInvalidAvailability(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
 	}
 
 	s.Replace(remoteDevice0, remote0Have)
@@ -383,17 +377,16 @@ func TestLocalDeleted(t *testing.T) {
 		t.Fatal(err)
 	}
 	m := db.NewFileSet("test", ldb)
-	lamport.Default = lamport.Clock{}
 
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: 1000},
-		{Name: "b", Version: 1000},
-		{Name: "c", Version: 1000},
-		{Name: "d", Version: 1000},
-		{Name: "z", Version: 1000, Flags: protocol.FlagDirectory},
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Flags: protocol.FlagDirectory},
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local1)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local1, myID)
 
 	m.ReplaceWithDelete(protocol.LocalDeviceID, []protocol.FileInfo{
 		local1[0],
@@ -401,25 +394,25 @@ func TestLocalDeleted(t *testing.T) {
 		local1[2],
 		local1[3],
 		local1[4],
-	})
+	}, myID)
 	m.ReplaceWithDelete(protocol.LocalDeviceID, []protocol.FileInfo{
 		local1[0],
 		local1[2],
 		// [3] removed
 		local1[4],
-	})
+	}, myID)
 	m.ReplaceWithDelete(protocol.LocalDeviceID, []protocol.FileInfo{
 		local1[0],
 		local1[2],
 		// [4] removed
-	})
+	}, myID)
 
 	expectedGlobal1 := []protocol.FileInfo{
 		local1[0],
-		{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
 		local1[2],
-		{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
-		{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
 	g := globalList(m)
@@ -433,14 +426,14 @@ func TestLocalDeleted(t *testing.T) {
 	m.ReplaceWithDelete(protocol.LocalDeviceID, []protocol.FileInfo{
 		local1[0],
 		// [2] removed
-	})
+	}, myID)
 
 	expectedGlobal2 := []protocol.FileInfo{
 		local1[0],
-		{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
-		{Name: "c", Version: 1004, Flags: protocol.FlagDeleted},
-		{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
-		{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
 	g = globalList(m)
@@ -460,20 +453,20 @@ func Benchmark10kReplace(b *testing.B) {
 
 	var local []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		m := db.NewFileSet("test", ldb)
-		m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+		m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 	}
 }
 
 func Benchmark10kUpdateChg(b *testing.B) {
 	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -486,16 +479,16 @@ func Benchmark10kUpdateChg(b *testing.B) {
 
 	var local []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		b.StopTimer()
 		for j := range local {
-			local[j].Version++
+			local[j].Version = local[j].Version.Update(myID)
 		}
 		b.StartTimer()
 		m.Update(protocol.LocalDeviceID, local)
@@ -505,7 +498,7 @@ func Benchmark10kUpdateChg(b *testing.B) {
 func Benchmark10kUpdateSme(b *testing.B) {
 	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -517,10 +510,10 @@ func Benchmark10kUpdateSme(b *testing.B) {
 
 	var local []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -531,7 +524,7 @@ func Benchmark10kUpdateSme(b *testing.B) {
 func Benchmark10kNeed2k(b *testing.B) {
 	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -544,13 +537,13 @@ func Benchmark10kNeed2k(b *testing.B) {
 
 	var local []protocol.FileInfo
 	for i := 0; i < 8000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 	for i := 8000; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -564,7 +557,7 @@ func Benchmark10kNeed2k(b *testing.B) {
 func Benchmark10kHaveFullList(b *testing.B) {
 	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -577,13 +570,13 @@ func Benchmark10kHaveFullList(b *testing.B) {
 
 	var local []protocol.FileInfo
 	for i := 0; i < 2000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 	for i := 2000; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -597,7 +590,7 @@ func Benchmark10kHaveFullList(b *testing.B) {
 func Benchmark10kGlobal(b *testing.B) {
 	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 
 	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -610,13 +603,13 @@ func Benchmark10kGlobal(b *testing.B) {
 
 	var local []protocol.FileInfo
 	for i := 0; i < 2000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
 	}
 	for i := 2000; i < 10000; i++ {
-		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -636,20 +629,20 @@ func TestGlobalReset(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: 1000},
-		{Name: "b", Version: 1000},
-		{Name: "c", Version: 1000},
-		{Name: "d", Version: 1000},
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: 1000},
-		{Name: "b", Version: 1001},
-		{Name: "c", Version: 1002},
-		{Name: "e", Version: 1000},
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 	g := globalList(m)
 	sort.Sort(fileList(g))
 
@@ -677,26 +670,26 @@ func TestNeed(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: 1000},
-		{Name: "b", Version: 1000},
-		{Name: "c", Version: 1000},
-		{Name: "d", Version: 1000},
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: 1000},
-		{Name: "b", Version: 1001},
-		{Name: "c", Version: 1002},
-		{Name: "e", Version: 1000},
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
 	shouldNeed := []protocol.FileInfo{
-		{Name: "b", Version: 1001},
-		{Name: "c", Version: 1002},
-		{Name: "e", Version: 1000},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 	m.Replace(remoteDevice0, remote)
 
 	need := needList(m, protocol.LocalDeviceID)
@@ -718,30 +711,30 @@ func TestLocalVersion(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: 1000},
-		{Name: "b", Version: 1000},
-		{Name: "c", Version: 1000},
-		{Name: "d", Version: 1000},
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
 	local2 := []protocol.FileInfo{
 		local1[0],
 		// [1] deleted
 		local1[2],
-		{Name: "d", Version: 1002},
-		{Name: "e", Version: 1000},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local1)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local1, myID)
 	c0 := m.LocalVersion(protocol.LocalDeviceID)
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local2)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local2, myID)
 	c1 := m.LocalVersion(protocol.LocalDeviceID)
 	if !(c1 > c0) {
 		t.Fatal("Local version number should have incremented")
 	}
 
-	m.ReplaceWithDelete(protocol.LocalDeviceID, local2)
+	m.ReplaceWithDelete(protocol.LocalDeviceID, local2, myID)
 	c2 := m.LocalVersion(protocol.LocalDeviceID)
 	if c2 != c1 {
 		t.Fatal("Local version number should be unchanged")
@@ -756,17 +749,17 @@ func TestListDropFolder(t *testing.T) {
 
 	s0 := db.NewFileSet("test0", ldb)
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: 1000},
-		{Name: "b", Version: 1000},
-		{Name: "c", Version: 1000},
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 	s0.Replace(protocol.LocalDeviceID, local1)
 
 	s1 := db.NewFileSet("test1", ldb)
 	local2 := []protocol.FileInfo{
-		{Name: "d", Version: 1002},
-		{Name: "e", Version: 1002},
-		{Name: "f", Version: 1002},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "f", Version: protocol.Vector{{ID: myID, Value: 1002}}},
 	}
 	s1.Replace(remoteDevice0, local2)
 
@@ -808,24 +801,24 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test1", ldb)
 
 	rem0 := fileList{
-		protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: 1002, Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
 	}
 	s.Replace(remoteDevice0, rem0)
 
 	rem1 := fileList{
-		protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: 1002, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: 1002, Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
 	}
 	s.Replace(remoteDevice1, rem1)
 
 	total := fileList{
 		// There's a valid copy of each file, so it should be merged
-		protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: 1002, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
 	}
 
 	need := fileList(needList(s, protocol.LocalDeviceID))
@@ -854,10 +847,10 @@ func TestLongPath(t *testing.T) {
 	name := b.String() // 5000 characters
 
 	local := []protocol.FileInfo{
-		{Name: string(name), Version: 1000},
+		{Name: string(name), Version: protocol.Vector{{ID: myID, Value: 1000}}},
 	}
 
-	s.ReplaceWithDelete(protocol.LocalDeviceID, local)
+	s.ReplaceWithDelete(protocol.LocalDeviceID, local, myID)
 
 	gf := globalList(s)
 	if l := len(gf); l != 1 {

internal/discover/discover.go

@@ -27,14 +27,17 @@ type Discoverer struct {
 	localBcastIntv  time.Duration
 	localBcastStart time.Time
 	cacheLifetime   time.Duration
+	negCacheCutoff  time.Duration
 	broadcastBeacon beacon.Interface
 	multicastBeacon beacon.Interface
-	registry        map[protocol.DeviceID][]CacheEntry
-	registryLock    sync.RWMutex
 	extPort         uint16
 	localBcastTick  <-chan time.Time
 	forcedBcastTick chan time.Time
 
+	registryLock sync.RWMutex
+	registry     map[protocol.DeviceID][]CacheEntry
+	lastLookup   map[protocol.DeviceID]time.Time
+
 	clients []Client
 	mut     sync.RWMutex
 }
@@ -54,7 +57,9 @@ func NewDiscoverer(id protocol.DeviceID, addresses []string) *Discoverer {
 		listenAddrs:    addresses,
 		localBcastIntv: 30 * time.Second,
 		cacheLifetime:  5 * time.Minute,
+		negCacheCutoff: 3 * time.Minute,
 		registry:       make(map[protocol.DeviceID][]CacheEntry),
+		lastLookup:     make(map[protocol.DeviceID]time.Time),
 	}
 }
 
@@ -155,18 +160,28 @@ func (d *Discoverer) ExtAnnounceOK() map[string]bool {
 func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
 	d.registryLock.RLock()
 	cached := d.filterCached(d.registry[device])
+	lastLookup := d.lastLookup[device]
 	d.registryLock.RUnlock()
 
 	d.mut.RLock()
 	defer d.mut.RUnlock()
 
-	var addrs []string
 	if len(cached) > 0 {
-		addrs = make([]string, len(cached))
+		// There are cached address entries.
+		addrs := make([]string, len(cached))
 		for i := range cached {
 			addrs[i] = cached[i].Address
 		}
-	} else if len(d.clients) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
+		return addrs
+	}
+
+	if time.Since(lastLookup) < d.negCacheCutoff {
+		// We have recently tried to lookup this address and failed. Lets
+		// chill for a while.
+		return nil
+	}
+
+	if len(d.clients) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
 		// Only perform external lookups if we have at least one external
 		// server client and one local announcement interval has passed. This is
 		// to avoid finding local peers on their remote address at startup.
@@ -187,6 +202,7 @@ func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
 		seen := make(map[string]struct{})
 		now := time.Now()
 
+		var addrs []string
 		for result := range results {
 			for _, addr := range result {
 				_, ok := seen[addr]
@@ -203,9 +219,13 @@ func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
 
 		d.registryLock.Lock()
 		d.registry[device] = cached
+		d.lastLookup[device] = time.Now()
 		d.registryLock.Unlock()
+
+		return addrs
 	}
-	return addrs
+
+	return nil
 }
 
 func (d *Discoverer) Hint(device string, addrs []string) {

internal/discover/packets_xdr.go

@@ -37,7 +37,7 @@ struct Query {
 
 func (o Query) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Query) MarshalXDR() ([]byte, error) {
@@ -55,11 +55,11 @@ func (o Query) MustMarshalXDR() []byte {
 func (o Query) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Query) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Query) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.Magic)
 	if l := len(o.DeviceID); l > 32 {
 		return xw.Tot(), xdr.ElementSizeExceeded("DeviceID", l, 32)
@@ -70,16 +70,16 @@ func (o Query) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Query) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Query) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Query) decodeXDR(xr *xdr.Reader) error {
+func (o *Query) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Magic = xr.ReadUint32()
 	o.DeviceID = xr.ReadBytesMax(32)
 	return xr.Error()
@@ -94,7 +94,9 @@ Announce Structure:
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                             Magic                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                            Device                             |
+/                                                               /
+\                       Device Structure                        \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                        Number of Extra                        |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -114,7 +116,7 @@ struct Announce {
 
 func (o Announce) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Announce) MarshalXDR() ([]byte, error) {
@@ -132,13 +134,13 @@ func (o Announce) MustMarshalXDR() []byte {
 func (o Announce) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Announce) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Announce) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.Magic)
-	_, err := o.This.encodeXDR(xw)
+	_, err := o.This.EncodeXDRInto(xw)
 	if err != nil {
 		return xw.Tot(), err
 	}
@@ -147,7 +149,7 @@ func (o Announce) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Extra)))
 	for i := range o.Extra {
-		_, err := o.Extra[i].encodeXDR(xw)
+		_, err := o.Extra[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -157,25 +159,25 @@ func (o Announce) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Announce) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Announce) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Announce) decodeXDR(xr *xdr.Reader) error {
+func (o *Announce) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Magic = xr.ReadUint32()
-	(&o.This).decodeXDR(xr)
+	(&o.This).DecodeXDRFrom(xr)
 	_ExtraSize := int(xr.ReadUint32())
 	if _ExtraSize > 16 {
 		return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
 	}
 	o.Extra = make([]Device, _ExtraSize)
 	for i := range o.Extra {
-		(&o.Extra[i]).decodeXDR(xr)
+		(&o.Extra[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -210,7 +212,7 @@ struct Device {
 
 func (o Device) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Device) MarshalXDR() ([]byte, error) {
@@ -228,11 +230,11 @@ func (o Device) MustMarshalXDR() []byte {
 func (o Device) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Device) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.ID); l > 32 {
 		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32)
 	}
@@ -242,7 +244,7 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Addresses)))
 	for i := range o.Addresses {
-		_, err := o.Addresses[i].encodeXDR(xw)
+		_, err := o.Addresses[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -252,16 +254,16 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Device) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Device) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Device) decodeXDR(xr *xdr.Reader) error {
+func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ID = xr.ReadBytesMax(32)
 	_AddressesSize := int(xr.ReadUint32())
 	if _AddressesSize > 16 {
@@ -269,7 +271,7 @@ func (o *Device) decodeXDR(xr *xdr.Reader) error {
 	}
 	o.Addresses = make([]Address, _AddressesSize)
 	for i := range o.Addresses {
-		(&o.Addresses[i]).decodeXDR(xr)
+		(&o.Addresses[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -300,7 +302,7 @@ struct Address {
 
 func (o Address) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Address) MarshalXDR() ([]byte, error) {
@@ -318,11 +320,11 @@ func (o Address) MustMarshalXDR() []byte {
 func (o Address) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Address) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Address) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.IP); l > 16 {
 		return xw.Tot(), xdr.ElementSizeExceeded("IP", l, 16)
 	}
@@ -333,16 +335,16 @@ func (o Address) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Address) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Address) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Address) decodeXDR(xr *xdr.Reader) error {
+func (o *Address) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.IP = xr.ReadBytesMax(16)
 	o.Port = xr.ReadUint16()
 	return xr.Error()

internal/events/events.go

@@ -31,6 +31,8 @@ const (
 	FolderRejected
 	ConfigSaved
 	DownloadProgress
+	FolderSummary
+	FolderCompletion
 
 	AllEvents = (1 << iota) - 1
 )
@@ -67,6 +69,10 @@ func (t EventType) String() string {
 		return "ConfigSaved"
 	case DownloadProgress:
 		return "DownloadProgress"
+	case FolderSummary:
+		return "FolderSummary"
+	case FolderCompletion:
+		return "FolderCompletion"
 	default:
 		return "Unknown"
 	}

internal/fnmatch/fnmatch.go

@@ -38,6 +38,13 @@ func Convert(pattern string, flags int) (*regexp.Regexp, error) {
 		}
 	}
 
+	// Support case insensitive ignores
+	ignore := strings.TrimPrefix(pattern, "(?i)")
+	if ignore != pattern {
+		flags |= FNM_CASEFOLD
+		pattern = ignore
+	}
+
 	if flags&FNM_NOESCAPE != 0 {
 		pattern = strings.Replace(pattern, "\\", "\\\\", -1)
 	} else {

internal/fnmatch/fnmatch_test.go

@@ -53,6 +53,8 @@ var testcases = []testcase{
 	{"**/foo.txt", "bar/baz/foo.txt", FNM_PATHNAME, true},
 
 	{"foo.txt", "foo.TXT", FNM_CASEFOLD, true},
+	{"foo.txt", "foo.TXT", 0, false},
+	{"(?i)foo.txt", "foo.TXT", 0, true},
 
 	// These characters are literals in glob, but not in regexp.
 	{"hey$hello", "hey$hello", 0, true},

internal/lamport/clock.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-// Package lamport implements a simple Lamport Clock for versioning
-package lamport
-
-import "sync"
-
-var Default = Clock{}
-
-type Clock struct {
-	val int64
-	mut sync.Mutex
-}
-
-func (c *Clock) Tick(v int64) int64 {
-	c.mut.Lock()
-	if v > c.val {
-		c.val = v + 1
-		c.mut.Unlock()
-		return v + 1
-	} else {
-		c.val++
-		v = c.val
-		c.mut.Unlock()
-		return v
-	}
-}

internal/lamport/clock_test.go

@@ -1,24 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package lamport
-
-import "testing"
-
-var inputs = []int64{0, 42, 2, 3, 4, 8, 9, 33, 44, 112, 100}
-
-func TestClock(t *testing.T) {
-	c := Clock{}
-
-	var prev int64
-	for _, input := range inputs {
-		cur := c.Tick(input)
-		if cur <= prev || cur <= input {
-			t.Error("Clock moving backwards")
-		}
-		prev = cur
-	}
-}

internal/model/model.go

@@ -27,7 +27,6 @@ import (
 	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syncthing/syncthing/internal/events"
 	"github.com/syncthing/syncthing/internal/ignore"
-	"github.com/syncthing/syncthing/internal/lamport"
 	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/scanner"
 	"github.com/syncthing/syncthing/internal/stats"
@@ -59,6 +58,8 @@ type Model struct {
 	db              *leveldb.DB
 	finder          *db.BlockFinder
 	progressEmitter *ProgressEmitter
+	id              protocol.DeviceID
+	shortID         uint64
 
 	deviceName    string
 	clientName    string
@@ -84,19 +85,20 @@ type Model struct {
 }
 
 var (
-	ErrNoSuchFile = errors.New("no such file")
-	ErrInvalid    = errors.New("file is invalid")
-
 	SymlinkWarning = sync.Once{}
 )
 
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local folder in any way.
-func NewModel(cfg *config.Wrapper, deviceName, clientName, clientVersion string, ldb *leveldb.DB) *Model {
+func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB) *Model {
 	m := &Model{
 		cfg:             cfg,
 		db:              ldb,
+		finder:          db.NewBlockFinder(ldb, cfg),
+		progressEmitter: NewProgressEmitter(cfg),
+		id:              id,
+		shortID:         id.Short(),
 		deviceName:      deviceName,
 		clientName:      clientName,
 		clientVersion:   clientVersion,
@@ -111,8 +113,6 @@ func NewModel(cfg *config.Wrapper, deviceName, clientName, clientVersion string,
 		protoConn:       make(map[protocol.DeviceID]protocol.Connection),
 		rawConn:         make(map[protocol.DeviceID]io.Closer),
 		deviceVer:       make(map[protocol.DeviceID]string),
-		finder:          db.NewBlockFinder(ldb, cfg),
-		progressEmitter: NewProgressEmitter(cfg),
 	}
 	if cfg.Options().ProgressUpdateIntervalS > -1 {
 		go m.progressEmitter.Serve()
@@ -420,7 +420,12 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]db.FileInfoTruncated,
 
 // Index is called when a new device is connected and we receive their full index.
 // Implements the protocol.Model interface.
-func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
+func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
+	if flags != 0 {
+		l.Warnln("protocol error: unknown flags 0x%x in Index message", flags)
+		return
+	}
+
 	if debug {
 		l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
 	}
@@ -443,7 +448,6 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 	}
 
 	for i := 0; i < len(fs); {
-		lamport.Default.Tick(fs[i].Version)
 		if fs[i].Flags&^protocol.FlagsAll != 0 {
 			if debug {
 				l.Debugln("dropping update for file with unknown bits set", fs[i])
@@ -473,7 +477,12 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 // IndexUpdate is called for incremental updates to connected devices' indexes.
 // Implements the protocol.Model interface.
-func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
+func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
+	if flags != 0 {
+		l.Warnln("protocol error: unknown flags 0x%x in IndexUpdate message", flags)
+		return
+	}
+
 	if debug {
 		l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
 	}
@@ -492,7 +501,6 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
 	}
 
 	for i := 0; i < len(fs); {
-		lamport.Default.Tick(fs[i].Version)
 		if fs[i].Flags&^protocol.FlagsAll != 0 {
 			if debug {
 				l.Debugln("dropping update for file with unknown bits set", fs[i])
@@ -671,14 +679,19 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 
 // Request returns the specified data segment by reading it from local disk.
 // Implements the protocol.Model interface.
-func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, size int) ([]byte, error) {
+func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) {
 	if offset < 0 || size < 0 {
-		return nil, ErrNoSuchFile
+		return nil, protocol.ErrNoSuchFile
 	}
 
 	if !m.folderSharedWith(folder, deviceID) {
 		l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
-		return nil, ErrNoSuchFile
+		return nil, protocol.ErrNoSuchFile
+	}
+
+	if flags != 0 {
+		// We don't currently support or expect any flags.
+		return nil, fmt.Errorf("protocol error: unknown flags 0x%x in Request message", flags)
 	}
 
 	// Verify that the requested file exists in the local model.
@@ -688,26 +701,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 
 	if !ok {
 		l.Warnf("Request from %s for file %s in nonexistent folder %q", deviceID, name, folder)
-		return nil, ErrNoSuchFile
+		return nil, protocol.ErrNoSuchFile
 	}
 
 	lf, ok := folderFiles.Get(protocol.LocalDeviceID, name)
 	if !ok {
-		return nil, ErrNoSuchFile
+		return nil, protocol.ErrNoSuchFile
 	}
 
 	if lf.IsInvalid() || lf.IsDeleted() {
 		if debug {
 			l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, deviceID, folder, name, offset, size, lf)
 		}
-		return nil, ErrInvalid
+		return nil, protocol.ErrInvalid
 	}
 
 	if offset > lf.Size() {
 		if debug {
 			l.Debugf("%v REQ(in; nonexistent): %s: %q o=%d s=%d", m, deviceID, name, offset, size)
 		}
-		return nil, ErrNoSuchFile
+		return nil, protocol.ErrNoSuchFile
 	}
 
 	if debug && deviceID != protocol.LocalDeviceID {
@@ -748,7 +761,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 // ReplaceLocal replaces the local folder index with the given list of files.
 func (m *Model) ReplaceLocal(folder string, fs []protocol.FileInfo) {
 	m.fmut.RLock()
-	m.folderFiles[folder].ReplaceWithDelete(protocol.LocalDeviceID, fs)
+	m.folderFiles[folder].ReplaceWithDelete(protocol.LocalDeviceID, fs, m.shortID)
 	m.fmut.RUnlock()
 }
 
@@ -974,7 +987,7 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 
 		if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
 			if initial {
-				if err = conn.Index(folder, batch); err != nil {
+				if err = conn.Index(folder, batch, 0, nil); err != nil {
 					return false
 				}
 				if debug {
@@ -982,7 +995,7 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 				}
 				initial = false
 			} else {
-				if err = conn.IndexUpdate(folder, batch); err != nil {
+				if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil {
 					return false
 				}
 				if debug {
@@ -1000,12 +1013,12 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 	})
 
 	if initial && err == nil {
-		err = conn.Index(folder, batch)
+		err = conn.Index(folder, batch, 0, nil)
 		if debug && err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch))
 		}
 	} else if len(batch) > 0 && err == nil {
-		err = conn.IndexUpdate(folder, batch)
+		err = conn.IndexUpdate(folder, batch, 0, nil)
 		if debug && err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch))
 		}
@@ -1028,7 +1041,7 @@ func (m *Model) updateLocal(folder string, f protocol.FileInfo) {
 	})
 }
 
-func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte) ([]byte, error) {
+func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) {
 	m.pmut.RLock()
 	nc, ok := m.protoConn[deviceID]
 	m.pmut.RUnlock()
@@ -1038,10 +1051,10 @@ func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o
 	}
 
 	if debug {
-		l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x", m, deviceID, folder, name, offset, size, hash)
+		l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x f=%x op=%s", m, deviceID, folder, name, offset, size, hash, flags, options)
 	}
 
-	return nc.Request(folder, name, offset, size)
+	return nc.Request(folder, name, offset, size, hash, flags, options)
 }
 
 func (m *Model) AddFolder(cfg config.FolderConfiguration) {
@@ -1091,7 +1104,11 @@ func (m *Model) ScanFolders() map[string]error {
 				errorsMut.Lock()
 				errors[folder] = err
 				errorsMut.Unlock()
-				m.cfg.InvalidateFolder(folder, err.Error())
+				// Potentially sets the error twice, once in the scanner just
+				// by doing a check, and once here, if the error returned is
+				// the same one as returned by CheckFolderHealth, though
+				// duplicate set is handled by SetFolderError
+				m.cfg.SetFolderError(folder, err)
 			}
 			wg.Done()
 		}()
@@ -1101,13 +1118,16 @@ func (m *Model) ScanFolders() map[string]error {
 }
 
 func (m *Model) ScanFolder(folder string) error {
-	return m.ScanFolderSub(folder, "")
+	return m.ScanFolderSubs(folder, nil)
 }
 
-func (m *Model) ScanFolderSub(folder, sub string) error {
-	sub = osutil.NativeFilename(sub)
-	if p := filepath.Clean(filepath.Join(folder, sub)); !strings.HasPrefix(p, folder) {
-		return errors.New("invalid subpath")
+func (m *Model) ScanFolderSubs(folder string, subs []string) error {
+	for i, sub := range subs {
+		sub = osutil.NativeFilename(sub)
+		if p := filepath.Clean(filepath.Join(folder, sub)); !strings.HasPrefix(p, folder) {
+			return errors.New("invalid subpath")
+		}
+		subs[i] = sub
 	}
 
 	m.fmut.Lock()
@@ -1128,19 +1148,30 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 
 	// Required to make sure that we start indexing at a directory we're already
 	// aware off.
-	for sub != "" {
-		if _, ok = fs.Get(protocol.LocalDeviceID, sub); ok {
-			break
+	var unifySubs []string
+nextSub:
+	for _, sub := range subs {
+		for sub != "" {
+			if _, ok = fs.Get(protocol.LocalDeviceID, sub); ok {
+				break
+			}
+			sub = filepath.Dir(sub)
+			if sub == "." || sub == string(filepath.Separator) {
+				sub = ""
+			}
 		}
-		sub = filepath.Dir(sub)
-		if sub == "." || sub == string(filepath.Separator) {
-			sub = ""
+		for _, us := range unifySubs {
+			if strings.HasPrefix(sub, us) {
+				continue nextSub
+			}
 		}
+		unifySubs = append(unifySubs, sub)
 	}
+	subs = unifySubs
 
 	w := &scanner.Walker{
 		Dir:           folderCfg.Path,
-		Sub:           sub,
+		Subs:          subs,
 		Matcher:       ignores,
 		BlockSize:     protocol.BlockSize,
 		TempNamer:     defTempNamer,
@@ -1149,12 +1180,15 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 		IgnorePerms:   folderCfg.IgnorePerms,
 		AutoNormalize: folderCfg.AutoNormalize,
 		Hashers:       folderCfg.Hashers,
+		ShortID:       m.shortID,
 	}
 
 	runner.setState(FolderScanning)
+	defer runner.setState(FolderIdle)
 	fchan, err := w.Walk()
 
 	if err != nil {
+		m.cfg.SetFolderError(folder, err)
 		return err
 	}
 	batchSize := 100
@@ -1168,12 +1202,20 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 			"size":     f.Size(),
 		})
 		if len(batch) == batchSize {
+			if err := m.CheckFolderHealth(folder); err != nil {
+				l.Infof("Stopping folder %s mid-scan due to folder error: %s", folder, err)
+				return err
+			}
 			fs.Update(protocol.LocalDeviceID, batch)
 			batch = batch[:0]
 		}
 		batch = append(batch, f)
 	}
-	if len(batch) > 0 {
+
+	if err := m.CheckFolderHealth(folder); err != nil {
+		l.Infof("Stopping folder %s mid-scan due to folder error: %s", folder, err)
+		return err
+	} else if len(batch) > 0 {
 		fs.Update(protocol.LocalDeviceID, batch)
 	}
 
@@ -1182,10 +1224,17 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 	seenPrefix := false
 	fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 		f := fi.(db.FileInfoTruncated)
-		if !strings.HasPrefix(f.Name, sub) {
-			// Return true so that we keep iterating, until we get to the part
-			// of the tree we are interested in. Then return false so we stop
-			// iterating when we've passed the end of the subtree.
+		hasPrefix := len(subs) == 0
+		for _, sub := range subs {
+			if strings.HasPrefix(f.Name, sub) {
+				hasPrefix = true
+				break
+			}
+		}
+		// Return true so that we keep iterating, until we get to the part
+		// of the tree we are interested in. Then return false so we stop
+		// iterating when we've passed the end of the subtree.
+		if !hasPrefix {
 			return !seenPrefix
 		}
 
@@ -1233,7 +1282,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 					Name:     f.Name,
 					Flags:    f.Flags | protocol.FlagDeleted,
 					Modified: f.Modified,
-					Version:  lamport.Default.Tick(f.Version),
+					Version:  f.Version.Update(m.shortID),
 				}
 				events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
 					"folder":   folder,
@@ -1251,7 +1300,6 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 		fs.Update(protocol.LocalDeviceID, batch)
 	}
 
-	runner.setState(FolderIdle)
 	return nil
 }
 
@@ -1329,7 +1377,7 @@ func (m *Model) Override(folder string) {
 			// We have the file, replace with our version
 			need = have
 		}
-		need.Version = lamport.Default.Tick(need.Version)
+		need.Version = need.Version.Update(m.shortID)
 		need.LocalVersion = 0
 		batch = append(batch, need)
 		return true
@@ -1475,6 +1523,73 @@ func (m *Model) BringToFront(folder, file string) {
 	}
 }
 
+// Returns current folder error, or nil if the folder is healthy.
+// Updates the Invalid field on the folder configuration struct, and emits a
+// ConfigSaved event which causes a GUI refresh.
+func (m *Model) CheckFolderHealth(id string) error {
+	folder, ok := m.cfg.Folders()[id]
+	if !ok {
+		return errors.New("Folder does not exist")
+	}
+
+	fi, err := os.Stat(folder.Path)
+	if m.CurrentLocalVersion(id) > 0 {
+		// Safety check. If the cached index contains files but the
+		// folder doesn't exist, we have a problem. We would assume
+		// that all files have been deleted which might not be the case,
+		// so mark it as invalid instead.
+		if err != nil || !fi.IsDir() {
+			err = errors.New("Folder path missing")
+		} else if !folder.HasMarker() {
+			err = errors.New("Folder marker missing")
+		}
+	} else if os.IsNotExist(err) {
+		// If we don't have any files in the index, and the directory
+		// doesn't exist, try creating it.
+		err = os.MkdirAll(folder.Path, 0700)
+		if err == nil {
+			err = folder.CreateMarker()
+		}
+	} else if !folder.HasMarker() {
+		// If we don't have any files in the index, and the path does exist
+		// but the marker is not there, create it.
+		err = folder.CreateMarker()
+	}
+
+	if err == nil {
+		if folder.Invalid != "" {
+			l.Infof("Starting folder %q after error %q", folder.ID, folder.Invalid)
+			m.cfg.SetFolderError(id, nil)
+		}
+
+		if folder, ok := m.cfg.Folders()[id]; !ok || folder.Invalid != "" {
+			panic("Unable to unset folder \"" + id + "\" error.")
+		}
+
+		return nil
+	}
+
+	if folder.Invalid == err.Error() {
+		return err
+	}
+
+	// folder is a copy of the original struct, hence Invalid value is
+	// preserved after the set.
+	m.cfg.SetFolderError(id, err)
+
+	if folder.Invalid == "" {
+		l.Warnf("Stopping folder %q - %v", folder.ID, err)
+	} else {
+		l.Infof("Folder %q error changed: %q -> %q", folder.ID, folder.Invalid, err)
+	}
+
+	if folder, ok := m.cfg.Folders()[id]; !ok || folder.Invalid != err.Error() {
+		panic("Unable to set folder \"" + id + "\" error.")
+	}
+
+	return err
+}
+
 func (m *Model) String() string {
 	return fmt.Sprintf("model@%p", m)
 }

internal/model/model_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
@@ -86,7 +87,7 @@ func init() {
 func TestRequest(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 
 	// device1 shares default, but device2 doesn't
 	m.AddFolder(defaultFolderConfig)
@@ -94,7 +95,7 @@ func TestRequest(t *testing.T) {
 	m.ScanFolder("default")
 
 	// Existing, shared file
-	bs, err := m.Request(device1, "default", "foo", 0, 6)
+	bs, err := m.Request(device1, "default", "foo", 0, 6, nil, 0, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -103,7 +104,7 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Existing, nonshared file
-	bs, err = m.Request(device2, "default", "foo", 0, 6)
+	bs, err = m.Request(device2, "default", "foo", 0, 6, nil, 0, nil)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -112,7 +113,7 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Nonexistent file
-	bs, err = m.Request(device1, "default", "nonexistent", 0, 6)
+	bs, err = m.Request(device1, "default", "nonexistent", 0, 6, nil, 0, nil)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -121,7 +122,7 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Shared folder, but disallowed file name
-	bs, err = m.Request(device1, "default", "../walk.go", 0, 6)
+	bs, err = m.Request(device1, "default", "../walk.go", 0, 6, nil, 0, nil)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -130,7 +131,7 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Larger block than available
-	bs, err = m.Request(device1, "default", "foo", 0, 42)
+	bs, err = m.Request(device1, "default", "foo", 0, 42, nil, 0, nil)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -139,7 +140,7 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Negative offset
-	bs, err = m.Request(device1, "default", "foo", -4, 6)
+	bs, err = m.Request(device1, "default", "foo", -4, 6, nil, 0, nil)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -148,7 +149,7 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Negative size
-	bs, err = m.Request(device1, "default", "foo", 4, -4)
+	bs, err = m.Request(device1, "default", "foo", 4, -4, nil, 0, nil)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -173,71 +174,71 @@ func genFiles(n int) []protocol.FileInfo {
 
 func BenchmarkIndex10000(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.Index(device1, "default", files)
+		m.Index(device1, "default", files, 0, nil)
 	}
 }
 
 func BenchmarkIndex00100(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(100)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.Index(device1, "default", files)
+		m.Index(device1, "default", files, 0, nil)
 	}
 }
 
 func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", files)
+		m.IndexUpdate(device1, "default", files, 0, nil)
 	}
 }
 
 func BenchmarkIndexUpdate10000f00100(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	ufiles := genFiles(100)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", ufiles)
+		m.IndexUpdate(device1, "default", ufiles, 0, nil)
 	}
 }
 
 func BenchmarkIndexUpdate10000f00001(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	ufiles := genFiles(1)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", ufiles)
+		m.IndexUpdate(device1, "default", ufiles, 0, nil)
 	}
 }
 
@@ -262,15 +263,15 @@ func (f FakeConnection) Option(string) string {
 	return ""
 }
 
-func (FakeConnection) Index(string, []protocol.FileInfo) error {
+func (FakeConnection) Index(string, []protocol.FileInfo, uint32, []protocol.Option) error {
 	return nil
 }
 
-func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error {
+func (FakeConnection) IndexUpdate(string, []protocol.FileInfo, uint32, []protocol.Option) error {
 	return nil
 }
 
-func (f FakeConnection) Request(folder, name string, offset int64, size int) ([]byte, error) {
+func (f FakeConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) {
 	return f.requestData, nil
 }
 
@@ -286,7 +287,7 @@ func (FakeConnection) Statistics() protocol.Statistics {
 
 func BenchmarkRequest(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 
@@ -306,11 +307,11 @@ func BenchmarkRequest(b *testing.B) {
 		requestData: []byte("some data to return"),
 	}
 	m.AddConnection(fc, fc)
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil)
+		data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, 0, nil)
 		if err != nil {
 			b.Error(err)
 		}
@@ -336,7 +337,7 @@ func TestDeviceRename(t *testing.T) {
 	}
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(config.Wrap("tmpconfig.xml", cfg), "device", "syncthing", "dev", db)
+	m := NewModel(config.Wrap("tmpconfig.xml", cfg), protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	if cfg.Devices[0].Name != "" {
 		t.Errorf("Device already has a name")
 	}
@@ -403,7 +404,7 @@ func TestClusterConfig(t *testing.T) {
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 
-	m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
+	m := NewModel(config.Wrap("/tmp/test", cfg), protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(cfg.Folders[0])
 	m.AddFolder(cfg.Folders[1])
 
@@ -469,7 +470,7 @@ func TestIgnores(t *testing.T) {
 	}
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.StartFolderRO("default")
 
@@ -543,7 +544,7 @@ func TestIgnores(t *testing.T) {
 
 func TestRefuseUnknownBits(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 
 	m.ScanFolder("default")
@@ -564,7 +565,7 @@ func TestRefuseUnknownBits(t *testing.T) {
 			Name:  "valid",
 			Flags: protocol.FlagsAll &^ (protocol.FlagInvalid | protocol.FlagSymlink),
 		},
-	})
+	}, 0, nil)
 
 	for _, name := range []string{"invalid1", "invalid2", "invalid3"} {
 		f, ok := m.CurrentGlobalFile("default", name)
@@ -578,9 +579,169 @@ func TestRefuseUnknownBits(t *testing.T) {
 	}
 }
 
+func TestROScanRecovery(t *testing.T) {
+	ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	set := db.NewFileSet("default", ldb)
+	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{Name: "dummyfile"},
+	})
+
+	fcfg := config.FolderConfiguration{
+		ID:              "default",
+		Path:            "testdata/rotestfolder",
+		RescanIntervalS: 1,
+	}
+	cfg := config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+		Devices: []config.DeviceConfiguration{
+			{
+				DeviceID: device1,
+			},
+		},
+	})
+
+	os.RemoveAll(fcfg.Path)
+
+	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+
+	m.AddFolder(fcfg)
+	m.StartFolderRO("default")
+
+	waitFor := func(status string) error {
+		timeout := time.Now().Add(2 * time.Second)
+		for {
+			if time.Now().After(timeout) {
+				return fmt.Errorf("Timed out waiting for status: %s, current status: %s", status, m.cfg.Folders()["default"].Invalid)
+			}
+			if m.cfg.Folders()["default"].Invalid == status {
+				return nil
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	if err := waitFor("Folder path missing"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Mkdir(fcfg.Path, 0700)
+
+	if err := waitFor("Folder marker missing"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	fd.Close()
+
+	if err := waitFor(""); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
+
+	if err := waitFor("Folder marker missing"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(fcfg.Path)
+
+	if err := waitFor("Folder path missing"); err != nil {
+		t.Error(err)
+		return
+	}
+}
+
+func TestRWScanRecovery(t *testing.T) {
+	ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	set := db.NewFileSet("default", ldb)
+	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{Name: "dummyfile"},
+	})
+
+	fcfg := config.FolderConfiguration{
+		ID:              "default",
+		Path:            "testdata/rwtestfolder",
+		RescanIntervalS: 1,
+	}
+	cfg := config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+		Devices: []config.DeviceConfiguration{
+			{
+				DeviceID: device1,
+			},
+		},
+	})
+
+	os.RemoveAll(fcfg.Path)
+
+	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+
+	m.AddFolder(fcfg)
+	m.StartFolderRW("default")
+
+	waitFor := func(status string) error {
+		timeout := time.Now().Add(2 * time.Second)
+		for {
+			if time.Now().After(timeout) {
+				return fmt.Errorf("Timed out waiting for status: %s, current status: %s", status, m.cfg.Folders()["default"].Invalid)
+			}
+			if m.cfg.Folders()["default"].Invalid == status {
+				return nil
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	if err := waitFor("Folder path missing"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Mkdir(fcfg.Path, 0700)
+
+	if err := waitFor("Folder marker missing"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	fd.Close()
+
+	if err := waitFor(""); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
+
+	if err := waitFor("Folder marker missing"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Remove(fcfg.Path)
+
+	if err := waitFor("Folder path missing"); err != nil {
+		t.Error(err)
+		return
+	}
+}
+
 func TestGlobalDirectoryTree(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
@@ -666,7 +827,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata)
+	m.Index(device1, "default", testdata, 0, nil)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -830,7 +991,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 func TestGlobalDirectorySelfFixing(t *testing.T) {
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
@@ -925,7 +1086,7 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata)
+	m.Index(device1, "default", testdata, 0, nil)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -991,12 +1152,12 @@ func genDeepFiles(n, d int) []protocol.FileInfo {
 
 func BenchmarkTree_10000_50(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genDeepFiles(10000, 50)
 
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -1006,12 +1167,12 @@ func BenchmarkTree_10000_50(b *testing.B) {
 
 func BenchmarkTree_10000_10(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genDeepFiles(10000, 10)
 
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -1021,12 +1182,12 @@ func BenchmarkTree_10000_10(b *testing.B) {
 
 func BenchmarkTree_00100_50(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genDeepFiles(100, 50)
 
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -1036,12 +1197,12 @@ func BenchmarkTree_00100_50(b *testing.B) {
 
 func BenchmarkTree_00100_10(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genDeepFiles(100, 10)
 
-	m.Index(device1, "default", files)
+	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

internal/model/rofolder.go

@@ -40,6 +40,12 @@ func (s *roFolder) Serve() {
 	timer := time.NewTimer(time.Millisecond)
 	defer timer.Stop()
 
+	reschedule := func() {
+		// Sleep a random time between 3/4 and 5/4 of the configured interval.
+		sleepNanos := (s.intv.Nanoseconds()*3 + rand.Int63n(2*s.intv.Nanoseconds())) / 4
+		timer.Reset(time.Duration(sleepNanos) * time.Nanosecond)
+	}
+
 	initialScanCompleted := false
 	for {
 		select {
@@ -47,16 +53,25 @@ func (s *roFolder) Serve() {
 			return
 
 		case <-timer.C:
+			if err := s.model.CheckFolderHealth(s.folder); err != nil {
+				l.Infoln("Skipping folder", s.folder, "scan due to folder error:", err)
+				reschedule()
+				continue
+			}
+
 			if debug {
 				l.Debugln(s, "rescan")
 			}
 
-			s.setState(FolderScanning)
 			if err := s.model.ScanFolder(s.folder); err != nil {
-				s.model.cfg.InvalidateFolder(s.folder, err.Error())
-				return
+				// Potentially sets the error twice, once in the scanner just
+				// by doing a check, and once here, if the error returned is
+				// the same one as returned by CheckFolderHealth, though
+				// duplicate set is handled by SetFolderError
+				s.model.cfg.SetFolderError(s.folder, err)
+				reschedule()
+				continue
 			}
-			s.setState(FolderIdle)
 
 			if !initialScanCompleted {
 				l.Infoln("Completed initial scan (ro) of folder", s.folder)
@@ -67,9 +82,7 @@ func (s *roFolder) Serve() {
 				return
 			}
 
-			// Sleep a random time between 3/4 and 5/4 of the configured interval.
-			sleepNanos := (s.intv.Nanoseconds()*3 + rand.Int63n(2*s.intv.Nanoseconds())) / 4
-			timer.Reset(time.Duration(sleepNanos) * time.Nanosecond)
+			reschedule()
 		}
 	}
 }

internal/model/rwfolder.go

@@ -114,10 +114,20 @@ func (p *rwFolder) Serve() {
 	var prevVer int64
 	var prevIgnoreHash string
 
+	rescheduleScan := func() {
+		// Sleep a random time between 3/4 and 5/4 of the configured interval.
+		sleepNanos := (p.scanIntv.Nanoseconds()*3 + rand.Int63n(2*p.scanIntv.Nanoseconds())) / 4
+		intv := time.Duration(sleepNanos) * time.Nanosecond
+
+		if debug {
+			l.Debugln(p, "next rescan in", intv)
+		}
+		scanTimer.Reset(intv)
+	}
+
 	// We don't start pulling files until a scan has been completed.
 	initialScanCompleted := false
 
-loop:
 	for {
 		select {
 		case <-p.stop:
@@ -130,7 +140,6 @@ loop:
 		// device A to device B, so we have something to work against.
 		case <-pullTimer.C:
 			if !initialScanCompleted {
-				// How did we even get here?
 				if debug {
 					l.Debugln(p, "skip (initial)")
 				}
@@ -219,24 +228,28 @@ loop:
 		// this is the easiest way to make sure we are not doing both at the
 		// same time.
 		case <-scanTimer.C:
+			if err := p.model.CheckFolderHealth(p.folder); err != nil {
+				l.Infoln("Skipping folder", p.folder, "scan due to folder error:", err)
+				rescheduleScan()
+				continue
+			}
+
 			if debug {
 				l.Debugln(p, "rescan")
 			}
-			p.setState(FolderScanning)
-			if err := p.model.ScanFolder(p.folder); err != nil {
-				p.model.cfg.InvalidateFolder(p.folder, err.Error())
-				break loop
-			}
-			p.setState(FolderIdle)
-			if p.scanIntv > 0 {
-				// Sleep a random time between 3/4 and 5/4 of the configured interval.
-				sleepNanos := (p.scanIntv.Nanoseconds()*3 + rand.Int63n(2*p.scanIntv.Nanoseconds())) / 4
-				intv := time.Duration(sleepNanos) * time.Nanosecond
 
-				if debug {
-					l.Debugln(p, "next rescan in", intv)
-				}
-				scanTimer.Reset(intv)
+			if err := p.model.ScanFolder(p.folder); err != nil {
+				// Potentially sets the error twice, once in the scanner just
+				// by doing a check, and once here, if the error returned is
+				// the same one as returned by CheckFolderHealth, though
+				// duplicate set is handled by SetFolderError
+				p.model.cfg.SetFolderError(p.folder, err)
+				rescheduleScan()
+				continue
+			}
+
+			if p.scanIntv > 0 {
+				rescheduleScan()
 			}
 			if !initialScanCompleted {
 				l.Infoln("Completed initial scan (rw) of folder", p.folder)
@@ -489,7 +502,11 @@ func (p *rwFolder) handleDir(file protocol.FileInfo) {
 		// we can pass it to InWritableDir. We use a regular Mkdir and
 		// not MkdirAll because the parent should already exist.
 		mkdir := func(path string) error {
-			return os.Mkdir(path, mode)
+			err = os.Mkdir(path, mode)
+			if err != nil || p.ignorePerms {
+				return err
+			}
+			return os.Chmod(path, mode)
 		}
 
 		if err = osutil.InWritableDir(mkdir, realName); err == nil {
@@ -571,7 +588,11 @@ func (p *rwFolder) deleteFile(file protocol.FileInfo) {
 
 	realName := filepath.Join(p.dir, file.Name)
 
-	if p.versioner != nil {
+	cur, ok := p.model.CurrentFolderFile(p.folder, file.Name)
+	if ok && cur.Version.Concurrent(file.Version) {
+		// There is a conflict here. Move the file to a conflict copy instead of deleting.
+		err = osutil.InWritableDir(moveForConflict, realName)
+	} else if p.versioner != nil {
 		err = osutil.InWritableDir(p.versioner.Archive, realName)
 	} else {
 		err = osutil.InWritableDir(os.Remove, realName)
@@ -704,9 +725,9 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
 
 		// block.String() returns a string unique to the block
-		existingBlocks := make(map[string]bool, len(tempCopyBlocks))
+		existingBlocks := make(map[string]struct{}, len(tempCopyBlocks))
 		for _, block := range tempCopyBlocks {
-			existingBlocks[block.String()] = true
+			existingBlocks[block.String()] = struct{}{}
 		}
 
 		// Since the blocks are already there, we don't need to get them.
@@ -739,6 +760,7 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		copyNeeded:  len(blocks),
 		reused:      reused,
 		ignorePerms: p.ignorePerms,
+		version:     curFile.Version,
 	}
 
 	if debug {
@@ -914,7 +936,7 @@ func (p *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul
 			// Fetch the block, while marking the selected device as in use so that
 			// leastBusy can select another device when someone else asks.
 			activity.using(selected)
-			buf, lastError := p.model.requestGlobal(selected, p.folder, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash)
+			buf, lastError := p.model.requestGlobal(selected, p.folder, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, 0, nil)
 			activity.done(selected)
 			if lastError != nil {
 				continue
@@ -949,6 +971,7 @@ func (p *rwFolder) performFinish(state *sharedPullerState) {
 			"error":  err,
 		})
 	}()
+
 	// Set the correct permission bits on the new file
 	if !p.ignorePerms {
 		err = os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777))
@@ -974,15 +997,22 @@ func (p *rwFolder) performFinish(state *sharedPullerState) {
 		}
 	}
 
-	// If we should use versioning, let the versioner archive the old
-	// file before we replace it. Archiving a non-existent file is not
-	// an error.
-	if p.versioner != nil {
+	if state.version.Concurrent(state.file.Version) {
+		// The new file has been changed in conflict with the existing one. We
+		// should file it away as a conflict instead of just removing or
+		// archiving.
+		err = osutil.InWritableDir(moveForConflict, state.realName)
+	} else if p.versioner != nil {
+		// If we should use versioning, let the versioner archive the old
+		// file before we replace it. Archiving a non-existent file is not
+		// an error.
 		err = p.versioner.Archive(state.realName)
-		if err != nil {
-			l.Warnln("Puller: final:", err)
-			return
-		}
+	} else {
+		err = nil
+	}
+	if err != nil {
+		l.Warnln("Puller: final:", err)
+		return
 	}
 
 	// If the target path is a symlink or a directory, we cannot copy
@@ -1078,3 +1108,10 @@ func removeDevice(devices []protocol.DeviceID, device protocol.DeviceID) []proto
 	}
 	return devices
 }
+
+func moveForConflict(name string) error {
+	ext := filepath.Ext(name)
+	withoutExt := name[:len(name)-len(ext)]
+	newName := withoutExt + time.Now().Format(".sync-conflict-20060102-150405") + ext
+	return os.Rename(name, newName)
+}

internal/model/rwfolder_test.go

@@ -67,7 +67,7 @@ func TestHandleFile(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocal("default", existingFile)
@@ -121,7 +121,7 @@ func TestHandleFileWithTemp(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocal("default", existingFile)
@@ -181,7 +181,7 @@ func TestCopierFinder(t *testing.T) {
 	requiredFile.Name = "file2"
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocal("default", existingFile)
@@ -256,7 +256,7 @@ func TestCopierCleanup(t *testing.T) {
 	}
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 
 	// Create a file
@@ -275,7 +275,7 @@ func TestCopierCleanup(t *testing.T) {
 	}
 
 	file.Blocks = []protocol.BlockInfo{blocks[1]}
-	file.Version++
+	file.Version = file.Version.Update(protocol.LocalDeviceID.Short())
 	// Update index (removing old blocks)
 	m.updateLocal("default", file)
 
@@ -288,7 +288,7 @@ func TestCopierCleanup(t *testing.T) {
 	}
 
 	file.Blocks = []protocol.BlockInfo{blocks[0]}
-	file.Version++
+	file.Version = file.Version.Update(protocol.LocalDeviceID.Short())
 	// Update index (removing old blocks)
 	m.updateLocal("default", file)
 
@@ -305,7 +305,7 @@ func TestCopierCleanup(t *testing.T) {
 // if it fails to find the block.
 func TestLastResortPulling(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 
 	// Add a file to index (with the incorrect block representation, as content
@@ -378,7 +378,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 
 	emitter := NewProgressEmitter(defaultConfig)
@@ -465,7 +465,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 
 	emitter := NewProgressEmitter(defaultConfig)

internal/model/sharedpullerstate.go

@@ -20,12 +20,13 @@ import (
 // updated along the way.
 type sharedPullerState struct {
 	// Immutable, does not require locking
-	file        protocol.FileInfo
+	file        protocol.FileInfo // The new file (desired end state)
 	folder      string
 	tempName    string
 	realName    string
 	reused      int // Number of blocks reused from temporary file
 	ignorePerms bool
+	version     protocol.Vector // The current (old) version
 
 	// Mutable, must be locked for access
 	err        error      // The first error we hit

internal/scanner/walk.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/internal/ignore"
-	"github.com/syncthing/syncthing/internal/lamport"
 	"github.com/syncthing/syncthing/internal/symlinks"
 	"golang.org/x/text/unicode/norm"
 )
@@ -40,8 +39,8 @@ func init() {
 type Walker struct {
 	// Dir is the base directory for the walk
 	Dir string
-	// Limit walking to this path within Dir, or no limit if Sub is blank
-	Sub string
+	// Limit walking to these paths within Dir, or no limit if Sub is empty
+	Subs []string
 	// BlockSize controls the size of the block used when hashing.
 	BlockSize int
 	// If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
@@ -61,6 +60,8 @@ type Walker struct {
 	AutoNormalize bool
 	// Number of routines to use for hashing
 	Hashers int
+	// Our vector clock id
+	ShortID uint64
 }
 
 type TempNamer interface {
@@ -79,7 +80,7 @@ type CurrentFiler interface {
 // file system. Files are blockwise hashed.
 func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 	if debug {
-		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.Matcher)
+		l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher)
 	}
 
 	err := checkDir(w.Dir)
@@ -98,7 +99,13 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 
 	go func() {
 		hashFiles := w.walkAndHashFiles(files)
-		filepath.Walk(filepath.Join(w.Dir, w.Sub), hashFiles)
+		if len(w.Subs) == 0 {
+			filepath.Walk(w.Dir, hashFiles)
+		} else {
+			for _, sub := range w.Subs {
+				filepath.Walk(filepath.Join(w.Dir, sub), hashFiles)
+			}
+		}
 		close(files)
 	}()
 
@@ -111,7 +118,8 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 		// Return value used when we are returning early and don't want to
 		// process the item. For directories, this means do-not-descend.
 		var skip error // nil
-		if info.IsDir() {
+		// info nil when error is not nil
+		if info != nil && info.IsDir() {
 			skip = filepath.SkipDir
 		}
 
@@ -202,6 +210,9 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 			rn = normalizedRn
 		}
 
+		var cf protocol.FileInfo
+		var ok bool
+
 		// Index wise symlinks are always files, regardless of what the target
 		// is, because symlinks carry their target path as their content.
 		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
@@ -242,7 +253,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 				//  - it wasn't invalid
 				//  - the symlink type (file/dir) was the same
 				//  - the block list (i.e. hash of target) was the same
-				cf, ok := w.CurrentFiler.CurrentFile(rn)
+				cf, ok = w.CurrentFiler.CurrentFile(rn)
 				if ok && !cf.IsDeleted() && cf.IsSymlink() && !cf.IsInvalid() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) {
 					return skip
 				}
@@ -250,7 +261,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 
 			f := protocol.FileInfo{
 				Name:     rn,
-				Version:  lamport.Default.Tick(0),
+				Version:  cf.Version.Update(w.ShortID),
 				Flags:    protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666,
 				Modified: 0,
 				Blocks:   blocks,
@@ -274,7 +285,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 				//  - was a directory previously (not a file or something else)
 				//  - was not a symlink (since it's a directory now)
 				//  - was not invalid (since it looks valid now)
-				cf, ok := w.CurrentFiler.CurrentFile(rn)
+				cf, ok = w.CurrentFiler.CurrentFile(rn)
 				permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode()))
 				if ok && permUnchanged && !cf.IsDeleted() && cf.IsDirectory() && !cf.IsSymlink() && !cf.IsInvalid() {
 					return nil
@@ -289,7 +300,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 			}
 			f := protocol.FileInfo{
 				Name:     rn,
-				Version:  lamport.Default.Tick(0),
+				Version:  cf.Version.Update(w.ShortID),
 				Flags:    flags,
 				Modified: info.ModTime().Unix(),
 			}
@@ -311,7 +322,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 				//  - was not a symlink (since it's a file now)
 				//  - was not invalid (since it looks valid now)
 				//  - has the same size as previously
-				cf, ok := w.CurrentFiler.CurrentFile(rn)
+				cf, ok = w.CurrentFiler.CurrentFile(rn)
 				permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode()))
 				if ok && permUnchanged && !cf.IsDeleted() && cf.Modified == info.ModTime().Unix() && !cf.IsDirectory() &&
 					!cf.IsSymlink() && !cf.IsInvalid() && cf.Size() == info.Size() {
@@ -330,7 +341,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 
 			f := protocol.FileInfo{
 				Name:     rn,
-				Version:  lamport.Default.Tick(0),
+				Version:  cf.Version.Update(w.ShortID),
 				Flags:    flags,
 				Modified: info.ModTime().Unix(),
 			}

internal/scanner/walk_test.go

@@ -60,7 +60,7 @@ func TestWalkSub(t *testing.T) {
 
 	w := Walker{
 		Dir:       "testdata",
-		Sub:       "dir2",
+		Subs:      []string{"dir2"},
 		BlockSize: 128 * 1024,
 		Matcher:   ignores,
 	}
@@ -259,6 +259,14 @@ func TestNormalization(t *testing.T) {
 	}
 }
 
+func TestIssue1507(t *testing.T) {
+	w := Walker{}
+	c := make(chan protocol.FileInfo, 100)
+	fn := w.walkAndHashFiles(c)
+
+	fn("", nil, protocol.ErrClosed)
+}
+
 func walkDir(dir string) ([]protocol.FileInfo, error) {
 	w := Walker{
 		Dir:           dir,

internal/upgrade/upgrade_supported.go

@@ -23,11 +23,12 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 )
 
 // Returns the latest release, including prereleases or not depending on the argument
-func LatestRelease(prerelease bool) (Release, error) {
+func LatestGithubRelease(version string) (Release, error) {
 	resp, err := http.Get("https://api.github.com/repos/syncthing/syncthing/releases?per_page=10")
 	if err != nil {
 		return Release{}, err
@@ -40,19 +41,47 @@ func LatestRelease(prerelease bool) (Release, error) {
 	json.NewDecoder(resp.Body).Decode(&rels)
 	resp.Body.Close()
 
+	return LatestRelease(version, rels)
+}
+
+type SortByRelease []Release
+
+func (s SortByRelease) Len() int {
+	return len(s)
+}
+func (s SortByRelease) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+func (s SortByRelease) Less(i, j int) bool {
+	return CompareVersions(s[i].Tag, s[j].Tag) > 0
+}
+
+func LatestRelease(version string, rels []Release) (Release, error) {
 	if len(rels) == 0 {
 		return Release{}, ErrVersionUnknown
 	}
 
-	if prerelease {
-		// We are a beta version. Use the latest.
-		return rels[0], nil
-	}
+	sort.Sort(SortByRelease(rels))
+	// Check for a beta build
+	beta := strings.Contains(version, "-beta")
 
-	// We are a regular release. Only consider non-prerelease versions for upgrade.
 	for _, rel := range rels {
-		if !rel.Prerelease {
-			return rel, nil
+		if rel.Prerelease && !beta {
+			continue
+		}
+		for _, asset := range rel.Assets {
+			assetName := path.Base(asset.Name)
+			// Check for the architecture
+			expectedRelease := releaseName(rel.Tag)
+			if debug {
+				l.Debugf("expected release asset %q", expectedRelease)
+			}
+			if debug {
+				l.Debugln("considering release", assetName)
+			}
+			if strings.HasPrefix(assetName, expectedRelease) {
+				return rel, nil
+			}
 		}
 	}
 	return Release{}, ErrVersionUnknown

internal/upgrade/upgrade_test.go

@@ -6,9 +6,13 @@
 
 package upgrade
 
-import "testing"
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
 
-var testcases = []struct {
+var versions = []struct {
 	a, b string
 	r    Relation
 }{
@@ -27,6 +31,7 @@ var testcases = []struct {
 	{"0.10.0", "0.2.0", MajorNewer},
 	{"30.10.0", "4.9.0", MajorNewer},
 	{"0.9.0-beta7", "0.9.0-beta6", Newer},
+	{"0.9.0-beta7", "1.0.0-alpha", MajorOlder},
 	{"1.0.0-alpha", "1.0.0-alpha.1", Older},
 	{"1.0.0-alpha.1", "1.0.0-alpha.beta", Older},
 	{"1.0.0-alpha.beta", "1.0.0-beta", Older},
@@ -44,9 +49,39 @@ var testcases = []struct {
 }
 
 func TestCompareVersions(t *testing.T) {
-	for _, tc := range testcases {
-		if r := CompareVersions(tc.a, tc.b); r != tc.r {
-			t.Errorf("compareVersions(%q, %q): %d != %d", tc.a, tc.b, r, tc.r)
+	for _, v := range versions {
+		if r := CompareVersions(v.a, v.b); r != v.r {
+			t.Errorf("compareVersions(%q, %q): %d != %d", v.a, v.b, r, v.r)
+		}
+	}
+}
+
+var upgrades = map[string]string{
+	"v0.10.21":                        "v0.10.30",
+	"v0.10.29":                        "v0.10.30",
+	"v0.10.31":                        "v0.10.30",
+	"v0.10.0-alpha":                   "v0.10.30",
+	"v0.10.0-beta":                    "v0.11.0-beta0",
+	"v0.11.0-beta0+40-g53cb66e-dirty": "v0.11.0-beta0",
+}
+
+func TestRelease(t *testing.T) {
+	fd, err := os.Open("testdata/github-releases.json")
+	if err != nil {
+		t.Errorf("Missing github-release test data")
+	}
+	defer fd.Close()
+
+	var rels []Release
+	json.NewDecoder(fd).Decode(&rels)
+
+	for old, target := range upgrades {
+		upgrade, err := LatestRelease(old, rels)
+		if err != nil {
+			t.Errorf("error retrieving latest version", err)
+		}
+		if upgrade.Tag != target {
+			t.Errorf("Invalid upgrade release: %v -> %v, but got %v", old, target, upgrade.Tag)
 		}
 	}
 }

test/.gitignore

@@ -15,5 +15,5 @@ dirs-*
 csrftokens.txt
 s4d
 http
-index
+h*/index*
 *.syncthing-reset*

test/cli_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func TestCLIReset(t *testing.T) {
-	dirs := []string{"s1", "s12-1", "h1/index"}
+	dirs := []string{"s1", "s12-1", "h1/index-v0.11.0.db"}
 
 	// Create directories that reset will remove
 

test/conflict_test.go

@@ -0,0 +1,242 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+package integration
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestConflict(t *testing.T) {
+	log.Println("Cleaning...")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Generating files...")
+	err = generateFiles("s1", 100, 20, "../LICENSE")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd, err := os.Create("s1/testfile.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fd.WriteString("hello\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = fd.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected, err := directoryContents("s1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Starting sender...")
+	sender := syncthingProcess{ // id1
+		instance: "1",
+		argv:     []string{"-home", "h1"},
+		port:     8081,
+		apiKey:   apiKey,
+	}
+	err = sender.start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer sender.stop()
+
+	// Wait for one scan to succeed, or up to 20 seconds... This is to let
+	// startup, UPnP etc complete and make sure the sender has the full index
+	// before they connect.
+	for i := 0; i < 20; i++ {
+		err := sender.rescan("default")
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
+	log.Println("Starting receiver...")
+	receiver := syncthingProcess{ // id2
+		instance: "2",
+		argv:     []string{"-home", "h2"},
+		port:     8082,
+		apiKey:   apiKey,
+	}
+	err = receiver.start()
+	if err != nil {
+		sender.stop()
+		t.Fatal(err)
+	}
+	defer receiver.stop()
+
+	if err = coCompletion(sender, receiver); err != nil {
+		t.Fatal(err)
+	}
+
+	sender.stop()
+	receiver.stop()
+
+	log.Println("Verifying...")
+
+	actual, err := directoryContents("s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = compareDirectoryContents(actual, expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Introducing a conflict (simultaneous edit)...")
+
+	fd, err = os.OpenFile("s1/testfile.txt", os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fd.WriteString("text added to s1\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = fd.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd, err = os.OpenFile("s2/testfile.txt", os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fd.WriteString("text added to s2\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = fd.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Syncing...")
+
+	err = receiver.start()
+	err = sender.start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err != nil {
+		sender.stop()
+		t.Fatal(err)
+	}
+
+	if err = coCompletion(sender, receiver); err != nil {
+		t.Fatal(err)
+	}
+
+	sender.stop()
+	receiver.stop()
+
+	// The conflict is expected on the s2 side due to how we calculate which
+	// file is the winner (based on device ID)
+
+	files, err := filepath.Glob("s2/*sync-conflict*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != 1 {
+		t.Errorf("Expected 1 conflicted files instead of %d", len(files))
+	}
+
+	log.Println("Introducing a conflict (edit plus delete)...")
+
+	err = os.Remove("s1/testfile.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd, err = os.OpenFile("s2/testfile.txt", os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fd.WriteString("more text added to s2\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = fd.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Syncing...")
+
+	err = receiver.start()
+	err = sender.start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err != nil {
+		sender.stop()
+		t.Fatal(err)
+	}
+
+	if err = coCompletion(sender, receiver); err != nil {
+		t.Fatal(err)
+	}
+
+	sender.stop()
+	receiver.stop()
+
+	// The conflict should manifest on the s2 side again, where we should have
+	// moved the file to a conflict copy instead of just deleting it.
+
+	files, err = filepath.Glob("s2/*sync-conflict*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != 2 {
+		t.Errorf("Expected 2 conflicted files instead of %d", len(files))
+	}
+}
+
+func coCompletion(p ...syncthingProcess) error {
+mainLoop:
+	for {
+		time.Sleep(2500 * time.Millisecond)
+
+		tot := 0
+		for i := range p {
+			comp, err := p[i].peerCompletion()
+			if err != nil {
+				if isTimeout(err) {
+					continue mainLoop
+				}
+				return err
+			}
+
+			for _, pct := range comp {
+				tot += pct
+			}
+		}
+
+		if tot == 100*(len(p)) {
+			return nil
+		}
+
+		log.Printf("%d / %d...", tot, 100*(len(p)))
+	}
+}

test/filetype_test.go

@@ -61,7 +61,7 @@ func TestFileTypeChangeStaggeredVersioning(t *testing.T) {
 
 func testFileTypeChange(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/h2/config.xml

@@ -3,7 +3,9 @@
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
-        <versioning type="staggered"></versioning>
+        <versioning type="simple">
+            <param key="keep" val="5"></param>
+        </versioning>
         <lenientMtimes>false</lenientMtimes>
         <copiers>1</copiers>
         <pullers>16</pullers>

test/http_test.go

@@ -78,6 +78,7 @@ func TestGetIndexAuth(t *testing.T) {
 		argv:     []string{"-home", "h1"},
 		port:     8081,
 		instance: "1",
+		apiKey:   "abc123",
 	}
 	err := st.start()
 	if err != nil {

test/httpstress_test.go

@@ -23,7 +23,7 @@ import (
 
 func TestStressHTTP(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s2", "h2/index")
+	err := removeAll("s2", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/ignore_test.go

@@ -22,7 +22,7 @@ func TestIgnores(t *testing.T) {
 	// Clean and start a syncthing instance
 
 	log.Println("Cleaning...")
-	err := removeAll("s1", "h1/index")
+	err := removeAll("s1", "h1/index*")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -37,6 +37,19 @@ func TestIgnores(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	// Wait for one scan to succeed, or up to 20 seconds... This is to let
+	// startup, UPnP etc complete and make sure that we've performed folder
+	// error checking which creates the folder path if it's missing.
+	for i := 0; i < 20; i++ {
+		err := p.rescan("default")
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
 	defer p.stop()
 
 	// Create eight empty files and directories
@@ -75,16 +88,11 @@ func TestIgnores(t *testing.T) {
 	// Wait for one scan to succeed, or up to 20 seconds...
 	// This is to let startup, UPnP etc complete.
 	for i := 0; i < 20; i++ {
-		resp, err := p.post("/rest/scan?folder=default", nil)
+		err := p.rescan("default")
 		if err != nil {
 			time.Sleep(time.Second)
 			continue
 		}
-		if resp.StatusCode != 200 {
-			resp.Body.Close()
-			time.Sleep(time.Second)
-			continue
-		}
 		break
 	}
 
@@ -114,7 +122,7 @@ func TestIgnores(t *testing.T) {
 
 	// Rescan and verify that we see them
 
-	p.post("/rest/scan?folder=default", nil)
+	p.rescan("default")
 	m, err = p.model("default")
 	if err != nil {
 		t.Fatal(err)
@@ -141,7 +149,7 @@ func TestIgnores(t *testing.T) {
 
 	// Rescan and verify that we see them
 
-	p.post("/rest/scan?folder=default", nil)
+	p.rescan("default")
 	m, err = p.model("default")
 	if err != nil {
 		t.Fatal(err)

test/manypeers_test.go

@@ -22,7 +22,7 @@ import (
 
 func TestManyPeers(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/parallell_scan_test.go

@@ -18,7 +18,7 @@ import (
 
 func TestParallellScan(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "h1/index")
+	err := removeAll("s1", "h1/index*")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -50,16 +50,11 @@ func TestParallellScan(t *testing.T) {
 	// Wait for one scan to succeed, or up to 20 seconds...
 	// This is to let startup, UPnP etc complete.
 	for i := 0; i < 20; i++ {
-		resp, err := st.post("/rest/scan?folder=default", nil)
+		err := st.rescan("default")
 		if err != nil {
 			time.Sleep(time.Second)
 			continue
 		}
-		if resp.StatusCode != 200 {
-			resp.Body.Close()
-			time.Sleep(time.Second)
-			continue
-		}
 		break
 	}
 
@@ -73,16 +68,12 @@ func TestParallellScan(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			resp, err := st.post("/rest/scan?folder=default", nil)
+			err := st.rescan("default")
 			log.Println(j)
 			if err != nil {
 				log.Println(err)
 				t.Fatal(err)
 			}
-			if resp.StatusCode != 200 {
-				t.Fatalf("%d != 200", resp.StatusCode)
-			}
-			resp.Body.Close()
 		}()
 	}
 

test/reconnect_test.go

@@ -32,13 +32,13 @@ func TestRestartSenderAndReceiverDuringTransfer(t *testing.T) {
 
 func testRestartDuringTransfer(t *testing.T, restartSender, restartReceiver bool, senderDelay, receiverDelay time.Duration) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	log.Println("Generating files...")
-	err = generateFiles("s1", 1000, 22, "../LICENSE")
+	err = generateFiles("s1", 250, 20, "../LICENSE")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -138,7 +138,7 @@ func testRestartDuringTransfer(t *testing.T, restartSender, restartReceiver bool
 			prevComp = curComp
 		}
 
-		time.Sleep(250 * time.Millisecond)
+		time.Sleep(time.Second)
 	}
 
 	err = sender.stop()

test/symlink_test.go

@@ -86,7 +86,7 @@ func TestSymlinksStaggeredVersioning(t *testing.T) {
 
 func testSymlinks(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/sync_test.go

@@ -61,21 +61,25 @@ func TestSyncClusterStaggeredVersioning(t *testing.T) {
 }
 
 func testSyncCluster(t *testing.T) {
-	/*
+	// This tests syncing files back and forth between three cluster members.
+	// Their configs are in h1, h2 and h3. The folder "default" is shared
+	// between all and stored in s1, s2 and s3 respectively.
+	//
+	// Another folder is shared between 1 and 2 only, in s12-1 and s12-2. A
+	// third folders is shared between 2 and 3, in s23-2 and s23-3.
 
-		This tests syncing files back and forth between three cluster members.
-		Their configs are in h1, h2 and h3. The folder "default" is shared
-		between all and stored in s1, s2 and s3 respectively.
+	const (
+		numFiles    = 100
+		fileSizeExp = 20
+		iterations  = 3
+	)
+	log.Printf("Testing with numFiles=%d, fileSizeExp=%d, iterations=%d", numFiles, fileSizeExp, iterations)
 
-		Another folder is shared between 1 and 2 only, in s12-1 and s12-2. A
-		third folders is shared between 2 and 3, in s23-2 and s23-3.
-
-	*/
 	log.Println("Cleaning...")
 	err := removeAll("s1", "s12-1",
 		"s2", "s12-2", "s23-2",
 		"s3", "s23-3",
-		"h1/index", "h2/index", "h3/index")
+		"h1/index*", "h2/index*", "h3/index*")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,11 +90,11 @@ func testSyncCluster(t *testing.T) {
 
 	log.Println("Generating files...")
 
-	err = generateFiles("s1", 1000, 21, "../LICENSE")
+	err = generateFiles("s1", numFiles, fileSizeExp, "../LICENSE")
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = generateFiles("s12-1", 1000, 21, "../LICENSE")
+	err = generateFiles("s12-1", numFiles, fileSizeExp, "../LICENSE")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -109,16 +113,16 @@ func testSyncCluster(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = generateFiles("s2", 1000, 21, "../LICENSE")
+	err = generateFiles("s2", numFiles, fileSizeExp, "../LICENSE")
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = generateFiles("s23-2", 1000, 21, "../LICENSE")
+	err = generateFiles("s23-2", numFiles, fileSizeExp, "../LICENSE")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	err = generateFiles("s3", 1000, 21, "../LICENSE")
+	err = generateFiles("s3", numFiles, fileSizeExp, "../LICENSE")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -158,17 +162,37 @@ func testSyncCluster(t *testing.T) {
 		}
 	}()
 
-	for count := 0; count < 5; count++ {
+	log.Println("Waiting for startup...")
+	// Wait for one scan to succeed, or up to 20 seconds...
+	// This is to let startup, UPnP etc complete.
+	for _, device := range p {
+		for i := 0; i < 20; i++ {
+			err := device.rescan("default")
+			if err != nil {
+				time.Sleep(time.Second)
+				continue
+			}
+			break
+		}
+	}
+
+	for count := 0; count < iterations; count++ {
 		log.Println("Forcing rescan...")
 
 		// Force rescan of folders
-		for i := range p {
-			p[i].post("/rest/scan?folder=default", nil)
-			if i < 3 {
-				p[i].post("/rest/scan?folder=s12", nil)
+		for i, device := range p {
+			if err := device.rescan("default"); err != nil {
+				t.Fatal(err)
+			}
+			if i < 2 {
+				if err := device.rescan("s12"); err != nil {
+					t.Fatal(err)
+				}
 			}
 			if i > 1 {
-				p[i].post("/rest/scan?folder=s23", nil)
+				if err := device.rescan("s23"); err != nil {
+					t.Fatal(err)
+				}
 			}
 		}
 
@@ -284,38 +308,23 @@ func scStartProcesses() ([]syncthingProcess, error) {
 }
 
 func scSyncAndCompare(p []syncthingProcess, expected [][]fileInfo) error {
-	ids := []string{id1, id2, id3}
-
 	log.Println("Syncing...")
 
-mainLoop:
 	for {
-		time.Sleep(2500 * time.Millisecond)
+		time.Sleep(2 * time.Second)
 
-		for i := range p {
-			comp, err := p[i].peerCompletion()
-			if err != nil {
-				if isTimeout(err) {
-					continue mainLoop
-				}
-				return err
-			}
-
-			for id, pct := range comp {
-				if id == ids[i] {
-					// Don't check for self, which will be 0%
-					continue
-				}
-				if pct != 100 {
-					log.Printf("%s not done yet: %d%%", id, pct)
-					continue mainLoop
-				}
-			}
+		if err := allDevicesInSync(p); err != nil {
+			log.Println(err)
+			continue
 		}
 
 		break
 	}
 
+	// This is necessary, or all files won't be in place even when everything
+	// is already reported in sync. Why?!
+	time.Sleep(5 * time.Second)
+
 	log.Println("Checking...")
 
 	for _, dir := range []string{"s1", "s2", "s3"} {

test/syncthingprocess.go

@@ -15,10 +15,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
 	"os/exec"
 	"time"
+
+	"github.com/syncthing/protocol"
 )
 
 var env = []string{
@@ -34,6 +38,7 @@ type syncthingProcess struct {
 	apiKey    string
 	csrfToken string
 	lastEvent int
+	id        protocol.DeviceID
 
 	cmd   *exec.Cmd
 	logfd *os.File
@@ -72,12 +77,32 @@ func (p *syncthingProcess) start() error {
 	p.cmd = cmd
 
 	for {
-		resp, err := p.get("/")
-		if err == nil {
-			resp.Body.Close()
-			return nil
-		}
 		time.Sleep(250 * time.Millisecond)
+
+		resp, err := p.get("/rest/system")
+		if err != nil {
+			continue
+		}
+
+		var sysData map[string]interface{}
+		err = json.NewDecoder(resp.Body).Decode(&sysData)
+		resp.Body.Close()
+		if err != nil {
+			// This one is unexpected. Print it.
+			log.Println("/rest/system (JSON):", err)
+			continue
+		}
+
+		id, err := protocol.DeviceIDFromString(sysData["myID"].(string))
+		if err != nil {
+			// This one is unexpected. Print it.
+			log.Println("/rest/system (myID):", err)
+			continue
+		}
+
+		p.id = id
+
+		return nil
 	}
 }
 
@@ -173,9 +198,31 @@ func (p *syncthingProcess) peerCompletion() (map[string]int, error) {
 
 	comp := map[string]int{}
 	err = json.NewDecoder(resp.Body).Decode(&comp)
+
+	// Remove ourselves from the set. In the remaining map, all peers should
+	// be att 100% if we're in sync.
+	for id := range comp {
+		if id == p.id.String() {
+			delete(comp, id)
+		}
+	}
+
 	return comp, err
 }
 
+func (p *syncthingProcess) allPeersInSync() error {
+	comp, err := p.peerCompletion()
+	if err != nil {
+		return err
+	}
+	for id, val := range comp {
+		if val != 100 {
+			return fmt.Errorf("%.7s at %d%%", id, val)
+		}
+	}
+	return nil
+}
+
 type model struct {
 	GlobalBytes   int
 	GlobalDeleted int
@@ -249,3 +296,25 @@ func (p *syncthingProcess) version() (string, error) {
 	}
 	return v.Version, nil
 }
+
+func (p *syncthingProcess) rescan(folder string) error {
+	resp, err := p.post("/rest/scan?folder="+folder, nil)
+	if err != nil {
+		return err
+	}
+	data, _ := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("Rescan %q: status code %d: %s", folder, resp.StatusCode, data)
+	}
+	return nil
+}
+
+func allDevicesInSync(p []syncthingProcess) error {
+	for _, device := range p {
+		if err := device.allPeersInSync(); err != nil {
+			return fmt.Errorf("%.7s: %v", device.id.String(), err)
+		}
+	}
+	return nil
+}

test/transfer-bench_test.go

@@ -16,7 +16,7 @@ import (
 
 func TestBenchmarkTransfer(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/util.go

@@ -241,17 +241,23 @@ func (i *inifiteReader) Read(bs []byte) (int, error) {
 // rm -rf
 func removeAll(dirs ...string) error {
 	for _, dir := range dirs {
-		// Set any non-writeable files and dirs to writeable. This is necessary for os.RemoveAll to work on Windows.
-		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.Mode()&0700 != 0700 {
-				os.Chmod(path, 0777)
-			}
-			return nil
-		})
-		os.RemoveAll(dir)
+		files, err := filepath.Glob(dir)
+		if err != nil {
+			return err
+		}
+		for _, file := range files {
+			// Set any non-writeable files and dirs to writeable. This is necessary for os.RemoveAll to work on Windows.
+			filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.Mode()&0700 != 0700 {
+					os.Chmod(path, 0777)
+				}
+				return nil
+			})
+			os.RemoveAll(file)
+		}
 	}
 	return nil
 }