lib/protocol, lib/discover, lib/db: Use protocol buffer serialization (fixes #3080)

This changes the BEP protocol to use protocol buffer serialization
instead of XDR, and therefore also the database format. The local
discovery protocol is also updated to be protocol buffer format.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3276
LGTM: AudriusButkevicius

build.go

@@ -238,8 +238,8 @@ func runCommand(cmd string, target target) {
 	case "assets":
 		rebuildAssets()
 
-	case "xdr":
-		xdr()
+	case "proto":
+		proto()
 
 	case "translate":
 		translate()
@@ -271,9 +271,12 @@ func runCommand(cmd string, target target) {
 	case "metalint":
 		if isGometalinterInstalled() {
 			dirs := []string{".", "./cmd/...", "./lib/..."}
-			gometalinter("deadcode", dirs, "test/util.go")
-			gometalinter("structcheck", dirs)
-			gometalinter("varcheck", dirs)
+			ok := gometalinter("deadcode", dirs, "test/util.go")
+			ok = gometalinter("structcheck", dirs) && ok
+			ok = gometalinter("varcheck", dirs) && ok
+			if !ok {
+				os.Exit(1)
+			}
 		}
 
 	default:
@@ -576,8 +579,8 @@ func shouldRebuildAssets() bool {
 	return assetsAreNewer
 }
 
-func xdr() {
-	runPrint("go", "generate", "./lib/discover", "./lib/db", "./lib/protocol", "./lib/relay/protocol")
+func proto() {
+	runPrint("go", "generate", "./lib/...")
 }
 
 func translate() {
@@ -717,10 +720,18 @@ func getBranchSuffix() string {
 }
 
 func buildStamp() int64 {
+	// If SOURCE_DATE_EPOCH is set, use that.
+	if s, _ := strconv.ParseInt(os.Getenv("SOURCE_DATE_EPOCH"), 10, 64); s > 0 {
+		return s
+	}
+
+	// Try to get the timestamp of the latest commit.
 	bs, err := runError("git", "show", "-s", "--format=%ct")
 	if err != nil {
+		// Fall back to "now".
 		return time.Now().Unix()
 	}
+
 	s, _ := strconv.ParseInt(string(bs), 10, 64)
 	return s
 }
@@ -948,13 +959,17 @@ func lint(pkg string) {
 	}
 
 	analCommentPolicy := regexp.MustCompile(`exported (function|method|const|type|var) [^\s]+ should have comment`)
-	for _, line := range bytes.Split(bs, []byte("\n")) {
-		if analCommentPolicy.Match(line) {
+	for _, line := range strings.Split(string(bs), "\n") {
+		if line == "" {
 			continue
 		}
-		if len(line) > 0 {
-			log.Printf("%s", line)
+		if analCommentPolicy.MatchString(line) {
+			continue
 		}
+		if strings.Contains(line, ".pb.go:") {
+			continue
+		}
+		log.Println(line)
 	}
 }
 
@@ -995,7 +1010,7 @@ func isGometalinterInstalled() bool {
 	return true
 }
 
-func gometalinter(linter string, dirs []string, excludes ...string) {
+func gometalinter(linter string, dirs []string, excludes ...string) bool {
 	params := []string{"--disable-all"}
 	params = append(params, fmt.Sprintf("--deadline=%ds", 60))
 	params = append(params, "--enable="+linter)
@@ -1008,12 +1023,19 @@ func gometalinter(linter string, dirs []string, excludes ...string) {
 		params = append(params, dir)
 	}
 
-	bs, err := runError("gometalinter", params...)
+	bs, _ := runError("gometalinter", params...)
 
-	if len(bs) > 0 {
-		log.Printf("%s", bs)
-	}
-	if err != nil {
-		log.Printf("%v", err)
+	nerr := 0
+	for _, line := range strings.Split(string(bs), "\n") {
+		if line == "" {
+			continue
+		}
+		if strings.Contains(line, ".pb.go:") {
+			continue
+		}
+		log.Println(line)
+		nerr++
 	}
+
+	return nerr == 0
 }

cmd/discosrv/querysrv.go

@@ -330,6 +330,16 @@ func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID p
 
 		ip := net.ParseIP(host)
 		if host == "" || ip.IsUnspecified() {
+			// Do not use IPv6 remote address if requested scheme is tcp4
+			if uri.Scheme == "tcp4" && remote.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 remote address if requested scheme is tcp6
+			if uri.Scheme == "tcp6" && remote.To4() != nil {
+				continue
+			}
+
 			host = remote.String()
 		}
 

cmd/relaysrv/main.go

@@ -88,7 +88,7 @@ func main() {
 	flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
 	flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
 	flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
-	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertising as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
+	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
 
 	flag.Parse()
 
@@ -111,7 +111,7 @@ func main() {
 
 		go monitorLimits()
 	} else if err != nil && runtime.GOOS != "windows" {
-		log.Println("Assuming no connection limit, due to error retrievign rlimits:", err)
+		log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
 	}
 
 	sessionAddress = addr.IP[:]

cmd/stdisco/main.go

@@ -9,6 +9,7 @@ package main
 import (
 	"bytes"
 	"crypto/rand"
+	"encoding/binary"
 	"flag"
 	"log"
 	"strings"
@@ -66,24 +67,25 @@ func recv(bc beacon.Interface) {
 	seen := make(map[string]bool)
 	for {
 		data, src := bc.Recv()
-		var ann discover.Announce
-		ann.UnmarshalXDR(data)
+		if m := binary.BigEndian.Uint32(data); m != discover.Magic {
+			log.Printf("Incorrect magic %x in announcement from %v", m, src)
+			continue
+		}
 
-		if bytes.Equal(ann.This.ID, myID) {
+		var ann discover.Announce
+		ann.Unmarshal(data[4:])
+
+		if bytes.Equal(ann.ID, myID) {
 			// This is one of our own fake packets, don't print it.
 			continue
 		}
 
 		// Print announcement details for the first packet from a given
 		// device ID and source address, or if -all was given.
-		key := string(ann.This.ID) + src.String()
+		key := string(ann.ID) + src.String()
 		if all || !seen[key] {
 			log.Printf("Announcement from %v\n", src)
-			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.This.ID), strings.Join(addrStrs(ann.This), ", "))
-
-			for _, dev := range ann.Extra {
-				log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(dev.ID), strings.Join(addrStrs(dev), ", "))
-			}
+			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.ID), strings.Join(ann.Addresses, ", "))
 			seen[key] = true
 		}
 	}
@@ -92,15 +94,10 @@ func recv(bc beacon.Interface) {
 // sends fake discovery announcements once every second
 func send(bc beacon.Interface) {
 	ann := discover.Announce{
-		Magic: discover.AnnouncementMagic,
-		This: discover.Device{
-			ID: myID,
-			Addresses: []discover.Address{
-				{URL: "tcp://fake.example.com:12345"},
-			},
-		},
+		ID:        myID,
+		Addresses: []string{"tcp://fake.example.com:12345"},
 	}
-	bs, _ := ann.MarshalXDR()
+	bs, _ := ann.Marshal()
 
 	for {
 		bc.Send(bs)
@@ -108,15 +105,6 @@ func send(bc beacon.Interface) {
 	}
 }
 
-// returns the list of address URLs
-func addrStrs(dev discover.Device) []string {
-	ss := make([]string, len(dev.Addresses))
-	for i, addr := range dev.Addresses {
-		ss[i] = addr.URL
-	}
-	return ss
-}
-
 // returns a random but recognizable device ID
 func randomDeviceID() []byte {
 	var id [32]byte

cmd/stindex/dump.go

@@ -13,50 +13,61 @@ import (
 
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
-func dump(ldb *leveldb.DB) {
+func dump(ldb *db.Instance) {
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
 
 			var f protocol.FileInfo
-			err := f.UnmarshalXDR(it.Value())
+			err := f.Unmarshal(it.Value())
 			if err != nil {
 				log.Fatal(err)
 			}
-			fmt.Printf("  N:%q\n  F:%#o\n  M:%d\n  V:%v\n  S:%d\n  B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks))
+			fmt.Printf(" V:%v\n", f)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
 			var flv db.VersionList
-			flv.UnmarshalXDR(it.Value())
-			fmt.Printf("[global] F:%q N:%q V: %s\n", folder, name, flv)
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
 
 		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeVirtualMtime:
-			fmt.Printf("[mtime]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[mtime] K:%x V:%x\n", it.Key(), it.Value())
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			val := it.Value()
+			if len(val) == 0 {
+				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			} else {
+				dev := protocol.DeviceIDFromBytes(val)
+				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
+			}
 
 		default:
 			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())

cmd/stindex/dumpsize.go

@@ -8,11 +8,10 @@ package main
 
 import (
 	"container/heap"
+	"encoding/binary"
 	"fmt"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type SizedElement struct {
@@ -38,33 +37,31 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *leveldb.DB) {
+func dumpsize(ldb *db.Instance) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			ele.key = fmt.Sprintf("DEVICE:%s:%s:%s", dev, folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
-			ele.key = fmt.Sprintf("GLOBAL:%s:%s", folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			ele.key = fmt.Sprintf("BLOCK:%s:%x:%s", folder, hash, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
 
 		case db.KeyTypeDeviceStatistic:
 			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
@@ -75,6 +72,14 @@ func dumpsize(ldb *leveldb.DB) {
 		case db.KeyTypeVirtualMtime:
 			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
 
+		case db.KeyTypeFolderIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
+
+		case db.KeyTypeDeviceIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
+
 		default:
 			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
 		}

cmd/stindex/main.go

@@ -13,8 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syncthing/syncthing/lib/db"
 )
 
 func main() {
@@ -28,16 +27,12 @@ func main() {
 
 	path := flag.Arg(0)
 	if path == "" {
-		path = filepath.Join(defaultConfigDir(), "index-v0.11.0.db")
+		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
 	fmt.Println("Path:", path)
 
-	ldb, err := leveldb.OpenFile(path, &opt.Options{
-		ErrorIfMissing:         true,
-		Strict:                 opt.StrictAll,
-		OpenFilesCacheCapacity: 100,
-	})
+	ldb, err := db.Open(path)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/syncthing/gui.go

@@ -86,7 +86,7 @@ type modelIntf interface {
 	DelayScan(folder string, next time.Duration)
 	ScanFolder(folder string) error
 	ScanFolders() map[string]error
-	ScanFolderSubs(folder string, subs []string) error
+	ScanFolderSubdirs(folder string, subs []string) error
 	BringToFront(folder, file string)
 	ConnectedTo(deviceID protocol.DeviceID) bool
 	GlobalSize(folder string) (nfiles, deleted int, bytes int64)
@@ -577,7 +577,7 @@ func (s *apiService) getDBStatus(w http.ResponseWriter, r *http.Request) {
 func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interface{} {
 	var res = make(map[string]interface{})
 
-	res["invalid"] = cfg.Folders()[folder].Invalid
+	res["invalid"] = "" // Deprecated, retains external API for now
 
 	globalFiles, globalDeleted, globalBytes := m.GlobalSize(folder)
 	res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
@@ -690,7 +690,7 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 	to, err := config.ReadJSON(r.Body, myID)
 	r.Body.Close()
 	if err != nil {
-		l.Warnln("decoding posted config:", err)
+		l.Warnln("Decoding posted config:", err)
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
@@ -1071,7 +1071,7 @@ func (s *apiService) postDBScan(w http.ResponseWriter, r *http.Request) {
 		}
 
 		subs := qs["sub"]
-		err = s.model.ScanFolderSubs(folder, subs)
+		err = s.model.ScanFolderSubdirs(folder, subs)
 		if err != nil {
 			http.Error(w, err.Error(), 500)
 			return
@@ -1173,13 +1173,17 @@ type jsonFileInfo protocol.FileInfo
 
 func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
 	return json.Marshal(map[string]interface{}{
-		"name":         f.Name,
-		"size":         protocol.FileInfo(f).Size(),
-		"flags":        fmt.Sprintf("%#o", f.Flags),
-		"modified":     time.Unix(f.Modified, 0),
-		"localVersion": f.LocalVersion,
-		"numBlocks":    len(f.Blocks),
-		"version":      jsonVersionVector(f.Version),
+		"name":          f.Name,
+		"type":          f.Type,
+		"size":          f.Size,
+		"permissions":   fmt.Sprintf("%#o", f.Permissions),
+		"deleted":       f.Deleted,
+		"invalid":       f.Invalid,
+		"noPermissions": f.NoPermissions,
+		"modified":      time.Unix(f.Modified, 0),
+		"localVersion":  f.LocalVersion,
+		"numBlocks":     len(f.Blocks),
+		"version":       jsonVersionVector(f.Version),
 	})
 }
 
@@ -1187,20 +1191,23 @@ type jsonDBFileInfo db.FileInfoTruncated
 
 func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
 	return json.Marshal(map[string]interface{}{
-		"name":         f.Name,
-		"size":         db.FileInfoTruncated(f).Size(),
-		"flags":        fmt.Sprintf("%#o", f.Flags),
-		"modified":     time.Unix(f.Modified, 0),
-		"localVersion": f.LocalVersion,
-		"version":      jsonVersionVector(f.Version),
+		"name":          f.Name,
+		"type":          f.Type,
+		"size":          f.Size,
+		"permissions":   fmt.Sprintf("%#o", f.Permissions),
+		"deleted":       f.Deleted,
+		"invalid":       f.Invalid,
+		"noPermissions": f.NoPermissions,
+		"modified":      time.Unix(f.Modified, 0),
+		"localVersion":  f.LocalVersion,
 	})
 }
 
 type jsonVersionVector protocol.Vector
 
 func (v jsonVersionVector) MarshalJSON() ([]byte, error) {
-	res := make([]string, len(v))
-	for i, c := range v {
+	res := make([]string, len(v.Counters))
+	for i, c := range v.Counters {
 		res[i] = fmt.Sprintf("%v:%d", c.ID, c.Value)
 	}
 	return json.Marshal(res)

cmd/syncthing/gui_test.go

@@ -11,6 +11,7 @@ import (
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -613,3 +614,55 @@ func TestRandomString(t *testing.T) {
 		t.Errorf("Expected 27 random characters, got %q of length %d", res["random"], len(res["random"]))
 	}
 }
+
+func TestConfigPostOK(t *testing.T) {
+	cfg := bytes.NewBuffer([]byte(`{
+		"version": 15,
+		"folders": [
+			{"id": "foo"}
+		]
+	}`))
+
+	resp, err := testConfigPost(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Error("Expected 200 OK, not", resp.Status)
+	}
+}
+
+func TestConfigPostDupFolder(t *testing.T) {
+	cfg := bytes.NewBuffer([]byte(`{
+		"version": 15,
+		"folders": [
+			{"id": "foo"},
+			{"id": "foo"}
+		]
+	}`))
+
+	resp, err := testConfigPost(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Error("Expected 400 Bad Request, not", resp.Status)
+	}
+}
+
+func testConfigPost(data io.Reader) (*http.Response, error) {
+	const testAPIKey = "foobarbaz"
+	cfg := new(mockedConfig)
+	cfg.gui.APIKey = testAPIKey
+	baseURL, err := startHTTP(cfg)
+	if err != nil {
+		return nil, err
+	}
+	cli := &http.Client{
+		Timeout: time.Second,
+	}
+
+	req, _ := http.NewRequest("POST", baseURL+"/rest/system/config", data)
+	req.Header.Set("X-API-Key", testAPIKey)
+	return cli.Do(req)
+}

cmd/syncthing/locations.go

@@ -48,7 +48,7 @@ var locations = map[locationEnum]string{
 	locKeyFile:       "${config}/key.pem",
 	locHTTPSCertFile: "${config}/https-cert.pem",
 	locHTTPSKeyFile:  "${config}/https-key.pem",
-	locDatabase:      "${config}/index-v0.13.0.db",
+	locDatabase:      "${config}/index-v0.14.0.db",
 	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
 	locCsrfTokens:    "${config}/csrftokens.txt",
 	locPanicLog:      "${config}/panic-${timestamp}.log",

cmd/syncthing/main.go

@@ -539,7 +539,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	errors := logger.NewRecorder(l, logger.LevelWarn, maxSystemErrors, 0)
 	systemLog := logger.NewRecorder(l, logger.LevelDebug, maxSystemLog, initialSystemLog)
 
-	// Event subscription for the API; must start early to catch the early events.  The LocalDiskUpdated
+	// Event subscription for the API; must start early to catch the early events.  The LocalChangeDetected
 	// event might overwhelm the event reciever in some situations so we will not subscribe to it here.
 	apiSub := events.NewBufferedSubscription(events.Default.Subscribe(events.AllEvents&^events.LocalChangeDetected), 1000)
 
@@ -690,7 +690,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 			if device == myID {
 				continue
 			}
-			m.Index(device, folderCfg.ID, nil, 0, nil)
+			m.Index(device, folderCfg.ID, nil)
 		}
 		m.StartFolder(folderCfg.ID)
 	}
@@ -863,7 +863,6 @@ func loadConfig() (*config.Wrapper, error) {
 	cfg, err := config.Load(cfgFile, myID)
 
 	if err != nil {
-		l.Infoln("Error loading config file; using defaults for now")
 		myName, _ := os.Hostname()
 		newCfg := defaultConfig(myName)
 		cfg = config.Wrap(cfgFile, newCfg)

cmd/syncthing/mocked_model_test.go

@@ -85,7 +85,7 @@ func (m *mockedModel) ScanFolders() map[string]error {
 	return nil
 }
 
-func (m *mockedModel) ScanFolderSubs(folder string, subs []string) error {
+func (m *mockedModel) ScanFolderSubdirs(folder string, subs []string) error {
 	return nil
 }
 

gui/default/assets/lang/lang-sv.json

@@ -1,6 +1,6 @@
 {
     "A device with that ID is already added.": "En enhet med det ID är redan tillagt.",
-    "A negative number of days doesn't make sense.": "Negativt antal dagar är inte troligt.",
+    "A negative number of days doesn't make sense.": "Ett negativt antal dagar är inte troligt.",
     "A new major version may not be compatible with previous versions.": "En ny huvudversion kan eventuellt vara inkompatibel med tidigare versioner.",
     "API Key": "API-nyckel",
     "About": "Om",
@@ -16,16 +16,16 @@
     "Advanced Configuration": "Avancerad konfiguration",
     "Advanced settings": "Avancerade inställningar",
     "All Data": "All data",
-    "Allow Anonymous Usage Reporting?": "Tillåt anonym användarstatistik?",
+    "Allow Anonymous Usage Reporting?": "Tillåt anonym användarstatistiksrapportering?",
     "Alphabetic": "Alfabetisk",
     "An external command handles the versioning. It has to remove the file from the synced folder.": "Ett externt kommando sköter versionshanteringen. Det måste ta bort filen från den synkroniserade mappen.",
     "Anonymous Usage Reporting": "Anonym användarstatistik",
     "Any devices configured on an introducer device will be added to this device as well.": "Enheter konfigurerade på en introduktörsenhet kommer också att läggas till den här enheten.",
-    "Automatic upgrades": "Automatisk uppgradering",
+    "Automatic upgrades": "Automatiska uppgraderingar",
     "Be careful!": "Var aktsam!",
     "Bugs": "Buggar",
     "CPU Utilization": "CPU-användning",
-    "Changelog": "Changelog",
+    "Changelog": "Ändringslogg",
     "Clean out after": "Rensa efteråt",
     "Close": "Stäng",
     "Command": "Kommando",
@@ -38,7 +38,7 @@
     "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 följande bidragande:",
     "Copyright © 2015 the following Contributors:": "Copyright © 2015 följande medverkande:",
     "Danger!": "Fara!",
-    "Delete": "Radera",
+    "Delete": "Ta bort",
     "Deleted": "Borttaget",
     "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "Enhet \"{{name}}\" ({{device}} på {{address}}) vill ansluta. Lägg till ny enhet?",
     "Device ID": "Enhets-ID",
@@ -107,7 +107,7 @@
     "Local State (Total)": "Lokal status (Total)",
     "Major Upgrade": "Stor uppgradering",
     "Master": "Huvud",
-    "Maximum Age": "Högsta åldersgräns",
+    "Maximum Age": "Högsta ålder",
     "Metadata Only": "Endast metadata",
     "Minimum Free Disk Space": "Minimum ledigt diskutrymme",
     "Move to top of queue": "Flytta till överst i kön",
@@ -125,8 +125,8 @@
     "Oldest First": "Äldst först",
     "Optional descriptive label for the folder. Can be different on each device.": "Valfri beskrivande etikett för katalogen. Kan vara olika på varje enhet.",
     "Options": "Alternativ",
-    "Out of Sync": "Osynkad",
-    "Out of Sync Items": "Osynkade poster",
+    "Out of Sync": "Osynkroniserad",
+    "Out of Sync Items": "Osynkroniserade poster",
     "Outgoing Rate Limit (KiB/s)": "Max uppladdningshastighet (KiB/s)",
     "Override Changes": "Skriv över ändringar",
     "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Sökväg till katalogen på din dator. Kommer att skapas om det inte finns. Tecknet tilde (~) kan användas som en genväg för",
@@ -144,7 +144,7 @@
     "Relay Servers": "Reläservrar",
     "Relayed via": "Vidarbefordras via",
     "Relays": "Vidarbefordringar",
-    "Release Notes": "versionsnyheter",
+    "Release Notes": "Versionsanteckningar",
     "Remote Devices": "Fjärrenheter",
     "Remove": "Ta bort",
     "Required identifier for the folder. Must be the same on all cluster devices.": "Krävs identifierare för katalogen. Måste vara densamma på alla kluster enheter.",
@@ -186,7 +186,7 @@
     "Support": "Support",
     "Sync Protocol Listen Addresses": "Address för inkommande anslutningar",
     "Syncing": "Synkroniserar",
-    "Syncthing has been shut down.": "Syncthing har stängts ner.",
+    "Syncthing has been shut down.": "Syncthing har stängts.",
     "Syncthing includes the following software or portions thereof:": "Syncthing innehåller följande mjukvarupaket eller delar av dem:",
     "Syncthing is restarting.": "Syncthing startar om.",
     "Syncthing is upgrading.": "Syncthing uppgraderas.",
@@ -219,7 +219,7 @@
     "The rate limit must be a non-negative number (0: no limit)": "Frekvensgränsen måste vara ett icke-negativt tal (0: ingen gräns)",
     "The rescan interval must be a non-negative number of seconds.": "Förnyelseintervallet måste vara ett positivt antal sekunder",
     "They are retried automatically and will be synced when the error is resolved.": "De omprövas automatiskt och kommer att synkroniseras när felet är löst.",
-    "This Device": "Enheten",
+    "This Device": "Denna enhet",
     "This can easily give hackers access to read and change any files on your computer.": "Detta kan lätt ge hackare tillgång till att läsa och ändra några filer på datorn.",
     "This is a major version upgrade.": "Det här är en stor uppgradering.",
     "Trash Can File Versioning": "Versionshantering på filer i papperskorgen",

gui/default/syncthing/core/syncthingController.js

@@ -1318,7 +1318,7 @@ angular.module('syncthing.core')
             $scope.editingExisting = false;
             $scope.folderEditor.$setPristine();
             $http.get(urlbase + '/svc/random/string?length=10').success(function (data) {
-                $scope.currentFolder.id = data.random.substr(0, 5) + '-' + data.random.substr(5, 5);
+                $scope.currentFolder.id = (data.random.substr(0, 5) + '-' + data.random.substr(5, 5)).toLowerCase();
                 $('#editFolder').modal();
             });
         };
@@ -1453,7 +1453,7 @@ angular.module('syncthing.core')
                 }
             }
 
-            folders.sort();
+            folders.sort(folderCompare);
             return folders;
         };
 

lib/config/config.go

@@ -10,6 +10,7 @@ package config
 import (
 	"encoding/json"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/url"
@@ -81,11 +82,15 @@ func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
 	util.SetDefaults(&cfg.Options)
 	util.SetDefaults(&cfg.GUI)
 
-	err := xml.NewDecoder(r).Decode(&cfg)
+	if err := xml.NewDecoder(r).Decode(&cfg); err != nil {
+		return Configuration{}, err
+	}
 	cfg.OriginalVersion = cfg.Version
 
-	cfg.prepare(myID)
-	return cfg, err
+	if err := cfg.prepare(myID); err != nil {
+		return Configuration{}, err
+	}
+	return cfg, nil
 }
 
 func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
@@ -97,14 +102,16 @@ func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
 
 	bs, err := ioutil.ReadAll(r)
 	if err != nil {
-		return cfg, err
+		return Configuration{}, err
 	}
 
 	err = json.Unmarshal(bs, &cfg)
 	cfg.OriginalVersion = cfg.Version
 
-	cfg.prepare(myID)
-	return cfg, err
+	if err := cfg.prepare(myID); err != nil {
+		return Configuration{}, err
+	}
+	return cfg, nil
 }
 
 type Configuration struct {
@@ -154,7 +161,7 @@ func (cfg *Configuration) WriteXML(w io.Writer) error {
 	return err
 }
 
-func (cfg *Configuration) prepare(myID protocol.DeviceID) {
+func (cfg *Configuration) prepare(myID protocol.DeviceID) error {
 	util.FillNilSlices(&cfg.Options)
 
 	// Initialize any empty slices
@@ -168,19 +175,19 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		cfg.Options.AlwaysLocalNets = []string{}
 	}
 
-	// Check for missing, bad or duplicate folder ID:s
-	var seenFolders = map[string]*FolderConfiguration{}
+	// Prepare folders and check for duplicates. Duplicates are bad and
+	// dangerous, can't currently be resolved in the GUI, and shouldn't
+	// happen when configured by the GUI. We return with an error in that
+	// situation.
+	seenFolders := make(map[string]struct{})
 	for i := range cfg.Folders {
 		folder := &cfg.Folders[i]
 		folder.prepare()
 
-		if seen, ok := seenFolders[folder.ID]; ok {
-			l.Warnf("Multiple folders with ID %q; disabling", folder.ID)
-			seen.Invalid = "duplicate folder ID"
-			folder.Invalid = "duplicate folder ID"
-		} else {
-			seenFolders[folder.ID] = folder
+		if _, ok := seenFolders[folder.ID]; ok {
+			return fmt.Errorf("duplicate folder ID %q in configuration", folder.ID)
 		}
+		seenFolders[folder.ID] = struct{}{}
 	}
 
 	cfg.Options.ListenAddresses = util.UniqueStrings(cfg.Options.ListenAddresses)
@@ -257,6 +264,8 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 	if cfg.GUI.APIKey == "" {
 		cfg.GUI.APIKey = rand.String(32)
 	}
+
+	return nil
 }
 
 func convertV14V15(cfg *Configuration) {

lib/config/config_test.go

@@ -595,22 +595,14 @@ func TestGUIConfigURL(t *testing.T) {
 	}
 }
 
-func TestRemoveDuplicateDevicesFolders(t *testing.T) {
-	wrapper, err := Load("testdata/duplicates.xml", device1)
+func TestDuplicateDevices(t *testing.T) {
+	// Duplicate devices should be removed
+
+	wrapper, err := Load("testdata/dupdevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// All folders are loaded, but the duplicate ones are disabled.
-	if l := len(wrapper.Raw().Folders); l != 3 {
-		t.Errorf("Incorrect number of folders, %d != 3", l)
-	}
-	for i, f := range wrapper.Raw().Folders {
-		if f.ID == "f1" && f.Invalid == "" {
-			t.Errorf("Folder %d (%q) is not set invalid", i, f.ID)
-		}
-	}
-
 	if l := len(wrapper.Raw().Devices); l != 3 {
 		t.Errorf("Incorrect number of devices, %d != 3", l)
 	}
@@ -621,6 +613,30 @@ func TestRemoveDuplicateDevicesFolders(t *testing.T) {
 	}
 }
 
+func TestDuplicateFolders(t *testing.T) {
+	// Duplicate folders are a loading error
+
+	_, err := Load("testdata/dupfolders.xml", device1)
+	if err == nil || !strings.HasPrefix(err.Error(), "duplicate folder ID") {
+		t.Fatal(`Expected error to mention "duplicate folder ID":`, err)
+	}
+}
+
+func TestEmptyFolderPaths(t *testing.T) {
+	// Empty folder paths are allowed at the loading stage, and should not
+	// get messed up by the prepare steps (e.g., become the current dir or
+	// get a slash added so that it becomes the root directory or similar).
+
+	wrapper, err := Load("testdata/nopath.xml", device1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	folder := wrapper.Folders()["f1"]
+	if folder.Path() != "" {
+		t.Errorf("Expected %q to be empty", folder.Path())
+	}
+}
+
 func TestV14ListenAddressesMigration(t *testing.T) {
 	tcs := [][3][]string{
 

lib/config/folderconfiguration.go

@@ -39,7 +39,6 @@ type FolderConfiguration struct {
 	DisableSparseFiles    bool                        `xml:"disableSparseFiles" json:"disableSparseFiles"`
 	DisableTempIndexes    bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
 
-	Invalid    string `xml:"-" json:"invalid"` // Set at runtime when there is an error, not saved
 	cachedPath string
 
 	DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
@@ -70,7 +69,7 @@ func (f FolderConfiguration) Path() string {
 	// This is intentionally not a pointer method, because things like
 	// cfg.Folders["default"].Path() should be valid.
 
-	if f.cachedPath == "" {
+	if f.cachedPath == "" && f.RawPath != "" {
 		l.Infoln("bug: uncached path call (should only happen in tests)")
 		return f.cleanedPath()
 	}
@@ -108,31 +107,24 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
 }
 
 func (f *FolderConfiguration) prepare() {
-	if len(f.RawPath) == 0 {
-		f.Invalid = "no directory configured"
-		return
-	}
+	if f.RawPath != "" {
+		// The reason it's done like this:
+		// C:          ->  C:\            ->  C:\        (issue that this is trying to fix)
+		// C:\somedir  ->  C:\somedir\    ->  C:\somedir
+		// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
+		// This way in the tests, we get away without OS specific separators
+		// in the test configs.
+		f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator))
 
-	// The reason it's done like this:
-	// C:          ->  C:\            ->  C:\        (issue that this is trying to fix)
-	// C:\somedir  ->  C:\somedir\    ->  C:\somedir
-	// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
-	// This way in the tests, we get away without OS specific separators
-	// in the test configs.
-	f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator))
-
-	// If we're not on Windows, we want the path to end with a slash to
-	// penetrate symlinks. On Windows, paths must not end with a slash.
-	if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator {
-		f.RawPath = f.RawPath + string(filepath.Separator)
+		// If we're not on Windows, we want the path to end with a slash to
+		// penetrate symlinks. On Windows, paths must not end with a slash.
+		if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator {
+			f.RawPath = f.RawPath + string(filepath.Separator)
+		}
 	}
 
 	f.cachedPath = f.cleanedPath()
 
-	if f.ID == "" {
-		f.ID = "default"
-	}
-
 	if f.RescanIntervalS > MaxRescanIntervalS {
 		f.RescanIntervalS = MaxRescanIntervalS
 	} else if f.RescanIntervalS < 0 {
@@ -145,6 +137,10 @@ func (f *FolderConfiguration) prepare() {
 }
 
 func (f *FolderConfiguration) cleanedPath() string {
+	if f.RawPath == "" {
+		return ""
+	}
+
 	cleaned := f.RawPath
 
 	// Attempt tilde expansion; leave unchanged in case of error

lib/config/testdata/dupdevices.xml

@@ -15,15 +15,6 @@
         <!-- duplicate, will be removed -->
         <address>192.0.2.5</address>
     </device>
-    <folder id="f1" directory="testdata/">
-        <!-- duplicate, will be disabled -->
-        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
-        <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA"></device>
-    </folder>
-    <folder id="f1" directory="testdata/">
-        <!-- duplicate, will be disabled -->
-        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
-    </folder>
     <folder id="f2" directory="testdata/">
         <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
         <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA"></device>

lib/config/testdata/dupfolders.xml

@@ -0,0 +1,6 @@
+<configuration version="15">
+    <folder id="f1" directory="testdata/">
+    </folder>
+    <folder id="f1" directory="testdata/">
+    </folder>
+</configuration>

lib/config/testdata/nopath.xml

@@ -0,0 +1,4 @@
+<configuration version="15">
+    <folder id="f1">
+    </folder>
+</configuration>

lib/connections/relay_dial.go

@@ -52,7 +52,7 @@ func (d *relayDialer) Dial(id protocol.DeviceID, uri *url.URL) (IntermediateConn
 		tc = tls.Client(conn, d.tlsCfg)
 	}
 
-	err = tc.Handshake()
+	err = tlsTimedHandshake(tc)
 	if err != nil {
 		tc.Close()
 		return IntermediateConnection{}, err

lib/connections/relay_listen.go

@@ -85,7 +85,7 @@ func (t *relayListener) Serve() {
 				tc = tls.Client(conn, t.tlsCfg)
 			}
 
-			err = tc.Handshake()
+			err = tlsTimedHandshake(tc)
 			if err != nil {
 				tc.Close()
 				l.Infoln("TLS handshake (BEP/relay):", err)

lib/connections/service.go

@@ -36,7 +36,10 @@ var (
 	listeners = make(map[string]listenerFactory, 0)
 )
 
-const perDeviceWarningRate = 1.0 / (15 * 60) // Once per 15 minutes
+const (
+	perDeviceWarningRate = 1.0 / (15 * 60) // Once per 15 minutes
+	tlsHandshakeTimeout  = 10 * time.Second
+)
 
 // Service listens and dials all configured unconnected devices, via supported
 // dialers. Successful connections are handed to the model.
@@ -159,8 +162,17 @@ next:
 		if err != nil {
 			if protocol.IsVersionMismatch(err) {
 				// The error will be a relatively user friendly description
-				// of what's wrong with the version compatibility
-				msg := fmt.Sprintf("Connecting to %s (%s): %s", remoteID, c.RemoteAddr(), err)
+				// of what's wrong with the version compatibility. By
+				// default identify the other side by device ID and IP.
+				remote := fmt.Sprintf("%v (%v)", remoteID, c.RemoteAddr())
+				if hello.DeviceName != "" {
+					// If the name was set in the hello return, use that to
+					// give the user more info about which device is the
+					// affected one. It probably says more than the remote
+					// IP.
+					remote = fmt.Sprintf("%q (%s %s, %v)", hello.DeviceName, hello.ClientName, hello.ClientVersion, remoteID)
+				}
+				msg := fmt.Sprintf("Connecting to %s: %s", remote, err)
 				warningFor(remoteID, msg)
 			} else {
 				// It's something else - connection reset or whatever
@@ -347,7 +359,7 @@ func (s *Service) connect() {
 				}
 
 				if connected && dialerFactory.Priority() >= ct.Priority {
-					l.Debugf("Not dialing using %s as priorty is less than current connection (%d >= %d)", dialerFactory, dialerFactory.Priority(), ct.Priority)
+					l.Debugf("Not dialing using %s as priority is less than current connection (%d >= %d)", dialerFactory, dialerFactory.Priority(), ct.Priority)
 					continue
 				}
 
@@ -607,3 +619,9 @@ func warningFor(dev protocol.DeviceID, msg string) {
 		l.Warnln(msg)
 	}
 }
+
+func tlsTimedHandshake(tc *tls.Conn) error {
+	tc.SetDeadline(time.Now().Add(tlsHandshakeTimeout))
+	defer tc.SetDeadline(time.Time{})
+	return tc.Handshake()
+}

lib/connections/structs.go

@@ -70,7 +70,7 @@ type Model interface {
 	ConnectedTo(remoteID protocol.DeviceID) bool
 	IsPaused(remoteID protocol.DeviceID) bool
 	OnHello(protocol.DeviceID, net.Addr, protocol.HelloResult)
-	GetHello(protocol.DeviceID) protocol.Version13HelloMessage
+	GetHello(protocol.DeviceID) protocol.HelloIntf
 }
 
 // serviceFunc wraps a function to create a suture.Service without stop

lib/connections/tcp_dial.go

@@ -40,7 +40,7 @@ func (d *tcpDialer) Dial(id protocol.DeviceID, uri *url.URL) (IntermediateConnec
 	}
 
 	tc := tls.Client(conn, d.tlsCfg)
-	err = tc.Handshake()
+	err = tlsTimedHandshake(tc)
 	if err != nil {
 		tc.Close()
 		return IntermediateConnection{}, err

lib/connections/tcp_listen.go

@@ -108,7 +108,7 @@ func (t *tcpListener) Serve() {
 		}
 
 		tc := tls.Server(conn, t.tlsCfg)
-		err = tc.Handshake()
+		err = tlsTimedHandshake(tc)
 		if err != nil {
 			l.Infoln("TLS handshake (BEP/tcp):", err)
 			tc.Close()

lib/db/blockmap_test.go

@@ -73,7 +73,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 
 	m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
 
-	f3.Flags |= protocol.FlagDirectory
+	f3.Type = protocol.FileInfoTypeDirectory
 
 	err := m.Add([]protocol.FileInfo{f1, f2, f3})
 	if err != nil {
@@ -99,9 +99,11 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		return true
 	})
 
-	f3.Flags = f1.Flags
-	f1.Flags |= protocol.FlagDeleted
-	f2.Flags |= protocol.FlagInvalid
+	f3.Permissions = f1.Permissions
+	f3.Deleted = f1.Deleted
+	f3.Invalid = f1.Invalid
+	f1.Deleted = true
+	f2.Invalid = true
 
 	// Should remove
 	err = m.Update([]protocol.FileInfo{f1, f2, f3})
@@ -145,9 +147,15 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal("db not empty")
 	}
 
-	f1.Flags = 0
-	f2.Flags = 0
-	f3.Flags = 0
+	f1.Deleted = false
+	f1.Invalid = false
+	f1.Permissions = 0
+	f2.Deleted = false
+	f2.Invalid = false
+	f2.Permissions = 0
+	f3.Deleted = false
+	f3.Invalid = false
+	f3.Permissions = 0
 }
 
 func TestBlockFinderLookup(t *testing.T) {
@@ -187,7 +195,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		t.Fatal("Incorrect count", counter)
 	}
 
-	f1.Flags |= protocol.FlagDeleted
+	f1.Deleted = true
 
 	err = m1.Update([]protocol.FileInfo{f1})
 	if err != nil {
@@ -212,7 +220,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		t.Fatal("Incorrect count")
 	}
 
-	f1.Flags = 0
+	f1.Deleted = false
 }
 
 func TestBlockFinderFix(t *testing.T) {

lib/db/leveldb.go

@@ -4,9 +4,6 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o leveldb_xdr.go leveldb.go
-
 package db
 
 import (
@@ -46,25 +43,16 @@ const (
 	KeyTypeDeviceIdx
 )
 
-type fileVersion struct {
-	version protocol.Vector
-	device  []byte
-}
-
-type VersionList struct {
-	versions []fileVersion
-}
-
 func (l VersionList) String() string {
 	var b bytes.Buffer
 	var id protocol.DeviceID
 	b.WriteString("{")
-	for i, v := range l.versions {
+	for i, v := range l.Versions {
 		if i > 0 {
 			b.WriteString(", ")
 		}
-		copy(id[:], v.device)
-		fmt.Fprintf(&b, "{%d, %v}", v.version, id)
+		copy(id[:], v.Device)
+		fmt.Fprintf(&b, "{%d, %v}", v.Version, id)
 	}
 	b.WriteString("}")
 	return b.String()
@@ -101,7 +89,7 @@ func getFile(db dbReader, key []byte) (protocol.FileInfo, bool) {
 	}
 
 	var f protocol.FileInfo
-	err = f.UnmarshalXDR(bs)
+	err = f.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}

lib/db/leveldb_convert.go

@@ -1,114 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"bytes"
-
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-// convertKeyFormat converts from the v0.12 to the v0.13 database format, to
-// avoid having to do rescan. The change is in the key format for folder
-// labels, so we basically just iterate over the database rewriting keys as
-// necessary.
-func convertKeyFormat(from, to *leveldb.DB) error {
-	l.Infoln("Converting database key format")
-	blocks, files, globals, unchanged := 0, 0, 0, 0
-
-	dbi := newDBInstance(to)
-	i := from.NewIterator(nil, nil)
-	for i.Next() {
-		key := i.Key()
-		switch key[0] {
-		case KeyTypeBlock:
-			folder, file := oldFromBlockKey(key)
-			folderIdx := dbi.folderIdx.ID([]byte(folder))
-			hash := key[1+64:]
-			newKey := blockKeyInto(nil, hash, folderIdx, file)
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			blocks++
-
-		case KeyTypeDevice:
-			newKey := dbi.deviceKey(oldDeviceKeyFolder(key), oldDeviceKeyDevice(key), oldDeviceKeyName(key))
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			files++
-
-		case KeyTypeGlobal:
-			newKey := dbi.globalKey(oldGlobalKeyFolder(key), oldGlobalKeyName(key))
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			globals++
-
-		case KeyTypeVirtualMtime:
-			// Cannot be converted, we drop it instead :(
-
-		default:
-			if err := to.Put(key, i.Value(), nil); err != nil {
-				return err
-			}
-			unchanged++
-		}
-	}
-
-	l.Infof("Converted %d blocks, %d files, %d globals (%d unchanged).", blocks, files, globals, unchanged)
-
-	return nil
-}
-
-func oldDeviceKeyFolder(key []byte) []byte {
-	folder := key[1 : 1+64]
-	izero := bytes.IndexByte(folder, 0)
-	if izero < 0 {
-		return folder
-	}
-	return folder[:izero]
-}
-
-func oldDeviceKeyDevice(key []byte) []byte {
-	return key[1+64 : 1+64+32]
-}
-
-func oldDeviceKeyName(key []byte) []byte {
-	return key[1+64+32:]
-}
-
-func oldGlobalKeyName(key []byte) []byte {
-	return key[1+64:]
-}
-
-func oldGlobalKeyFolder(key []byte) []byte {
-	folder := key[1 : 1+64]
-	izero := bytes.IndexByte(folder, 0)
-	if izero < 0 {
-		return folder
-	}
-	return folder[:izero]
-}
-
-func oldFromBlockKey(data []byte) (string, string) {
-	if len(data) < 1+64+32+1 {
-		panic("Incorrect key length")
-	}
-	if data[0] != KeyTypeBlock {
-		panic("Incorrect key type")
-	}
-
-	file := string(data[1+64+32:])
-
-	slice := data[1 : 1+64]
-	izero := bytes.IndexByte(slice, 0)
-	if izero > -1 {
-		return string(slice[:izero]), file
-	}
-	return string(slice), file
-}

lib/db/leveldb_convert_test.go

@@ -1,136 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"archive/zip"
-	"io"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-func TestLabelConversion(t *testing.T) {
-	os.RemoveAll("testdata/oldformat.db")
-	defer os.RemoveAll("testdata/oldformat.db")
-	os.RemoveAll("testdata/newformat.db")
-	defer os.RemoveAll("testdata/newformat.db")
-
-	if err := unzip("testdata/oldformat.db.zip", "testdata"); err != nil {
-		t.Fatal(err)
-	}
-
-	odb, err := leveldb.OpenFile("testdata/oldformat.db", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ldb, err := leveldb.OpenFile("testdata/newformat.db", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err = convertKeyFormat(odb, ldb); err != nil {
-		t.Fatal(err)
-	}
-	ldb.Close()
-	odb.Close()
-
-	inst, err := Open("testdata/newformat.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fs := NewFileSet("default", inst)
-	files, deleted, _ := fs.GlobalSize()
-	if files+deleted != 953 {
-		// Expected number of global entries determined by
-		// ../../bin/stindex testdata/oldformat.db/ | grep global | grep -c default
-		t.Errorf("Conversion error, global list differs (%d != 953)", files+deleted)
-	}
-
-	files, deleted, _ = fs.LocalSize()
-	if files+deleted != 953 {
-		t.Errorf("Conversion error, device list differs (%d != 953)", files+deleted)
-	}
-
-	f := NewBlockFinder(inst)
-	// [block] F:"default" H:1c25dea9003cc16216e2a22900be1ec1cc5aaf270442904e2f9812c314e929d8 N:"f/f2/f25f1b3e6e029231b933531b2138796d" I:3
-	h := []byte{0x1c, 0x25, 0xde, 0xa9, 0x00, 0x3c, 0xc1, 0x62, 0x16, 0xe2, 0xa2, 0x29, 0x00, 0xbe, 0x1e, 0xc1, 0xcc, 0x5a, 0xaf, 0x27, 0x04, 0x42, 0x90, 0x4e, 0x2f, 0x98, 0x12, 0xc3, 0x14, 0xe9, 0x29, 0xd8}
-	found := 0
-	f.Iterate([]string{"default"}, h, func(folder, file string, idx int32) bool {
-		if folder == "default" && file == filepath.FromSlash("f/f2/f25f1b3e6e029231b933531b2138796d") && idx == 3 {
-			found++
-		}
-		return true
-	})
-	if found != 1 {
-		t.Errorf("Found %d blocks instead of expected 1", found)
-	}
-
-	inst.Close()
-}
-
-func unzip(src, dest string) error {
-	r, err := zip.OpenReader(src)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := r.Close(); err != nil {
-			panic(err)
-		}
-	}()
-
-	os.MkdirAll(dest, 0755)
-
-	// Closure to address file descriptors issue with all the deferred .Close() methods
-	extractAndWriteFile := func(f *zip.File) error {
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := rc.Close(); err != nil {
-				panic(err)
-			}
-		}()
-
-		path := filepath.Join(dest, f.Name)
-
-		if f.FileInfo().IsDir() {
-			os.MkdirAll(path, f.Mode())
-		} else {
-			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-			if err != nil {
-				return err
-			}
-			defer func() {
-				if err := f.Close(); err != nil {
-					panic(err)
-				}
-			}()
-
-			_, err = io.Copy(f, rc)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	for _, f := range r.File {
-		err := extractAndWriteFile(f)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

lib/db/leveldb_dbinstance.go

@@ -10,12 +10,10 @@ import (
 	"bytes"
 	"encoding/binary"
 	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 	"sync/atomic"
 
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -29,10 +27,10 @@ import (
 type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64
 
 type Instance struct {
+	committed int64 // this must be the first attribute in the struct to ensure 64 bit alignment on 32 bit plaforms
 	*leveldb.DB
 	folderIdx *smallIndex
 	deviceIdx *smallIndex
-	committed int64
 }
 
 const (
@@ -48,16 +46,6 @@ func Open(file string) (*Instance, error) {
 		WriteBuffer:            4 << 20,
 	}
 
-	if _, err := os.Stat(file); os.IsNotExist(err) {
-		// The file we are looking to open does not exist. This may be the
-		// first launch so we should look for an old version and try to
-		// convert it.
-		if err := checkConvertDatabase(file); err != nil {
-			l.Infoln("Converting old database:", err)
-			l.Infoln("Will rescan from scratch.")
-		}
-	}
-
 	db, err := leveldb.OpenFile(file, opts)
 	if leveldbIsCorrupted(err) {
 		db, err = leveldb.RecoverFile(file, opts)
@@ -151,12 +139,12 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
 
 		case moreFs && moreDb && cmp == 0:
 			// File exists on both sides - compare versions. We might get an
-			// update with the same version and different flags if a device has
-			// marked a file as invalid, so handle that too.
+			// update with the same version if a device has marked a file as
+			// invalid, so handle that too.
 			l.Debugln("generic replace; exists - compare")
 			var ef FileInfoTruncated
-			ef.UnmarshalXDR(dbi.Value())
-			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Flags != ef.Flags {
+			ef.Unmarshal(dbi.Value())
+			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Invalid != ef.Invalid {
 				l.Debugln("generic replace; differs - insert")
 				if lv := t.insertFile(folder, device, fs[fsi]); lv > maxLocalVer {
 					maxLocalVer = lv
@@ -232,13 +220,12 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, l
 		}
 
 		var ef FileInfoTruncated
-		err = ef.UnmarshalXDR(bs)
+		err = ef.Unmarshal(bs)
 		if err != nil {
 			panic(err)
 		}
-		// Flags might change without the version being bumped when we set the
-		// invalid flag on an existing file.
-		if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
+		// The Invalid flag might change without the version being bumped.
+		if !ef.Version.Equal(f.Version) || ef.Invalid != f.Invalid {
 			if isLocalDevice {
 				localSize.removeFile(ef)
 				localSize.addFile(f)
@@ -308,7 +295,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 		// struct, which in turn references the buffer it was unmarshalled
 		// from. dbi.Value() just returns an internal slice that it reuses, so
 		// we need to copy it.
-		err := f.UnmarshalXDR(append([]byte{}, dbi.Value()...))
+		err := f.Unmarshal(append([]byte{}, dbi.Value()...))
 		if err != nil {
 			panic(err)
 		}
@@ -347,16 +334,16 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
 	}
 
 	var vl VersionList
-	err = vl.UnmarshalXDR(bs)
+	err = vl.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}
-	if len(vl.versions) == 0 {
+	if len(vl.Versions) == 0 {
 		l.Debugln(k)
 		panic("no versions?")
 	}
 
-	k = db.deviceKey(folder, vl.versions[0].device, file)
+	k = db.deviceKey(folder, vl.Versions[0].Device, file)
 	bs, err = t.Get(k, nil)
 	if err != nil {
 		panic(err)
@@ -384,11 +371,11 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 	var fk []byte
 	for dbi.Next() {
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
-		if len(vl.versions) == 0 {
+		if len(vl.Versions) == 0 {
 			l.Debugln(dbi.Key())
 			panic("no versions?")
 		}
@@ -398,13 +385,13 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 			return
 		}
 
-		fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[0].device, name)
+		fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[0].Device, name)
 		bs, err := t.Get(fk, nil)
 		if err != nil {
 			l.Debugf("folder: %q (%x)", folder, folder)
 			l.Debugf("key: %q (%x)", dbi.Key(), dbi.Key())
 			l.Debugf("vl: %v", vl)
-			l.Debugf("vl.versions[0].device: %x", vl.versions[0].device)
+			l.Debugf("vl.Versions[0].Device: %x", vl.Versions[0].Device)
 			l.Debugf("name: %q (%x)", name, name)
 			l.Debugf("fk: %q", fk)
 			l.Debugf("fk: %x %x %x",
@@ -436,17 +423,17 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
 	}
 
 	var vl VersionList
-	err = vl.UnmarshalXDR(bs)
+	err = vl.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}
 
 	var devices []protocol.DeviceID
-	for _, v := range vl.versions {
-		if !v.version.Equal(vl.versions[0].version) {
+	for _, v := range vl.Versions {
+		if !v.Version.Equal(vl.Versions[0].Version) {
 			break
 		}
-		n := protocol.DeviceIDFromBytes(v.device)
+		n := protocol.DeviceIDFromBytes(v.Device)
 		devices = append(devices, n)
 	}
 
@@ -464,11 +451,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 nextFile:
 	for dbi.Next() {
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
-		if len(vl.versions) == 0 {
+		if len(vl.Versions) == 0 {
 			l.Debugln(dbi.Key())
 			panic("no versions?")
 		}
@@ -476,29 +463,29 @@ nextFile:
 		have := false // If we have the file, any version
 		need := false // If we have a lower version of the file
 		var haveVersion protocol.Vector
-		for _, v := range vl.versions {
-			if bytes.Equal(v.device, device) {
+		for _, v := range vl.Versions {
+			if bytes.Equal(v.Device, device) {
 				have = true
-				haveVersion = v.version
+				haveVersion = v.Version
 				// XXX: This marks Concurrent (i.e. conflicting) changes as
 				// needs. Maybe we should do that, but it needs special
 				// handling in the puller.
-				need = !v.version.GreaterEqual(vl.versions[0].version)
+				need = !v.Version.GreaterEqual(vl.Versions[0].Version)
 				break
 			}
 		}
 
 		if need || !have {
 			name := db.globalKeyName(dbi.Key())
-			needVersion := vl.versions[0].version
+			needVersion := vl.Versions[0].Version
 
 		nextVersion:
-			for i := range vl.versions {
-				if !vl.versions[i].version.Equal(needVersion) {
+			for i := range vl.Versions {
+				if !vl.Versions[i].Version.Equal(needVersion) {
 					// We haven't found a valid copy of the file with the needed version.
 					continue nextFile
 				}
-				fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[i].device, name)
+				fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[i].Device, name)
 				bs, err := t.Get(fk, nil)
 				if err != nil {
 					var id protocol.DeviceID
@@ -528,7 +515,7 @@ nextFile:
 					continue nextFile
 				}
 
-				l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.versions[0].version)
+				l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.Versions[0].Version)
 
 				if cont := fn(gf); !cont {
 					return
@@ -601,7 +588,7 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 	for dbi.Next() {
 		gk := dbi.Key()
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
@@ -613,8 +600,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 
 		name := db.globalKeyName(gk)
 		var newVL VersionList
-		for i, version := range vl.versions {
-			fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
+		for i, version := range vl.Versions {
+			fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.Device, name)
 
 			_, err := t.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
@@ -623,10 +610,10 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 			if err != nil {
 				panic(err)
 			}
-			newVL.versions = append(newVL.versions, version)
+			newVL.Versions = append(newVL.Versions, version)
 
 			if i == 0 {
-				fi, ok := t.getFile(folder, version.device, name)
+				fi, ok := t.getFile(folder, version.Device, name)
 				if !ok {
 					panic("nonexistent global master file")
 				}
@@ -634,8 +621,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 			}
 		}
 
-		if len(newVL.versions) != len(vl.versions) {
-			t.Put(dbi.Key(), newVL.MustMarshalXDR())
+		if len(newVL.Versions) != len(vl.Versions) {
+			t.Put(dbi.Key(), mustMarshal(&newVL))
 			t.checkFlush()
 		}
 	}
@@ -715,12 +702,12 @@ func (db *Instance) globalKeyFolder(key []byte) []byte {
 func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	if truncate {
 		var tf FileInfoTruncated
-		err := tf.UnmarshalXDR(bs)
+		err := tf.Unmarshal(bs)
 		return tf, err
 	}
 
 	var tf protocol.FileInfo
-	err := tf.UnmarshalXDR(bs)
+	err := tf.Unmarshal(bs)
 	return tf, err
 }
 
@@ -740,50 +727,6 @@ func leveldbIsCorrupted(err error) bool {
 	return false
 }
 
-// checkConvertDatabase tries to convert an existing old (v0.11) database to
-// new (v0.13) format.
-func checkConvertDatabase(dbFile string) error {
-	oldLoc := filepath.Join(filepath.Dir(dbFile), "index-v0.11.0.db")
-	if _, err := os.Stat(oldLoc); os.IsNotExist(err) {
-		// The old database file does not exist; that's ok, continue as if
-		// everything succeeded.
-		return nil
-	} else if err != nil {
-		// Any other error is weird.
-		return err
-	}
-
-	// There exists a database in the old format. We run a one time
-	// conversion from old to new.
-
-	fromDb, err := leveldb.OpenFile(oldLoc, nil)
-	if err != nil {
-		return err
-	}
-
-	toDb, err := leveldb.OpenFile(dbFile, nil)
-	if err != nil {
-		return err
-	}
-
-	err = convertKeyFormat(fromDb, toDb)
-	if err != nil {
-		return err
-	}
-
-	err = toDb.Close()
-	if err != nil {
-		return err
-	}
-
-	// We've done this one, we don't want to do it again (if the user runs
-	// -reset or so). We don't care too much about errors any more at this stage.
-	fromDb.Close()
-	osutil.Rename(oldLoc, oldLoc+".converted")
-
-	return nil
-}
-
 // A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
 // fast lookups in both directions and persists to the database. Don't use for
 // storing more items than fit comfortably in RAM.

lib/db/leveldb_transactions.go

@@ -83,7 +83,7 @@ func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.Fi
 
 	name := []byte(file.Name)
 	nk := t.db.deviceKey(folder, device, name)
-	t.Put(nk, file.MustMarshalXDR())
+	t.Put(nk, mustMarshal(&file))
 
 	return file.LocalVersion
 }
@@ -105,14 +105,14 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 	var hasOldFile bool
 	// Remove the device from the current version list
 	if len(svl) != 0 {
-		err = fl.UnmarshalXDR(svl)
+		err = fl.Unmarshal(svl)
 		if err != nil {
 			panic(err)
 		}
 
-		for i := range fl.versions {
-			if bytes.Equal(fl.versions[i].device, device) {
-				if fl.versions[i].version.Equal(file.Version) {
+		for i := range fl.Versions {
+			if bytes.Equal(fl.Versions[i].Device, device) {
+				if fl.Versions[i].Version.Equal(file.Version) {
 					// No need to do anything
 					return false
 				}
@@ -120,29 +120,29 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 				if i == 0 {
 					// Keep the current newest file around so we can subtract it from
 					// the globalSize if we replace it.
-					oldFile, hasOldFile = t.getFile(folder, fl.versions[0].device, name)
+					oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
 				}
 
-				fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+				fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
 				break
 			}
 		}
 	}
 
-	nv := fileVersion{
-		device:  device,
-		version: file.Version,
+	nv := FileVersion{
+		Device:  device,
+		Version: file.Version,
 	}
 
 	insertedAt := -1
 	// Find a position in the list to insert this file. The file at the front
 	// of the list is the newer, the "global".
-	for i := range fl.versions {
-		switch fl.versions[i].version.Compare(file.Version) {
+	for i := range fl.Versions {
+		switch fl.Versions[i].Version.Compare(file.Version) {
 		case protocol.Equal, protocol.Lesser:
 			// The version at this point in the list is equal to or lesser
 			// ("older") than us. We insert ourselves in front of it.
-			fl.versions = insertVersion(fl.versions, i, nv)
+			fl.Versions = insertVersion(fl.Versions, i, nv)
 			insertedAt = i
 			goto done
 
@@ -153,12 +153,12 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 			// "Greater" in the condition above is just based on the device
 			// IDs in the version vector, which is not the only thing we use
 			// to determine the winner.)
-			of, ok := t.getFile(folder, fl.versions[i].device, name)
+			of, ok := t.getFile(folder, fl.Versions[i].Device, name)
 			if !ok {
 				panic("file referenced in version list does not exist")
 			}
 			if file.WinsConflict(of) {
-				fl.versions = insertVersion(fl.versions, i, nv)
+				fl.Versions = insertVersion(fl.Versions, i, nv)
 				insertedAt = i
 				goto done
 			}
@@ -166,8 +166,8 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 	}
 
 	// We didn't find a position for an insert above, so append to the end.
-	fl.versions = append(fl.versions, nv)
-	insertedAt = len(fl.versions) - 1
+	fl.Versions = append(fl.Versions, nv)
+	insertedAt = len(fl.Versions) - 1
 
 done:
 	if insertedAt == 0 {
@@ -178,9 +178,9 @@ done:
 			if hasOldFile {
 				// We have the old file that was removed at the head of the list.
 				globalSize.removeFile(oldFile)
-			} else if len(fl.versions) > 1 {
+			} else if len(fl.Versions) > 1 {
 				// The previous newest version is now at index 1, grab it from there.
-				oldFile, ok := t.getFile(folder, fl.versions[1].device, name)
+				oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name)
 				if !ok {
 					panic("file referenced in version list does not exist")
 				}
@@ -190,7 +190,7 @@ done:
 	}
 
 	l.Debugf("new global after update: %v", fl)
-	t.Put(gk, fl.MustMarshalXDR())
+	t.Put(gk, mustMarshal(&fl))
 
 	return true
 }
@@ -210,14 +210,14 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 	}
 
 	var fl VersionList
-	err = fl.UnmarshalXDR(svl)
+	err = fl.Unmarshal(svl)
 	if err != nil {
 		panic(err)
 	}
 
 	removed := false
-	for i := range fl.versions {
-		if bytes.Equal(fl.versions[i].device, device) {
+	for i := range fl.Versions {
+		if bytes.Equal(fl.Versions[i].Device, device) {
 			if i == 0 && globalSize != nil {
 				f, ok := t.getFile(folder, device, file)
 				if !ok {
@@ -226,18 +226,18 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 				globalSize.removeFile(f)
 				removed = true
 			}
-			fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+			fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
 			break
 		}
 	}
 
-	if len(fl.versions) == 0 {
+	if len(fl.Versions) == 0 {
 		t.Delete(gk)
 	} else {
 		l.Debugf("new global after remove: %v", fl)
-		t.Put(gk, fl.MustMarshalXDR())
+		t.Put(gk, mustMarshal(&fl))
 		if removed {
-			f, ok := t.getFile(folder, fl.versions[0].device, file)
+			f, ok := t.getFile(folder, fl.Versions[0].Device, file)
 			if !ok {
 				panic("new global is nonexistent file")
 			}
@@ -246,9 +246,21 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 	}
 }
 
-func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
-	t := append(vl, fileVersion{})
+func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion {
+	t := append(vl, FileVersion{})
 	copy(t[i+1:], t[i:])
 	t[i] = v
 	return t
 }
+
+type marshaller interface {
+	Marshal() ([]byte, error)
+}
+
+func mustMarshal(f marshaller) []byte {
+	bs, err := f.Marshal()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}

lib/db/leveldb_xdr.go

@@ -1,142 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package db
-
-import (
-	"github.com/calmh/xdr"
-)
-
-/*
-
-fileVersion Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Vector Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                 device (length + padded data)                 \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct fileVersion {
-	Vector version;
-	opaque device<>;
-}
-
-*/
-
-func (o fileVersion) XDRSize() int {
-	return o.version.XDRSize() +
-		4 + len(o.device) + xdr.Padding(len(o.device))
-}
-
-func (o fileVersion) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o fileVersion) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o fileVersion) MarshalXDRInto(m *xdr.Marshaller) error {
-	if err := o.version.MarshalXDRInto(m); err != nil {
-		return err
-	}
-	m.MarshalBytes(o.device)
-	return m.Error
-}
-
-func (o *fileVersion) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *fileVersion) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	(&o.version).UnmarshalXDRFrom(u)
-	o.device = u.UnmarshalBytes()
-	return u.Error
-}
-
-/*
-
-VersionList Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                      Number of versions                       |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\              Zero or more fileVersion Structures              \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct VersionList {
-	fileVersion versions<>;
-}
-
-*/
-
-func (o VersionList) XDRSize() int {
-	return 4 + xdr.SizeOfSlice(o.versions)
-}
-
-func (o VersionList) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o VersionList) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o VersionList) MarshalXDRInto(m *xdr.Marshaller) error {
-	m.MarshalUint32(uint32(len(o.versions)))
-	for i := range o.versions {
-		if err := o.versions[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *VersionList) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *VersionList) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	_versionsSize := int(u.UnmarshalUint32())
-	if _versionsSize < 0 {
-		return xdr.ElementSizeExceeded("versions", _versionsSize, 0)
-	} else if _versionsSize == 0 {
-		o.versions = nil
-	} else {
-		if _versionsSize <= len(o.versions) {
-			o.versions = o.versions[:_versionsSize]
-		} else {
-			o.versions = make([]fileVersion, _versionsSize)
-		}
-		for i := range o.versions {
-			(&o.versions[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}

lib/db/set.go

@@ -31,9 +31,10 @@ type FileSet struct {
 }
 
 // FileIntf is the set of methods implemented by both protocol.FileInfo and
-// protocol.FileInfoTruncated.
+// FileInfoTruncated.
 type FileIntf interface {
-	Size() int64
+	FileSize() int64
+	FileName() string
 	IsDeleted() bool
 	IsInvalid() bool
 	IsDirectory() bool
@@ -42,7 +43,7 @@ type FileIntf interface {
 }
 
 // The Iterator is called with either a protocol.FileInfo or a
-// protocol.FileInfoTruncated (depending on the method) and returns true to
+// FileInfoTruncated (depending on the method) and returns true to
 // continue iteration, false to stop.
 type Iterator func(f FileIntf) bool
 
@@ -64,7 +65,7 @@ func (s *sizeTracker) addFile(f FileIntf) {
 	} else {
 		s.files++
 	}
-	s.bytes += f.Size()
+	s.bytes += f.FileSize()
 	s.mut.Unlock()
 }
 
@@ -79,7 +80,7 @@ func (s *sizeTracker) removeFile(f FileIntf) {
 	} else {
 		s.files--
 	}
-	s.bytes -= f.Size()
+	s.bytes -= f.FileSize()
 	if s.deleted < 0 || s.files < 0 {
 		panic("bug: removed more than added")
 	}

lib/db/set_test.go

@@ -88,7 +88,7 @@ func (l fileList) String() string {
 	var b bytes.Buffer
 	b.WriteString("[]protocol.FileList{\n")
 	for _, f := range l {
-		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
+		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, perms=%o\n", f.Name, f.Version, f.Size, len(f.Blocks), f.Permissions)
 	}
 	b.WriteString("}")
 	return b.String()
@@ -100,35 +100,35 @@ func TestGlobalSet(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 	local1 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
 	}
 	localTot := fileList{
 		local0[0],
 		local0[1],
 		local0[2],
 		local0[3],
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
 	}
 
 	remote0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(5)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(5)},
 	}
 	remote1 := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(6)},
-		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(6)},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(7)},
 	}
 	remoteTot := fileList{
 		remote0[0],
@@ -178,7 +178,7 @@ func TestGlobalSet(t *testing.T) {
 		} else {
 			globalFiles++
 		}
-		globalBytes += f.Size()
+		globalBytes += f.FileSize()
 	}
 	gsFiles, gsDeleted, gsBytes := m.GlobalSize()
 	if gsFiles != globalFiles {
@@ -208,7 +208,7 @@ func TestGlobalSet(t *testing.T) {
 		} else {
 			haveFiles++
 		}
-		haveBytes += f.Size()
+		haveBytes += f.FileSize()
 	}
 	lsFiles, lsDeleted, lsBytes := m.LocalSize()
 	if lsFiles != haveFiles {
@@ -303,23 +303,23 @@ func TestNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
 	}
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 
 	expectedNeed := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 
 	s.Replace(protocol.LocalDeviceID, localHave)
@@ -340,10 +340,10 @@ func TestUpdateToInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 
 	s.Replace(protocol.LocalDeviceID, localHave)
@@ -355,7 +355,7 @@ func TestUpdateToInvalid(t *testing.T) {
 		t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
 	}
 
-	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagInvalid}
+	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Invalid: true}
 	s.Update(protocol.LocalDeviceID, localHave[1:2])
 
 	have = fileList(haveList(s, protocol.LocalDeviceID))
@@ -372,16 +372,16 @@ func TestInvalidAvailability(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 
 	s.Replace(remoteDevice0, remote0Have)
@@ -410,17 +410,17 @@ func TestGlobalReset(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local)
@@ -448,23 +448,23 @@ func TestNeed(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	shouldNeed := []protocol.FileInfo{
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local)
@@ -486,18 +486,18 @@ func TestLocalVersion(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	local2 := []protocol.FileInfo{
 		local1[0],
 		// [1] deleted
 		local1[2],
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local1)
@@ -515,17 +515,17 @@ func TestListDropFolder(t *testing.T) {
 
 	s0 := db.NewFileSet("test0", ldb)
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 	s0.Replace(protocol.LocalDeviceID, local1)
 
 	s1 := db.NewFileSet("test1", ldb)
 	local2 := []protocol.FileInfo{
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "f", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "f", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
 	}
 	s1.Replace(remoteDevice0, local2)
 
@@ -566,24 +566,24 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test1", ldb)
 
 	rem0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
 	}
 	s.Replace(remoteDevice0, rem0)
 
 	rem1 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
 	}
 	s.Replace(remoteDevice1, rem1)
 
 	total := fileList{
 		// There's a valid copy of each file, so it should be merged
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
 	}
 
 	need := fileList(needList(s, protocol.LocalDeviceID))
@@ -609,7 +609,7 @@ func TestLongPath(t *testing.T) {
 	name := b.String() // 5000 characters
 
 	local := []protocol.FileInfo{
-		{Name: string(name), Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	s.Replace(protocol.LocalDeviceID, local)
@@ -633,7 +633,7 @@ func TestCommitted(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: string("file"), Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	// Adding a file should increase the counter
@@ -659,12 +659,12 @@ func TestCommitted(t *testing.T) {
 
 func BenchmarkUpdateOneFile(b *testing.B) {
 	local0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
 		// A longer name is more realistic and causes more allocations
-		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
+		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 
 	ldb, err := db.Open("testdata/benchmarkupdate.db")

lib/db/structs.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../script/protofmt.go structs.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. structs.proto
+
+package db
+
+import (
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func (f FileInfoTruncated) String() string {
+	return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}",
+		f.Name, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions)
+}
+
+func (f FileInfoTruncated) IsDeleted() bool {
+	return f.Deleted
+}
+
+func (f FileInfoTruncated) IsInvalid() bool {
+	return f.Invalid
+}
+
+func (f FileInfoTruncated) IsDirectory() bool {
+	return f.Type == protocol.FileInfoTypeDirectory
+}
+
+func (f FileInfoTruncated) IsSymlink() bool {
+	switch f.Type {
+	case protocol.FileInfoTypeSymlinkDirectory, protocol.FileInfoTypeSymlinkFile, protocol.FileInfoTypeSymlinkUnknown:
+		return true
+	default:
+		return false
+	}
+}
+
+func (f FileInfoTruncated) HasPermissionBits() bool {
+	return !f.NoPermissions
+}
+
+func (f FileInfoTruncated) FileSize() int64 {
+	if f.IsDirectory() || f.IsDeleted() {
+		return 128
+	}
+	return f.Size
+}
+
+func (f FileInfoTruncated) FileName() string {
+	return f.Name
+}

lib/db/structs.pb.go

@@ -0,0 +1,914 @@
+// Code generated by protoc-gen-gogo.
+// source: structs.proto
+// DO NOT EDIT!
+
+/*
+	Package db is a generated protocol buffer package.
+
+	It is generated from these files:
+		structs.proto
+
+	It has these top-level messages:
+		FileVersion
+		VersionList
+		FileInfoTruncated
+*/
+package db
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+import protocol "github.com/syncthing/syncthing/lib/protocol"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.GoGoProtoPackageIsVersion1
+
+type FileVersion struct {
+	Version protocol.Vector `protobuf:"bytes,1,opt,name=version" json:"version"`
+	Device  []byte          `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
+}
+
+func (m *FileVersion) Reset()                    { *m = FileVersion{} }
+func (m *FileVersion) String() string            { return proto.CompactTextString(m) }
+func (*FileVersion) ProtoMessage()               {}
+func (*FileVersion) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{0} }
+
+type VersionList struct {
+	Versions []FileVersion `protobuf:"bytes,1,rep,name=versions" json:"versions"`
+}
+
+func (m *VersionList) Reset()                    { *m = VersionList{} }
+func (*VersionList) ProtoMessage()               {}
+func (*VersionList) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{1} }
+
+// Must be the same as FileInfo but without the blocks field
+type FileInfoTruncated struct {
+	Name          string                `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Type          protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
+	Size          int64                 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
+	Permissions   uint32                `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
+	Modified      int64                 `protobuf:"varint,5,opt,name=modified,proto3" json:"modified,omitempty"`
+	Deleted       bool                  `protobuf:"varint,6,opt,name=deleted,proto3" json:"deleted,omitempty"`
+	Invalid       bool                  `protobuf:"varint,7,opt,name=invalid,proto3" json:"invalid,omitempty"`
+	NoPermissions bool                  `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
+	Version       protocol.Vector       `protobuf:"bytes,9,opt,name=version" json:"version"`
+	LocalVersion  int64                 `protobuf:"varint,10,opt,name=local_version,json=localVersion,proto3" json:"local_version,omitempty"`
+}
+
+func (m *FileInfoTruncated) Reset()                    { *m = FileInfoTruncated{} }
+func (*FileInfoTruncated) ProtoMessage()               {}
+func (*FileInfoTruncated) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{2} }
+
+func init() {
+	proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
+	proto.RegisterType((*VersionList)(nil), "db.VersionList")
+	proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated")
+}
+func (m *FileVersion) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *FileVersion) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	data[i] = 0xa
+	i++
+	i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
+	n1, err := m.Version.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n1
+	if len(m.Device) > 0 {
+		data[i] = 0x12
+		i++
+		i = encodeVarintStructs(data, i, uint64(len(m.Device)))
+		i += copy(data[i:], m.Device)
+	}
+	return i, nil
+}
+
+func (m *VersionList) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *VersionList) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for _, msg := range m.Versions {
+			data[i] = 0xa
+			i++
+			i = encodeVarintStructs(data, i, uint64(msg.ProtoSize()))
+			n, err := msg.MarshalTo(data[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	return i, nil
+}
+
+func (m *FileInfoTruncated) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *FileInfoTruncated) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Name) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintStructs(data, i, uint64(len(m.Name)))
+		i += copy(data[i:], m.Name)
+	}
+	if m.Type != 0 {
+		data[i] = 0x10
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Type))
+	}
+	if m.Size != 0 {
+		data[i] = 0x18
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Size))
+	}
+	if m.Permissions != 0 {
+		data[i] = 0x20
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Permissions))
+	}
+	if m.Modified != 0 {
+		data[i] = 0x28
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Modified))
+	}
+	if m.Deleted {
+		data[i] = 0x30
+		i++
+		if m.Deleted {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	if m.Invalid {
+		data[i] = 0x38
+		i++
+		if m.Invalid {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	if m.NoPermissions {
+		data[i] = 0x40
+		i++
+		if m.NoPermissions {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	data[i] = 0x4a
+	i++
+	i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
+	n2, err := m.Version.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n2
+	if m.LocalVersion != 0 {
+		data[i] = 0x50
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.LocalVersion))
+	}
+	return i, nil
+}
+
+func encodeFixed64Structs(data []byte, offset int, v uint64) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	data[offset+4] = uint8(v >> 32)
+	data[offset+5] = uint8(v >> 40)
+	data[offset+6] = uint8(v >> 48)
+	data[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Structs(data []byte, offset int, v uint32) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintStructs(data []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		data[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	data[offset] = uint8(v)
+	return offset + 1
+}
+func (m *FileVersion) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = m.Version.ProtoSize()
+	n += 1 + l + sovStructs(uint64(l))
+	l = len(m.Device)
+	if l > 0 {
+		n += 1 + l + sovStructs(uint64(l))
+	}
+	return n
+}
+
+func (m *VersionList) ProtoSize() (n int) {
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for _, e := range m.Versions {
+			l = e.ProtoSize()
+			n += 1 + l + sovStructs(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *FileInfoTruncated) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = len(m.Name)
+	if l > 0 {
+		n += 1 + l + sovStructs(uint64(l))
+	}
+	if m.Type != 0 {
+		n += 1 + sovStructs(uint64(m.Type))
+	}
+	if m.Size != 0 {
+		n += 1 + sovStructs(uint64(m.Size))
+	}
+	if m.Permissions != 0 {
+		n += 1 + sovStructs(uint64(m.Permissions))
+	}
+	if m.Modified != 0 {
+		n += 1 + sovStructs(uint64(m.Modified))
+	}
+	if m.Deleted {
+		n += 2
+	}
+	if m.Invalid {
+		n += 2
+	}
+	if m.NoPermissions {
+		n += 2
+	}
+	l = m.Version.ProtoSize()
+	n += 1 + l + sovStructs(uint64(l))
+	if m.LocalVersion != 0 {
+		n += 1 + sovStructs(uint64(m.LocalVersion))
+	}
+	return n
+}
+
+func sovStructs(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozStructs(x uint64) (n int) {
+	return sovStructs(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *FileVersion) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FileVersion: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FileVersion: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Device = append(m.Device[:0], data[iNdEx:postIndex]...)
+			if m.Device == nil {
+				m.Device = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *VersionList) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: VersionList: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: VersionList: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Versions = append(m.Versions, FileVersion{})
+			if err := m.Versions[len(m.Versions)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *FileInfoTruncated) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FileInfoTruncated: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FileInfoTruncated: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Name = string(data[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
+			}
+			m.Type = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Type |= (protocol.FileInfoType(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType)
+			}
+			m.Size = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Size |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Permissions", wireType)
+			}
+			m.Permissions = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Permissions |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 5:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Modified", wireType)
+			}
+			m.Modified = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Modified |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 6:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Deleted = bool(v != 0)
+		case 7:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Invalid = bool(v != 0)
+		case 8:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field NoPermissions", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.NoPermissions = bool(v != 0)
+		case 9:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 10:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field LocalVersion", wireType)
+			}
+			m.LocalVersion = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.LocalVersion |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipStructs(data []byte) (n int, err error) {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if data[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthStructs
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowStructs
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := data[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipStructs(data[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthStructs = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowStructs   = fmt.Errorf("proto: integer overflow")
+)
+
+var fileDescriptorStructs = []byte{
+	// 401 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x4f, 0xcb, 0xd3, 0x30,
+	0x1c, 0x6e, 0xb7, 0xba, 0xf5, 0x4d, 0xdf, 0x4e, 0x0d, 0x32, 0xca, 0x0e, 0xdd, 0x98, 0x08, 0x22,
+	0xd8, 0xe9, 0xc4, 0x8b, 0xc7, 0x1d, 0x06, 0x82, 0x07, 0x29, 0x32, 0x8f, 0xa3, 0x4d, 0xb2, 0x2e,
+	0xd0, 0x26, 0xa5, 0x49, 0x07, 0xf3, 0x93, 0x78, 0xdc, 0xc7, 0xd9, 0xd1, 0x2f, 0xa0, 0xe8, 0xfc,
+	0x22, 0x66, 0x49, 0x3b, 0x7b, 0x7c, 0x0f, 0x81, 0xdf, 0x93, 0xe7, 0xcf, 0xef, 0x21, 0x01, 0xbe,
+	0x90, 0x55, 0x8d, 0xa4, 0x88, 0xca, 0x8a, 0x4b, 0x0e, 0x7b, 0x38, 0x9d, 0xbc, 0xce, 0xa8, 0xdc,
+	0xd7, 0x69, 0x84, 0x78, 0xb1, 0xc8, 0x78, 0xc6, 0x17, 0x9a, 0x4a, 0xeb, 0x9d, 0x46, 0x1a, 0xe8,
+	0xc9, 0x58, 0x26, 0xef, 0x3b, 0x72, 0x71, 0x64, 0x48, 0xee, 0x29, 0xcb, 0x3a, 0x53, 0x4e, 0x53,
+	0x93, 0x80, 0x78, 0xbe, 0x48, 0x49, 0x69, 0x6c, 0xf3, 0xaf, 0xc0, 0x5b, 0xd3, 0x9c, 0x6c, 0x48,
+	0x25, 0x28, 0x67, 0xf0, 0x0d, 0x18, 0x1e, 0xcc, 0x18, 0xd8, 0x33, 0xfb, 0xa5, 0xb7, 0x7c, 0x12,
+	0xb5, 0xa6, 0x68, 0x43, 0x90, 0xe4, 0xd5, 0xca, 0x39, 0xff, 0x9a, 0x5a, 0x71, 0x2b, 0x83, 0x63,
+	0x30, 0xc0, 0xe4, 0x40, 0x11, 0x09, 0x7a, 0xca, 0x70, 0x1f, 0x37, 0x68, 0xbe, 0x06, 0x5e, 0x13,
+	0xfa, 0x89, 0x0a, 0x09, 0xdf, 0x02, 0xb7, 0x71, 0x08, 0x95, 0xdc, 0x57, 0xc9, 0x8f, 0x23, 0x9c,
+	0x46, 0x9d, 0xdd, 0x4d, 0xf0, 0x4d, 0xf6, 0xc1, 0xf9, 0x7e, 0x9a, 0x5a, 0xf3, 0x9f, 0x3d, 0xf0,
+	0xf4, 0xaa, 0xfa, 0xc8, 0x76, 0xfc, 0x4b, 0x55, 0x33, 0x94, 0x48, 0x82, 0x21, 0x04, 0x0e, 0x4b,
+	0x0a, 0xa2, 0x4b, 0xde, 0xc5, 0x7a, 0x86, 0xaf, 0x80, 0x23, 0x8f, 0xa5, 0xe9, 0x31, 0x5a, 0x8e,
+	0xff, 0x17, 0xbf, 0xd9, 0x15, 0x1b, 0x6b, 0xcd, 0xd5, 0x2f, 0xe8, 0x37, 0x12, 0xf4, 0x95, 0xb6,
+	0x1f, 0xeb, 0x19, 0xce, 0x80, 0x57, 0x92, 0xaa, 0xa0, 0xc2, 0xb4, 0x74, 0x14, 0xe5, 0xc7, 0xdd,
+	0x2b, 0x38, 0x01, 0x6e, 0xc1, 0x31, 0xdd, 0x51, 0x82, 0x83, 0x47, 0xda, 0x79, 0xc3, 0x30, 0x00,
+	0x43, 0x4c, 0x72, 0xa2, 0xca, 0x05, 0x03, 0x45, 0xb9, 0x71, 0x0b, 0xaf, 0x0c, 0x65, 0x87, 0x24,
+	0xa7, 0x38, 0x18, 0x1a, 0xa6, 0x81, 0xf0, 0x05, 0x18, 0x31, 0xbe, 0xed, 0x2e, 0x75, 0xb5, 0xc0,
+	0x67, 0xfc, 0x73, 0x67, 0x6d, 0xe7, 0x53, 0xee, 0x1e, 0xf6, 0x29, 0xcf, 0x81, 0x9f, 0x73, 0x94,
+	0xe4, 0xdb, 0xd6, 0x07, 0x74, 0xdb, 0x7b, 0x7d, 0xd9, 0xbc, 0xb7, 0x79, 0xdf, 0xd5, 0xb3, 0xf3,
+	0x9f, 0xd0, 0x3a, 0x5f, 0x42, 0xfb, 0x87, 0x3a, 0xbf, 0x2f, 0xa1, 0x75, 0xfa, 0x1b, 0xda, 0xe9,
+	0x40, 0x2f, 0x78, 0xf7, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe4, 0xb1, 0x7f, 0x07, 0x98, 0x02, 0x00,
+	0x00,
+}

lib/db/structs.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package db;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "github.com/syncthing/syncthing/lib/protocol/bep.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+
+message FileVersion {
+    protocol.Vector version = 1 [(gogoproto.nullable) = false];
+    bytes           device  = 2;
+}
+
+message VersionList {
+    option (gogoproto.goproto_stringer) = false;
+    repeated FileVersion versions = 1 [(gogoproto.nullable) = false];
+}
+
+// Must be the same as FileInfo but without the blocks field
+message FileInfoTruncated {
+    option (gogoproto.goproto_stringer) = false;
+    string                name           = 1;
+    protocol.FileInfoType type           = 2;
+    int64                 size           = 3;
+    uint32                permissions    = 4;
+    int64                 modified       = 5;
+    bool                  deleted        = 6;
+    bool                  invalid        = 7;
+    bool                  no_permissions = 8;
+    protocol.Vector       version        = 9 [(gogoproto.nullable) = false];
+    int64                 local_version  = 10;
+}

lib/db/truncated.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"github.com/calmh/xdr"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-type FileInfoTruncated struct {
-	protocol.FileInfo
-}
-
-func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
-	return o.UnmarshalXDRFrom(&xdr.Unmarshaller{Data: bs})
-}
-
-func (o *FileInfoTruncated) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.Name = u.UnmarshalStringMax(8192)
-	o.Flags = u.UnmarshalUint32()
-	o.Modified = int64(u.UnmarshalUint64())
-	(&o.Version).UnmarshalXDRFrom(u)
-	o.LocalVersion = int64(u.UnmarshalUint64())
-	_BlocksSize := int(u.UnmarshalUint32())
-	if _BlocksSize < 0 {
-		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
-	} else if _BlocksSize == 0 {
-		o.Blocks = nil
-	} else {
-		if _BlocksSize > 10000000 {
-			return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
-		}
-		for i := 0; i < _BlocksSize; i++ {
-			size := int64(u.UnmarshalUint32())
-			o.CachedSize += size
-			u.UnmarshalBytes()
-		}
-	}
-
-	return u.Error
-}
-
-func BlocksToSize(num int) int64 {
-	if num < 2 {
-		return protocol.BlockSize / 2
-	}
-	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
-}

lib/discover/local.go

@@ -4,10 +4,14 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
+//go:generate go run ../../script/protofmt.go local.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. local.proto
+
 package discover
 
 import (
 	"bytes"
+	"encoding/binary"
 	"encoding/hex"
 	"io"
 	"net"
@@ -38,6 +42,8 @@ type localClient struct {
 const (
 	BroadcastInterval = 30 * time.Second
 	CacheLifeTime     = 3 * BroadcastInterval
+	Magic             = uint32(0x2EA7D90B) // same as in BEP
+	v13Magic          = uint32(0x7D79BC40) // previous version
 )
 
 func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister) (FinderService, error) {
@@ -107,25 +113,19 @@ func (c *localClient) Error() error {
 }
 
 func (c *localClient) announcementPkt() Announce {
-	var addrs []Address
-	for _, addr := range c.addrList.AllAddresses() {
-		addrs = append(addrs, Address{
-			URL: addr,
-		})
-	}
-
 	return Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			ID:        c.myID[:],
-			Addresses: addrs,
-		},
+		ID:        c.myID[:],
+		Addresses: c.addrList.AllAddresses(),
 	}
 }
 
 func (c *localClient) sendLocalAnnouncements() {
+	msg := make([]byte, 4)
+	binary.BigEndian.PutUint32(msg, Magic)
+
 	var pkt = c.announcementPkt()
-	msg := pkt.MustMarshalXDR()
+	bs, _ := pkt.Marshal()
+	msg = append(msg, bs...)
 
 	for {
 		c.beacon.Send(msg)
@@ -138,26 +138,44 @@ func (c *localClient) sendLocalAnnouncements() {
 }
 
 func (c *localClient) recvAnnouncements(b beacon.Interface) {
+	warnedAbout := make(map[string]bool)
 	for {
 		buf, addr := b.Recv()
+		if len(buf) < 4 {
+			l.Debugf("discover: short packet from %s")
+			continue
+		}
+
+		magic := binary.BigEndian.Uint32(buf)
+		switch magic {
+		case Magic:
+			// All good
+
+		case v13Magic:
+			// Old version
+			if !warnedAbout[addr.String()] {
+				l.Warnf("Incompatible (v0.13) local discovery packet from %v - upgrade that device to connect", addr)
+				warnedAbout[addr.String()] = true
+			}
+			continue
+
+		default:
+			l.Debugf("discover: Incorrect magic %x from %s", magic, addr)
+			continue
+		}
 
 		var pkt Announce
-		err := pkt.UnmarshalXDR(buf)
+		err := pkt.Unmarshal(buf[4:])
 		if err != nil && err != io.EOF {
 			l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
 			continue
 		}
 
-		if pkt.Magic != AnnouncementMagic {
-			l.Debugf("discover: Incorrect magic from %s: %s != %s", addr, pkt.Magic, AnnouncementMagic)
-			continue
-		}
-
-		l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
+		l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.ID))
 
 		var newDevice bool
-		if !bytes.Equal(pkt.This.ID, c.myID[:]) {
-			newDevice = c.registerDevice(addr, pkt.This)
+		if !bytes.Equal(pkt.ID, c.myID[:]) {
+			newDevice = c.registerDevice(addr, pkt)
 		}
 
 		if newDevice {
@@ -171,7 +189,7 @@ func (c *localClient) recvAnnouncements(b beacon.Interface) {
 	}
 }
 
-func (c *localClient) registerDevice(src net.Addr, device Device) bool {
+func (c *localClient) registerDevice(src net.Addr, device Announce) bool {
 	var id protocol.DeviceID
 	copy(id[:], device.ID)
 
@@ -186,7 +204,7 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 	l.Debugln("discover: Registering addresses for", id)
 	var validAddresses []string
 	for _, addr := range device.Addresses {
-		u, err := url.Parse(addr.URL)
+		u, err := url.Parse(addr)
 		if err != nil {
 			continue
 		}
@@ -197,6 +215,21 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 		}
 
 		if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
+			srcAddr, err := net.ResolveTCPAddr("tcp", src.String())
+			if err != nil {
+				continue
+			}
+
+			// Do not use IPv6 source address if requested scheme is tcp4
+			if u.Scheme == "tcp4" && srcAddr.IP.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 source address if requested scheme is tcp6
+			if u.Scheme == "tcp6" && srcAddr.IP.To4() != nil {
+				continue
+			}
+
 			host, _, err := net.SplitHostPort(src.String())
 			if err != nil {
 				continue
@@ -204,10 +237,10 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 			u.Host = net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port))
 			l.Debugf("discover: Reconstructed URL is %#v", u)
 			validAddresses = append(validAddresses, u.String())
-			l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr.URL, u.String())
+			l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr, u.String())
 		} else {
-			validAddresses = append(validAddresses, addr.URL)
-			l.Debugf("discover: Accepted address %s verbatim", addr.URL)
+			validAddresses = append(validAddresses, addr)
+			l.Debugf("discover: Accepted address %s verbatim", addr)
 		}
 	}
 

lib/discover/local.pb.go

@@ -0,0 +1,368 @@
+// Code generated by protoc-gen-gogo.
+// source: local.proto
+// DO NOT EDIT!
+
+/*
+	Package discover is a generated protocol buffer package.
+
+	It is generated from these files:
+		local.proto
+
+	It has these top-level messages:
+		Announce
+*/
+package discover
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.GoGoProtoPackageIsVersion1
+
+type Announce struct {
+	ID        []byte   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	Addresses []string `protobuf:"bytes,2,rep,name=addresses" json:"addresses,omitempty"`
+}
+
+func (m *Announce) Reset()                    { *m = Announce{} }
+func (m *Announce) String() string            { return proto.CompactTextString(m) }
+func (*Announce) ProtoMessage()               {}
+func (*Announce) Descriptor() ([]byte, []int) { return fileDescriptorLocal, []int{0} }
+
+func init() {
+	proto.RegisterType((*Announce)(nil), "discover.Announce")
+}
+func (m *Announce) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *Announce) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.ID) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintLocal(data, i, uint64(len(m.ID)))
+		i += copy(data[i:], m.ID)
+	}
+	if len(m.Addresses) > 0 {
+		for _, s := range m.Addresses {
+			data[i] = 0x12
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				data[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			data[i] = uint8(l)
+			i++
+			i += copy(data[i:], s)
+		}
+	}
+	return i, nil
+}
+
+func encodeFixed64Local(data []byte, offset int, v uint64) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	data[offset+4] = uint8(v >> 32)
+	data[offset+5] = uint8(v >> 40)
+	data[offset+6] = uint8(v >> 48)
+	data[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Local(data []byte, offset int, v uint32) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintLocal(data []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		data[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	data[offset] = uint8(v)
+	return offset + 1
+}
+func (m *Announce) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = len(m.ID)
+	if l > 0 {
+		n += 1 + l + sovLocal(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, s := range m.Addresses {
+			l = len(s)
+			n += 1 + l + sovLocal(uint64(l))
+		}
+	}
+	return n
+}
+
+func sovLocal(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozLocal(x uint64) (n int) {
+	return sovLocal(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *Announce) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowLocal
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Announce: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Announce: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthLocal
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ID = append(m.ID[:0], data[iNdEx:postIndex]...)
+			if m.ID == nil {
+				m.ID = []byte{}
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthLocal
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, string(data[iNdEx:postIndex]))
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipLocal(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthLocal
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipLocal(data []byte) (n int, err error) {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowLocal
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if data[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthLocal
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowLocal
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := data[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipLocal(data[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthLocal = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowLocal   = fmt.Errorf("proto: integer overflow")
+)
+
+var fileDescriptorLocal = []byte{
+	// 161 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xc9, 0x4f, 0x4e,
+	0xcc, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x48, 0xc9, 0x2c, 0x4e, 0xce, 0x2f, 0x4b,
+	0x2d, 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf,
+	0x4f, 0xcf, 0xd7, 0x07, 0x2b, 0x48, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x51,
+	0xc9, 0x81, 0x8b, 0xc3, 0x31, 0x2f, 0x2f, 0xbf, 0x34, 0x2f, 0x39, 0x55, 0x48, 0x8c, 0x8b, 0x29,
+	0x33, 0x45, 0x82, 0x51, 0x81, 0x51, 0x83, 0xc7, 0x89, 0xed, 0xd1, 0x3d, 0x79, 0x26, 0x4f, 0x97,
+	0x20, 0xa0, 0x88, 0x90, 0x0c, 0x17, 0x67, 0x62, 0x4a, 0x4a, 0x51, 0x6a, 0x71, 0x71, 0x6a, 0xb1,
+	0x04, 0x93, 0x02, 0xb3, 0x06, 0x67, 0x10, 0x42, 0xc0, 0x49, 0xe4, 0xc4, 0x43, 0x39, 0x86, 0x13,
+	0x8f, 0xe4, 0x18, 0x2f, 0x00, 0xf1, 0x83, 0x47, 0x72, 0x0c, 0x0b, 0x1e, 0xcb, 0x31, 0x26, 0xb1,
+	0x81, 0x8d, 0x37, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xc9, 0xec, 0xea, 0xbc, 0xa6, 0x00, 0x00,
+	0x00,
+}

lib/discover/local.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+package discover;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+
+message Announce {
+    bytes           id        = 1 [(gogoproto.customname) = "ID"];
+    repeated string addresses = 2;
+}

lib/discover/localpackets.go

@@ -1,29 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o localpackets_xdr.go localpackets.go
-
-package discover
-
-const (
-	AnnouncementMagic = 0x7D79BC40
-)
-
-type Announce struct {
-	Magic uint32
-	This  Device
-	Extra []Device // max:16
-}
-
-type Device struct {
-	ID        []byte    // max:32
-	Addresses []Address // max:16
-}
-
-type Address struct {
-	URL string // max:2083
-}

lib/discover/localpackets_xdr.go

@@ -1,246 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package discover
-
-import (
-	"github.com/calmh/xdr"
-)
-
-/*
-
-Announce Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Magic                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Device Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                        Number of Extra                        |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                Zero or more Device Structures                 \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Announce {
-	unsigned int Magic;
-	Device This;
-	Device Extra<16>;
-}
-
-*/
-
-func (o Announce) XDRSize() int {
-	return 4 +
-		o.This.XDRSize() +
-		4 + xdr.SizeOfSlice(o.Extra)
-}
-
-func (o Announce) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Announce) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Announce) MarshalXDRInto(m *xdr.Marshaller) error {
-	m.MarshalUint32(o.Magic)
-	if err := o.This.MarshalXDRInto(m); err != nil {
-		return err
-	}
-	if l := len(o.Extra); l > 16 {
-		return xdr.ElementSizeExceeded("Extra", l, 16)
-	}
-	m.MarshalUint32(uint32(len(o.Extra)))
-	for i := range o.Extra {
-		if err := o.Extra[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *Announce) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Announce) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.Magic = u.UnmarshalUint32()
-	(&o.This).UnmarshalXDRFrom(u)
-	_ExtraSize := int(u.UnmarshalUint32())
-	if _ExtraSize < 0 {
-		return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
-	} else if _ExtraSize == 0 {
-		o.Extra = nil
-	} else {
-		if _ExtraSize > 16 {
-			return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
-		}
-		if _ExtraSize <= len(o.Extra) {
-			o.Extra = o.Extra[:_ExtraSize]
-		} else {
-			o.Extra = make([]Device, _ExtraSize)
-		}
-		for i := range o.Extra {
-			(&o.Extra[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}
-
-/*
-
-Device Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                   ID (length + padded data)                   \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                      Number of Addresses                      |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                Zero or more Address Structures                \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Device {
-	opaque ID<32>;
-	Address Addresses<16>;
-}
-
-*/
-
-func (o Device) XDRSize() int {
-	return 4 + len(o.ID) + xdr.Padding(len(o.ID)) +
-		4 + xdr.SizeOfSlice(o.Addresses)
-}
-
-func (o Device) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Device) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Device) MarshalXDRInto(m *xdr.Marshaller) error {
-	if l := len(o.ID); l > 32 {
-		return xdr.ElementSizeExceeded("ID", l, 32)
-	}
-	m.MarshalBytes(o.ID)
-	if l := len(o.Addresses); l > 16 {
-		return xdr.ElementSizeExceeded("Addresses", l, 16)
-	}
-	m.MarshalUint32(uint32(len(o.Addresses)))
-	for i := range o.Addresses {
-		if err := o.Addresses[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *Device) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Device) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.ID = u.UnmarshalBytesMax(32)
-	_AddressesSize := int(u.UnmarshalUint32())
-	if _AddressesSize < 0 {
-		return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
-	} else if _AddressesSize == 0 {
-		o.Addresses = nil
-	} else {
-		if _AddressesSize > 16 {
-			return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
-		}
-		if _AddressesSize <= len(o.Addresses) {
-			o.Addresses = o.Addresses[:_AddressesSize]
-		} else {
-			o.Addresses = make([]Address, _AddressesSize)
-		}
-		for i := range o.Addresses {
-			(&o.Addresses[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}
-
-/*
-
-Address Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                  URL (length + padded data)                   \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Address {
-	string URL<2083>;
-}
-
-*/
-
-func (o Address) XDRSize() int {
-	return 4 + len(o.URL) + xdr.Padding(len(o.URL))
-}
-
-func (o Address) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Address) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Address) MarshalXDRInto(m *xdr.Marshaller) error {
-	if l := len(o.URL); l > 2083 {
-		return xdr.ElementSizeExceeded("URL", l, 2083)
-	}
-	m.MarshalString(o.URL)
-	return m.Error
-}
-
-func (o *Address) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Address) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.URL = u.UnmarshalStringMax(2083)
-	return u.Error
-}

lib/events/events.go

@@ -111,16 +111,20 @@ func (t EventType) MarshalText() ([]byte, error) {
 const BufferSize = 64
 
 type Logger struct {
-	subs   []*Subscription
-	nextID int
-	mutex  sync.Mutex
+	subs                []*Subscription
+	nextSubscriptionIDs []int
+	nextGlobalID        int
+	mutex               sync.Mutex
 }
 
 type Event struct {
-	ID   int         `json:"id"`
-	Time time.Time   `json:"time"`
-	Type EventType   `json:"type"`
-	Data interface{} `json:"data"`
+	// Per-subscription sequential event ID. Named "id" for backwards compatibility with the REST API
+	SubscriptionID int `json:"id"`
+	// Global ID of the event across all subscriptions
+	GlobalID int         `json:"globalID"`
+	Time     time.Time   `json:"time"`
+	Type     EventType   `json:"type"`
+	Data     interface{} `json:"data"`
 }
 
 type Subscription struct {
@@ -144,16 +148,21 @@ func NewLogger() *Logger {
 
 func (l *Logger) Log(t EventType, data interface{}) {
 	l.mutex.Lock()
-	dl.Debugln("log", l.nextID, t, data)
-	l.nextID++
+	dl.Debugln("log", l.nextGlobalID, t, data)
+	l.nextGlobalID++
+
 	e := Event{
-		ID:   l.nextID,
-		Time: time.Now(),
-		Type: t,
-		Data: data,
+		GlobalID: l.nextGlobalID,
+		Time:     time.Now(),
+		Type:     t,
+		Data:     data,
 	}
-	for _, s := range l.subs {
+
+	for i, s := range l.subs {
 		if s.mask&t != 0 {
+			e.SubscriptionID = l.nextSubscriptionIDs[i]
+			l.nextSubscriptionIDs[i]++
+
 			select {
 			case s.events <- e:
 			default:
@@ -182,6 +191,7 @@ func (l *Logger) Subscribe(mask EventType) *Subscription {
 	}
 
 	l.subs = append(l.subs, s)
+	l.nextSubscriptionIDs = append(l.nextSubscriptionIDs, 1)
 	l.mutex.Unlock()
 	return s
 }
@@ -192,9 +202,15 @@ func (l *Logger) Unsubscribe(s *Subscription) {
 	for i, ss := range l.subs {
 		if s == ss {
 			last := len(l.subs) - 1
+
 			l.subs[i] = l.subs[last]
 			l.subs[last] = nil
 			l.subs = l.subs[:last]
+
+			l.nextSubscriptionIDs[i] = l.nextSubscriptionIDs[last]
+			l.nextSubscriptionIDs[last] = 0
+			l.nextSubscriptionIDs = l.nextSubscriptionIDs[:last]
+
 			break
 		}
 	}
@@ -234,7 +250,7 @@ type bufferedSubscription struct {
 	sub  *Subscription
 	buf  []Event
 	next int
-	cur  int
+	cur  int // Current SubscriptionID
 	mut  sync.Mutex
 	cond *stdsync.Cond
 }
@@ -270,7 +286,7 @@ func (s *bufferedSubscription) pollingLoop() {
 		s.mut.Lock()
 		s.buf[s.next] = ev
 		s.next = (s.next + 1) % len(s.buf)
-		s.cur = ev.ID
+		s.cur = ev.SubscriptionID
 		s.cond.Broadcast()
 		s.mut.Unlock()
 	}
@@ -285,12 +301,12 @@ func (s *bufferedSubscription) Since(id int, into []Event) []Event {
 	}
 
 	for i := s.next; i < len(s.buf); i++ {
-		if s.buf[i].ID > id {
+		if s.buf[i].SubscriptionID > id {
 			into = append(into, s.buf[i])
 		}
 	}
 	for i := 0; i < s.next; i++ {
-		if s.buf[i].ID > id {
+		if s.buf[i].SubscriptionID > id {
 			into = append(into, s.buf[i])
 		}
 	}

lib/events/events_test.go

@@ -128,7 +128,7 @@ func TestUnsubscribe(t *testing.T) {
 	}
 }
 
-func TestIDs(t *testing.T) {
+func TestGlobalIDs(t *testing.T) {
 	l := events.NewLogger()
 
 	s := l.Subscribe(events.AllEvents)
@@ -144,7 +144,7 @@ func TestIDs(t *testing.T) {
 	if ev.Data.(string) != "foo" {
 		t.Fatal("Incorrect event:", ev)
 	}
-	id := ev.ID
+	id := ev.GlobalID
 
 	ev, err = s.Poll(timeout)
 	if err != nil {
@@ -153,8 +153,48 @@ func TestIDs(t *testing.T) {
 	if ev.Data.(string) != "bar" {
 		t.Fatal("Incorrect event:", ev)
 	}
-	if ev.ID != id+1 {
-		t.Fatalf("ID not incremented (%d != %d)", ev.ID, id+1)
+	if ev.GlobalID != id+1 {
+		t.Fatalf("ID not incremented (%d != %d)", ev.GlobalID, id+1)
+	}
+}
+
+func TestSubscriptionIDs(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.DeviceConnected)
+	defer l.Unsubscribe(s)
+
+	l.Log(events.DeviceDisconnected, "a")
+	l.Log(events.DeviceConnected, "b")
+	l.Log(events.DeviceConnected, "c")
+	l.Log(events.DeviceDisconnected, "d")
+
+	ev, err := s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	if ev.GlobalID != 2 {
+		t.Fatal("Incorrect GlobalID:", ev.GlobalID)
+	}
+	if ev.SubscriptionID != 1 {
+		t.Fatal("Incorrect SubscriptionID:", ev.SubscriptionID)
+	}
+
+	ev, err = s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.GlobalID != 3 {
+		t.Fatal("Incorrect GlobalID:", ev.GlobalID)
+	}
+	if ev.SubscriptionID != 2 {
+		t.Fatal("Incorrect SubscriptionID:", ev.SubscriptionID)
+	}
+
+	ev, err = s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected error:", err)
 	}
 }
 
@@ -179,11 +219,92 @@ func TestBufferedSub(t *testing.T) {
 	for recv < 10*events.BufferSize {
 		evs := bs.Since(recv, nil)
 		for _, ev := range evs {
-			if ev.ID != recv+1 {
-				t.Fatalf("Incorrect ID; %d != %d", ev.ID, recv+1)
+			if ev.GlobalID != recv+1 {
+				t.Fatalf("Incorrect ID; %d != %d", ev.GlobalID, recv+1)
 			}
-			recv = ev.ID
+			recv = ev.GlobalID
+		}
+	}
+}
+
+func BenchmarkBufferedSub(b *testing.B) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	defer l.Unsubscribe(s)
+	bufferSize := events.BufferSize
+	bs := events.NewBufferedSubscription(s, bufferSize)
+
+	// The coord channel paces the sender according to the receiver,
+	// ensuring that no events are dropped. The benchmark measures sending +
+	// receiving + synchronization overhead.
+
+	coord := make(chan struct{}, bufferSize)
+	for i := 0; i < bufferSize-1; i++ {
+		coord <- struct{}{}
+	}
+
+	// Receive the events
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		recv := 0
+		var evs []events.Event
+		for i := 0; i < b.N; {
+			evs = bs.Since(recv, evs[:0])
+			for _, ev := range evs {
+				if ev.GlobalID != recv+1 {
+					b.Fatal("skipped event", ev.GlobalID, recv)
+				}
+				recv = ev.GlobalID
+				coord <- struct{}{}
+			}
+			i += len(evs)
+		}
+	}()
+
+	// Send the events
+	eventData := map[string]string{
+		"foo":   "bar",
+		"other": "data",
+		"and":   "something else",
+	}
+	for i := 0; i < b.N; i++ {
+		l.Log(events.DeviceConnected, eventData)
+		<-coord
+	}
+
+	<-done
+	b.ReportAllocs()
+}
+
+func TestSinceUsesSubscriptionId(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.DeviceConnected)
+	defer l.Unsubscribe(s)
+	bs := events.NewBufferedSubscription(s, 10*events.BufferSize)
+
+	l.Log(events.DeviceConnected, "a") // SubscriptionID = 1
+	l.Log(events.DeviceDisconnected, "b")
+	l.Log(events.DeviceDisconnected, "c")
+	l.Log(events.DeviceConnected, "d") // SubscriptionID = 2
+
+	// We need to loop for the events, as they may not all have been
+	// delivered to the buffered subscription when we get here.
+	t0 := time.Now()
+	for time.Since(t0) < time.Second {
+		events := bs.Since(0, nil)
+		if len(events) == 2 {
+			break
+		}
+		if len(events) > 2 {
+			t.Fatal("Incorrect number of events:", len(events))
 		}
 	}
 
+	events := bs.Since(1, nil)
+	if len(events) != 1 {
+		t.Fatal("Incorrect number of events:", len(events))
+	}
 }

lib/model/folder.go

@@ -10,7 +10,7 @@ import "time"
 
 type folder struct {
 	stateTracker
-	scan  folderscan
+	scan  folderScanner
 	model *Model
 	stop  chan struct{}
 }

lib/model/folderscan.go

@@ -1,49 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package model
-
-import (
-	"math/rand"
-	"time"
-)
-
-type rescanRequest struct {
-	subdirs []string
-	err     chan error
-}
-
-// bundle all folder scan activity
-type folderscan struct {
-	interval time.Duration
-	timer    *time.Timer
-	now      chan rescanRequest
-	delay    chan time.Duration
-}
-
-func (s *folderscan) reschedule() {
-	if s.interval == 0 {
-		return
-	}
-	// Sleep a random time between 3/4 and 5/4 of the configured interval.
-	sleepNanos := (s.interval.Nanoseconds()*3 + rand.Int63n(2*s.interval.Nanoseconds())) / 4
-	interval := time.Duration(sleepNanos) * time.Nanosecond
-	l.Debugln(s, "next rescan in", interval)
-	s.timer.Reset(interval)
-}
-
-func (s *folderscan) Scan(subdirs []string) error {
-	req := rescanRequest{
-		subdirs: subdirs,
-		err:     make(chan error),
-	}
-	s.now <- req
-	return <-req.err
-}
-
-func (s *folderscan) Delay(next time.Duration) {
-	s.delay <- next
-}

lib/model/folderscanner.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"github.com/syncthing/syncthing/lib/config"
+	"math/rand"
+	"time"
+)
+
+type rescanRequest struct {
+	subdirs []string
+	err     chan error
+}
+
+// bundle all folder scan activity
+type folderScanner struct {
+	interval time.Duration
+	timer    *time.Timer
+	now      chan rescanRequest
+	delay    chan time.Duration
+}
+
+func newFolderScanner(config config.FolderConfiguration) folderScanner {
+	return folderScanner{
+		interval: time.Duration(config.RescanIntervalS) * time.Second,
+		timer:    time.NewTimer(time.Millisecond), // The first scan should be done immediately.
+		now:      make(chan rescanRequest),
+		delay:    make(chan time.Duration),
+	}
+}
+
+func (f *folderScanner) Reschedule() {
+	if f.interval == 0 {
+		return
+	}
+	// Sleep a random time between 3/4 and 5/4 of the configured interval.
+	sleepNanos := (f.interval.Nanoseconds()*3 + rand.Int63n(2*f.interval.Nanoseconds())) / 4
+	interval := time.Duration(sleepNanos) * time.Nanosecond
+	l.Debugln(f, "next rescan in", interval)
+	f.timer.Reset(interval)
+}
+
+func (f *folderScanner) Scan(subdirs []string) error {
+	req := rescanRequest{
+		subdirs: subdirs,
+		err:     make(chan error),
+	}
+	f.now <- req
+	return <-req.err
+}
+
+func (f *folderScanner) Delay(next time.Duration) {
+	f.delay <- next
+}
+
+func (f *folderScanner) HasNoInterval() bool {
+	return f.interval == 0
+}

lib/model/folderstate.go

@@ -46,6 +46,13 @@ type stateTracker struct {
 	changed time.Time
 }
 
+func newStateTracker(id string) stateTracker {
+	return stateTracker{
+		folderID: id,
+		mut:      sync.NewMutex(),
+	}
+}
+
 // setState sets the new folder state, for states other than FolderError.
 func (s *stateTracker) setState(newState folderState) {
 	if newState == FolderError {

lib/model/model.go

@@ -40,25 +40,23 @@ import (
 
 // How many files to send in each Index/IndexUpdate message.
 const (
-	indexTargetSize   = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
-	indexPerFileSize  = 250        // Each FileInfo is approximately this big, in bytes, excluding BlockInfos
-	indexPerBlockSize = 40         // Each BlockInfo is approximately this big
-	indexBatchSize    = 1000       // Either way, don't include more files than this
+	indexTargetSize = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
+	indexBatchSize  = 1000       // Either way, don't include more files than this
 )
 
 type service interface {
-	Serve()
-	Stop()
-	Jobs() ([]string, []string) // In progress, Queued
 	BringToFront(string)
 	DelayScan(d time.Duration)
-	IndexUpdated() // Remote index was updated notification
+	IndexUpdated()              // Remote index was updated notification
+	Jobs() ([]string, []string) // In progress, Queued
 	Scan(subs []string) error
+	Serve()
+	Stop()
 
-	setState(state folderState)
-	setError(err error)
-	clearError()
 	getState() (folderState, time.Time, error)
+	setState(state folderState)
+	clearError()
+	setError(err error)
 }
 
 type Availability struct {
@@ -95,7 +93,7 @@ type Model struct {
 
 	conn              map[protocol.DeviceID]connections.Connection
 	helloMessages     map[protocol.DeviceID]protocol.HelloResult
-	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfigMessage
+	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfig
 	devicePaused      map[protocol.DeviceID]bool
 	deviceDownloads   map[protocol.DeviceID]*deviceDownloadState
 	pmut              sync.RWMutex // protects the above
@@ -110,6 +108,7 @@ var (
 
 // errors returned by the CheckFolderHealth method
 var (
+	errFolderPathEmpty     = errors.New("folder path empty")
 	errFolderPathMissing   = errors.New("folder path missing")
 	errFolderMarkerMissing = errors.New("folder marker missing")
 	errHomeDiskNoSpace     = errors.New("home disk has insufficient free space")
@@ -148,7 +147,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		folderStatRefs:     make(map[string]*stats.FolderStatisticsReference),
 		conn:               make(map[protocol.DeviceID]connections.Connection),
 		helloMessages:      make(map[protocol.DeviceID]protocol.HelloResult),
-		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfigMessage),
+		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfig),
 		devicePaused:       make(map[protocol.DeviceID]bool),
 		deviceDownloads:    make(map[protocol.DeviceID]*deviceDownloadState),
 		fmut:               sync.NewRWMutex(),
@@ -413,7 +412,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		// This might might be more than it really is, because some blocks can be of a smaller size.
 		downloaded = int64(counts[ft.Name] * protocol.BlockSize)
 
-		fileNeed = ft.Size() - downloaded
+		fileNeed = ft.Size - downloaded
 		if fileNeed < 0 {
 			fileNeed = 0
 		}
@@ -435,7 +434,7 @@ func sizeOfFile(f db.FileIntf) (files, deleted int, bytes int64) {
 	} else {
 		deleted++
 	}
-	bytes += f.Size()
+	bytes += f.FileSize()
 	return
 }
 
@@ -547,12 +546,7 @@ func (m *Model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 
 // Index is called when a new device is connected and we receive their full index.
 // Implements the protocol.Model interface.
-func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in Index message", flags)
-		return
-	}
-
+func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -574,7 +568,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 	}
 
 	if !ok {
-		l.Fatalf("Index for nonexistant folder %q", folder)
+		l.Fatalf("Index for nonexistent folder %q", folder)
 	}
 
 	m.pmut.RLock()
@@ -594,12 +588,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 // IndexUpdate is called for incremental updates to connected devices' indexes.
 // Implements the protocol.Model interface.
-func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in IndexUpdate message", flags)
-		return
-	}
-
+func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -615,7 +604,7 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
 	m.fmut.RUnlock()
 
 	if !ok {
-		l.Fatalf("IndexUpdate for nonexistant folder %q", folder)
+		l.Fatalf("IndexUpdate for nonexistent folder %q", folder)
 	}
 
 	m.pmut.RLock()
@@ -650,7 +639,7 @@ func (m *Model) folderSharedWithUnlocked(folder string, deviceID protocol.Device
 	return false
 }
 
-func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfigMessage) {
+func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfig) {
 	// Check the peer device's announced folders against our own. Emits events
 	// for folders that we don't expect (unknown or not shared).
 	// Also, collect a list of folders we do share, and if he's interested in
@@ -659,21 +648,7 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 	tempIndexFolders := make([]string, 0, len(cm.Folders))
 
 	m.fmut.Lock()
-nextFolder:
 	for _, folder := range cm.Folders {
-		cfg := m.folderCfgs[folder.ID]
-
-		if folder.Flags&^protocol.FlagFolderAll != 0 {
-			// There are flags set that we don't know what they mean. Scary!
-			l.Warnf("Device %v: unknown flags for folder %s", deviceID, folder.ID)
-			cfg.Invalid = fmt.Sprintf("Unknown flags from device %v", deviceID)
-			m.cfg.SetFolder(cfg)
-			if srv := m.folderRunners[folder.ID]; srv != nil {
-				srv.setError(fmt.Errorf(cfg.Invalid))
-			}
-			continue nextFolder
-		}
-
 		if !m.folderSharedWithUnlocked(folder.ID, deviceID) {
 			events.Default.Log(events.FolderRejected, map[string]string{
 				"folder":      folder.ID,
@@ -683,7 +658,7 @@ nextFolder:
 			l.Infof("Unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.ID, deviceID)
 			continue
 		}
-		if folder.Flags&protocol.FlagFolderDisabledTempIndexes == 0 {
+		if !folder.DisableTempIndexes {
 			tempIndexFolders = append(tempIndexFolders, folder.ID)
 		}
 	}
@@ -737,7 +712,7 @@ nextFolder:
 					}
 
 					// The introducers' introducers are also our introducers.
-					if device.Flags&protocol.FlagIntroducer != 0 {
+					if device.Introducer {
 						l.Infof("Device %v is now also an introducer", id)
 						newDeviceCfg.Introducer = true
 					}
@@ -808,7 +783,7 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 
 // Request returns the specified data segment by reading it from local disk.
 // Implements the protocol.Model interface.
-func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []protocol.Option, buf []byte) error {
+func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	if offset < 0 {
 		return protocol.ErrInvalid
 	}
@@ -817,14 +792,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
 		return protocol.ErrNoSuchFile
 	}
-
-	if flags != 0 && flags != protocol.FlagFromTemporary {
-		// We currently support only no flags, or FromTemporary flag.
-		return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", flags)
-	}
-
 	if deviceID != protocol.LocalDeviceID {
-		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d f=%d", m, deviceID, folder, name, offset, len(buf), flags)
+		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID, folder, name, offset, len(buf), fromTemporary)
 	}
 	m.fmut.RLock()
 	folderCfg := m.folderCfgs[folder]
@@ -884,7 +853,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 
 	// Only check temp files if the flag is set, and if we are set to advertise
 	// the temp indexes.
-	if flags&protocol.FlagFromTemporary != 0 && !folderCfg.DisableTempIndexes {
+	if fromTemporary && !folderCfg.DisableTempIndexes {
 		tempFn := filepath.Join(folderPath, defTempNamer.TempName(name))
 		if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
 			return nil
@@ -1031,8 +1000,8 @@ func (m *Model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protoco
 }
 
 // GetHello is called when we are about to connect to some remote device.
-func (m *Model) GetHello(protocol.DeviceID) protocol.Version13HelloMessage {
-	return protocol.Version13HelloMessage{
+func (m *Model) GetHello(protocol.DeviceID) protocol.HelloIntf {
+	return &protocol.Hello{
 		DeviceName:    m.deviceName,
 		ClientName:    m.clientName,
 		ClientVersion: m.clientVersion,
@@ -1105,7 +1074,7 @@ func (m *Model) PauseDevice(device protocol.DeviceID) {
 	events.Default.Log(events.DevicePaused, map[string]string{"device": device.String()})
 }
 
-func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
 	if !m.folderSharedWith(folder, device) {
 		return
 	}
@@ -1242,13 +1211,13 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 
 		if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
 			if initial {
-				if err = conn.Index(folder, batch, 0, nil); err != nil {
+				if err = conn.Index(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", deviceID, name, folder, len(batch), currentBatchSize)
 				initial = false
 			} else {
-				if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil {
+				if err = conn.IndexUpdate(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", deviceID, name, folder, len(batch), currentBatchSize)
@@ -1259,17 +1228,17 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 		}
 
 		batch = append(batch, f)
-		currentBatchSize += indexPerFileSize + len(f.Blocks)*indexPerBlockSize
+		currentBatchSize += f.ProtoSize()
 		return true
 	})
 
 	if initial && err == nil {
-		err = conn.Index(folder, batch, 0, nil)
+		err = conn.Index(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch))
 		}
 	} else if len(batch) > 0 && err == nil {
-		err = conn.IndexUpdate(folder, batch, 0, nil)
+		err = conn.IndexUpdate(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch))
 		}
@@ -1324,11 +1293,14 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
 		objType := "file"
 		action := "modified"
 
-		// If our local vector is verison 1 AND it is the only version vector so far seen for this file then
-		// it is a new file.  Else if it is > 1 it's not new, and if it is 1 but another shortId version vector
-		// exists then it is new for us but created elsewhere so the file is still not new but modified by us.
-		// Only if it is truly new do we change this to 'added', else we leave it as 'modified'.
-		if len(file.Version) == 1 && file.Version[0].Value == 1 {
+		// If our local vector is verison 1 AND it is the only version
+		// vector so far seen for this file then it is a new file.  Else if
+		// it is > 1 it's not new, and if it is 1 but another shortId
+		// version vector exists then it is new for us but created elsewhere
+		// so the file is still not new but modified by us. Only if it is
+		// truly new do we change this to 'added', else we leave it as
+		// 'modified'.
+		if len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1 {
 			action = "added"
 		}
 
@@ -1431,10 +1403,10 @@ func (m *Model) ScanFolders() map[string]error {
 }
 
 func (m *Model) ScanFolder(folder string) error {
-	return m.ScanFolderSubs(folder, nil)
+	return m.ScanFolderSubdirs(folder, nil)
 }
 
-func (m *Model) ScanFolderSubs(folder string, subs []string) error {
+func (m *Model) ScanFolderSubdirs(folder string, subs []string) error {
 	m.fmut.Lock()
 	runner, ok := m.folderRunners[folder]
 	m.fmut.Unlock()
@@ -1449,13 +1421,13 @@ func (m *Model) ScanFolderSubs(folder string, subs []string) error {
 	return runner.Scan(subs)
 }
 
-func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
-	for i, sub := range subs {
+func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error {
+	for i, sub := range subDirs {
 		sub = osutil.NativeFilename(sub)
 		if p := filepath.Clean(filepath.Join(folder, sub)); !strings.HasPrefix(p, folder) {
 			return errors.New("invalid subpath")
 		}
-		subs[i] = sub
+		subDirs[i] = sub
 	}
 
 	m.fmut.Lock()
@@ -1488,7 +1460,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 	// Clean the list of subitems to ensure that we start at a known
 	// directory, and don't scan subdirectories of things we've already
 	// scanned.
-	subs = unifySubs(subs, func(f string) bool {
+	subDirs = unifySubs(subDirs, func(f string) bool {
 		_, ok := fs.Get(protocol.LocalDeviceID, f)
 		return ok
 	})
@@ -1503,7 +1475,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 	fchan, err := scanner.Walk(scanner.Config{
 		Folder:                folderCfg.ID,
 		Dir:                   folderCfg.Path(),
-		Subs:                  subs,
+		Subs:                  subDirs,
 		Matcher:               ignores,
 		BlockSize:             protocol.BlockSize,
 		TempNamer:             defTempNamer,
@@ -1556,15 +1528,15 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 		m.updateLocalsFromScanning(folder, batch)
 	}
 
-	if len(subs) == 0 {
+	if len(subDirs) == 0 {
 		// If we have no specific subdirectories to traverse, set it to one
 		// empty prefix so we traverse the entire folder contents once.
-		subs = []string{""}
+		subDirs = []string{""}
 	}
 
 	// Do a scan of the database for each prefix, to check for deleted files.
 	batch = batch[:0]
-	for _, sub := range subs {
+	for _, sub := range subDirs {
 		var iterError error
 
 		fs.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
@@ -1583,10 +1555,14 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 					// File has been ignored or an unsupported symlink. Set invalid bit.
 					l.Debugln("setting invalid bit on ignored", f)
 					nf := protocol.FileInfo{
-						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagInvalid,
-						Modified: f.Modified,
-						Version:  f.Version, // The file is still the same, so don't bump version
+						Name:          f.Name,
+						Type:          f.Type,
+						Size:          f.Size,
+						Modified:      f.Modified,
+						Permissions:   f.Permissions,
+						NoPermissions: f.NoPermissions,
+						Invalid:       true,
+						Version:       f.Version, // The file is still the same, so don't bump version
 					}
 					batch = append(batch, nf)
 				} else if _, err := osutil.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
@@ -1601,16 +1577,13 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 
 					nf := protocol.FileInfo{
 						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagDeleted,
+						Type:     f.Type,
+						Size:     f.Size,
 						Modified: f.Modified,
+						Deleted:  true,
 						Version:  f.Version.Update(m.shortID),
 					}
 
-					// The deleted file might have been ignored at some
-					// point, but it currently isn't so we make sure to
-					// clear the invalid bit.
-					nf.Flags &^= protocol.FlagInvalid
-
 					batch = append(batch, nf)
 				}
 			}
@@ -1676,30 +1649,21 @@ func (m *Model) numHashers(folder string) int {
 
 // generateClusterConfig returns a ClusterConfigMessage that is correct for
 // the given peer device
-func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfigMessage {
-	var message protocol.ClusterConfigMessage
+func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfig {
+	var message protocol.ClusterConfig
 
 	m.fmut.RLock()
 	for _, folder := range m.deviceFolders[device] {
 		folderCfg := m.cfg.Folders()[folder]
 		protocolFolder := protocol.Folder{
-			ID:    folder,
-			Label: folderCfg.Label,
+			ID:                 folder,
+			Label:              folderCfg.Label,
+			ReadOnly:           folderCfg.Type == config.FolderTypeReadOnly,
+			IgnorePermissions:  folderCfg.IgnorePerms,
+			IgnoreDelete:       folderCfg.IgnoreDelete,
+			DisableTempIndexes: folderCfg.DisableTempIndexes,
 		}
-		var flags uint32
-		if folderCfg.Type == config.FolderTypeReadOnly {
-			flags |= protocol.FlagFolderReadOnly
-		}
-		if folderCfg.IgnorePerms {
-			flags |= protocol.FlagFolderIgnorePerms
-		}
-		if folderCfg.IgnoreDelete {
-			flags |= protocol.FlagFolderIgnoreDelete
-		}
-		if folderCfg.DisableTempIndexes {
-			flags |= protocol.FlagFolderDisabledTempIndexes
-		}
-		protocolFolder.Flags = flags
+
 		for _, device := range m.folderDevices[folder] {
 			// DeviceID is a value type, but with an underlying array. Copy it
 			// so we don't grab aliases to the same array later on in device[:]
@@ -1711,14 +1675,11 @@ func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.Cluster
 				ID:          device[:],
 				Name:        deviceCfg.Name,
 				Addresses:   deviceCfg.Addresses,
-				Compression: uint32(deviceCfg.Compression),
+				Compression: deviceCfg.Compression,
 				CertName:    deviceCfg.CertName,
-				Flags:       protocol.FlagShareTrusted,
+				Introducer:  deviceCfg.Introducer,
 			}
 
-			if deviceCfg.Introducer {
-				protocolDevice.Flags |= protocol.FlagIntroducer
-			}
 			protocolFolder.Devices = append(protocolFolder.Devices, protocolDevice)
 		}
 		message.Folders = append(message.Folders, protocolFolder)
@@ -1763,7 +1724,7 @@ func (m *Model) Override(folder string) {
 		have, ok := fs.Get(protocol.LocalDeviceID, need.Name)
 		if !ok || have.Name != need.Name {
 			// We are missing the file
-			need.Flags |= protocol.FlagDeleted
+			need.Deleted = true
 			need.Blocks = nil
 			need.Version = need.Version.Update(m.shortID)
 		} else {
@@ -1872,7 +1833,7 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
 
 		if !dirsonly && base != "" {
 			last[base] = []interface{}{
-				time.Unix(f.Modified, 0), f.Size(),
+				time.Unix(f.Modified, 0), f.FileSize(),
 			}
 		}
 
@@ -1953,6 +1914,10 @@ func (m *Model) CheckFolderHealth(id string) error {
 
 // checkFolderPath returns nil if the folder path exists and has the marker file.
 func (m *Model) checkFolderPath(folder config.FolderConfiguration) error {
+	if folder.Path() == "" {
+		return errFolderPathEmpty
+	}
+
 	if fi, err := os.Stat(folder.Path()); err != nil || !fi.IsDir() {
 		return errFolderPathMissing
 	}
@@ -2181,11 +2146,7 @@ func mapDeviceCfgs(devices []config.DeviceConfiguration) map[protocol.DeviceID]s
 
 func filterIndex(folder string, fs []protocol.FileInfo, dropDeletes bool, ignores *ignore.Matcher) []protocol.FileInfo {
 	for i := 0; i < len(fs); {
-		if fs[i].Flags&^protocol.FlagsAll != 0 {
-			l.Debugln("dropping update for file with unknown bits set", fs[i])
-			fs[i] = fs[len(fs)-1]
-			fs = fs[:len(fs)-1]
-		} else if fs[i].IsDeleted() && dropDeletes {
+		if fs[i].IsDeleted() && dropDeletes {
 			l.Debugln("dropping update for undesired delete", fs[i])
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]

lib/model/model_test.go

@@ -55,19 +55,19 @@ func init() {
 var testDataExpected = map[string]protocol.FileInfo{
 	"foo": {
 		Name:     "foo",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
 	},
 	"empty": {
 		Name:     "empty",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
 	},
 	"bar": {
 		Name:     "bar",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
 	},
@@ -77,8 +77,9 @@ func init() {
 	// Fix expected test data to match reality
 	for n, f := range testDataExpected {
 		fi, _ := os.Stat("testdata/" + n)
-		f.Flags = uint32(fi.Mode())
+		f.Permissions = uint32(fi.Mode())
 		f.Modified = fi.ModTime().Unix()
+		f.Size = fi.Size()
 		testDataExpected[n] = f
 	}
 }
@@ -98,7 +99,7 @@ func TestRequest(t *testing.T) {
 
 	// Existing, shared file
 	bs = bs[:6]
-	err := m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err := m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err != nil {
 		t.Error(err)
 	}
@@ -107,32 +108,32 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Existing, nonshared file
-	err = m.Request(device2, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device2, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Nonexistent file
-	err = m.Request(device1, "default", "nonexistent", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Shared folder, but disallowed file name
-	err = m.Request(device1, "default", "../walk.go", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Negative offset
-	err = m.Request(device1, "default", "foo", -4, nil, 0, nil, bs[:0])
+	err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Larger block than available
 	bs = bs[:42]
-	err = m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -168,11 +169,11 @@ func benchmarkIndex(b *testing.B, nfiles int) {
 	m.ServeBackground()
 
 	files := genFiles(nfiles)
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.Index(device1, "default", files, 0, nil)
+		m.Index(device1, "default", files)
 	}
 	b.ReportAllocs()
 }
@@ -199,11 +200,11 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 	files := genFiles(nfiles)
 	ufiles := genFiles(nufiles)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", ufiles, 0, nil)
+		m.IndexUpdate(device1, "default", ufiles)
 	}
 	b.ReportAllocs()
 }
@@ -211,8 +212,6 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 type downloadProgressMessage struct {
 	folder  string
 	updates []protocol.FileDownloadProgressUpdate
-	flags   uint32
-	options []protocol.Option
 }
 
 type FakeConnection struct {
@@ -240,11 +239,11 @@ func (f FakeConnection) Option(string) string {
 	return ""
 }
 
-func (FakeConnection) Index(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) Index(string, []protocol.FileInfo) error {
 	return nil
 }
 
-func (FakeConnection) IndexUpdate(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error {
 	return nil
 }
 
@@ -252,7 +251,7 @@ func (f FakeConnection) Request(folder, name string, offset int64, size int, has
 	return f.requestData, nil
 }
 
-func (FakeConnection) ClusterConfig(protocol.ClusterConfigMessage) {}
+func (FakeConnection) ClusterConfig(protocol.ClusterConfig) {}
 
 func (FakeConnection) Ping() bool {
 	return true
@@ -266,12 +265,10 @@ func (FakeConnection) Statistics() protocol.Statistics {
 	return protocol.Statistics{}
 }
 
-func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
 	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{
 		folder:  folder,
 		updates: updates,
-		flags:   flags,
-		options: options,
 	})
 }
 
@@ -305,7 +302,7 @@ func BenchmarkRequest(b *testing.B) {
 		},
 		Connection: fc,
 	}, protocol.HelloResult{})
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -451,13 +448,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 
@@ -471,13 +468,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 }
@@ -572,44 +569,6 @@ func TestIgnores(t *testing.T) {
 	}
 }
 
-func TestRefuseUnknownBits(t *testing.T) {
-	db := db.OpenMemory()
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
-	m.AddFolder(defaultFolderConfig)
-	m.ServeBackground()
-
-	m.ScanFolder("default")
-	m.Index(device1, "default", []protocol.FileInfo{
-		{
-			Name:  "invalid1",
-			Flags: (protocol.FlagsAll + 1) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid2",
-			Flags: (protocol.FlagsAll + 2) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid3",
-			Flags: (1 << 31) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "valid",
-			Flags: protocol.FlagsAll &^ (protocol.FlagInvalid | protocol.FlagSymlink),
-		},
-	}, 0, nil)
-
-	for _, name := range []string{"invalid1", "invalid2", "invalid3"} {
-		f, ok := m.CurrentGlobalFile("default", name)
-		if ok || f.Name == name {
-			t.Error("Invalid file found or name match")
-		}
-	}
-	f, ok := m.CurrentGlobalFile("default", "valid")
-	if !ok || f.Name != "valid" {
-		t.Error("Valid file not found or name mismatch", ok, f)
-	}
-}
-
 func TestROScanRecovery(t *testing.T) {
 	ldb := db.OpenMemory()
 	set := db.NewFileSet("default", ldb)
@@ -642,9 +601,6 @@ func TestROScanRecovery(t *testing.T) {
 	waitFor := func(status string) error {
 		timeout := time.Now().Add(2 * time.Second)
 		for {
-			if time.Now().After(timeout) {
-				return fmt.Errorf("Timed out waiting for status: %s, current status: %s", status, m.cfg.Folders()["default"].Invalid)
-			}
 			_, _, err := m.State("default")
 			if err == nil && status == "" {
 				return nil
@@ -652,6 +608,10 @@ func TestROScanRecovery(t *testing.T) {
 			if err != nil && err.Error() == status {
 				return nil
 			}
+
+			if time.Now().After(timeout) {
+				return fmt.Errorf("Timed out waiting for status: %s, current status: %v", status, err)
+			}
 			time.Sleep(10 * time.Millisecond)
 		}
 	}
@@ -727,9 +687,6 @@ func TestRWScanRecovery(t *testing.T) {
 	waitFor := func(status string) error {
 		timeout := time.Now().Add(2 * time.Second)
 		for {
-			if time.Now().After(timeout) {
-				return fmt.Errorf("Timed out waiting for status: %s, current status: %s", status, m.cfg.Folders()["default"].Invalid)
-			}
 			_, _, err := m.State("default")
 			if err == nil && status == "" {
 				return nil
@@ -737,6 +694,10 @@ func TestRWScanRecovery(t *testing.T) {
 			if err != nil && err.Error() == status {
 				return nil
 			}
+
+			if time.Now().After(timeout) {
+				return fmt.Errorf("Timed out waiting for status: %s, current status: %v", status, err)
+			}
 			time.Sleep(10 * time.Millisecond)
 		}
 	}
@@ -787,17 +748,18 @@ func TestGlobalDirectoryTree(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -869,7 +831,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1037,17 +999,18 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -1128,7 +1091,7 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1213,7 +1176,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
 	m.ScanFolder("default")
 	files := genDeepFiles(n1, n2)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -1242,12 +1205,12 @@ func TestIgnoreDelete(t *testing.T) {
 	}
 
 	// Mark it for deletion
-	f.Flags = protocol.FlagDeleted
+	f.Deleted = true
 	f.Version = f.Version.Update(142) // arbitrary short remote ID
 	f.Blocks = nil
 
 	// Send the index
-	m.Index(device1, "default", []protocol.FileInfo{f}, 0, nil)
+	m.Index(device1, "default", []protocol.FileInfo{f})
 
 	// Make sure we ignored it
 	f, ok = m.CurrentGlobalFile("default", "foo")
@@ -1402,7 +1365,7 @@ func TestIssue3028(t *testing.T) {
 
 	os.Remove("testdata/testrm")
 	os.Remove("testdata/testrm2")
-	m.ScanFolderSubs("default", []string{"testrm", "testrm2"})
+	m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"})
 
 	// Verify that the number of files decreased by two and the number of
 	// deleted files increases by two

lib/model/progressemitter.go

@@ -137,7 +137,7 @@ func (t *ProgressEmitter) sendDownloadProgressMessages() {
 			updates := state.update(folder, activePullers)
 
 			if len(updates) > 0 {
-				conn.DownloadProgress(folder, updates, 0, nil)
+				conn.DownloadProgress(folder, updates)
 			}
 		}
 	}

lib/model/progressemitter_test.go

@@ -100,7 +100,6 @@ func TestProgressEmitter(t *testing.T) {
 }
 
 func TestSendDownloadProgressMessages(t *testing.T) {
-
 	c := config.Wrap("/tmp/test", config.Configuration{})
 	c.SetOptions(config.OptionsConfiguration{
 		ProgressUpdateIntervalS: 0,
@@ -112,7 +111,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 	p := NewProgressEmitter(c)
 	p.temporaryIndexSubscribe(fc, []string{"folder", "folder2"})
 
-	expect := func(updateIdx int, state *sharedPullerState, updateType uint32, version protocol.Vector, blocks []int32, remove bool) {
+	expect := func(updateIdx int, state *sharedPullerState, updateType protocol.FileDownloadProgressUpdateType, version protocol.Vector, blocks []int32, remove bool) {
 		messageIdx := -1
 		for i, msg := range fc.downloadProgressMessages {
 			if msg.folder == state.folder {
@@ -346,7 +345,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state5",
 			Version: v1,
-			Flags:   protocol.FlagDirectory,
+			Type:    protocol.FileInfoTypeDirectory,
 			Blocks:  blocks,
 		},
 		mut:              sync.NewRWMutex(),
@@ -359,7 +358,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state6",
 			Version: v1,
-			Flags:   protocol.FlagSymlink,
+			Type:    protocol.FileInfoTypeSymlinkUnknown,
 		},
 		mut:              sync.NewRWMutex(),
 		available:        []int32{1, 2, 3},

lib/model/rofolder.go

@@ -8,10 +8,8 @@ package model
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/versioner"
 )
 
@@ -23,21 +21,13 @@ type roFolder struct {
 	folder
 }
 
-func newROFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner) service {
+func newROFolder(model *Model, config config.FolderConfiguration, ver versioner.Versioner) service {
 	return &roFolder{
 		folder: folder{
-			stateTracker: stateTracker{
-				folderID: cfg.ID,
-				mut:      sync.NewMutex(),
-			},
-			scan: folderscan{
-				interval: time.Duration(cfg.RescanIntervalS) * time.Second,
-				timer:    time.NewTimer(time.Millisecond),
-				now:      make(chan rescanRequest),
-				delay:    make(chan time.Duration),
-			},
-			stop:  make(chan struct{}),
-			model: model,
+			stateTracker: newStateTracker(config.ID),
+			scan:         newFolderScanner(config),
+			stop:         make(chan struct{}),
+			model:        model,
 		},
 	}
 }
@@ -59,7 +49,7 @@ func (f *roFolder) Serve() {
 		case <-f.scan.timer.C:
 			if err := f.model.CheckFolderHealth(f.folderID); err != nil {
 				l.Infoln("Skipping folder", f.folderID, "scan due to folder error:", err)
-				f.scan.reschedule()
+				f.scan.Reschedule()
 				continue
 			}
 
@@ -71,7 +61,7 @@ func (f *roFolder) Serve() {
 				// the same one as returned by CheckFolderHealth, though
 				// duplicate set is handled by setError.
 				f.setError(err)
-				f.scan.reschedule()
+				f.scan.Reschedule()
 				continue
 			}
 
@@ -80,11 +70,11 @@ func (f *roFolder) Serve() {
 				initialScanCompleted = true
 			}
 
-			if f.scan.interval == 0 {
+			if f.scan.HasNoInterval() {
 				continue
 			}
 
-			f.scan.reschedule()
+			f.scan.Reschedule()
 
 		case req := <-f.scan.now:
 			req.err <- f.scanSubdirsIfHealthy(req.subdirs)

lib/model/rwfolder.go

@@ -107,18 +107,10 @@ type rwFolder struct {
 func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner) service {
 	f := &rwFolder{
 		folder: folder{
-			stateTracker: stateTracker{
-				folderID: cfg.ID,
-				mut:      sync.NewMutex(),
-			},
-			scan: folderscan{
-				interval: time.Duration(cfg.RescanIntervalS) * time.Second,
-				timer:    time.NewTimer(time.Millisecond), // The first scan should be done immediately.
-				now:      make(chan rescanRequest),
-				delay:    make(chan time.Duration),
-			},
-			stop:  make(chan struct{}),
-			model: model,
+			stateTracker: newStateTracker(cfg.ID),
+			scan:         newFolderScanner(cfg),
+			stop:         make(chan struct{}),
+			model:        model,
 		},
 
 		virtualMtimeRepo: db.NewVirtualMtimeRepo(model.db, cfg.ID),
@@ -171,7 +163,7 @@ func (f *rwFolder) configureCopiersAndPullers(config config.FolderConfiguration)
 // set on the local host or the FlagNoPermBits has been set on the file/dir
 // which is being pulled.
 func (f *rwFolder) ignorePermissions(file protocol.FileInfo) bool {
-	return f.ignorePerms || file.Flags&protocol.FlagNoPermBits != 0
+	return f.ignorePerms || file.NoPermissions
 }
 
 // Serve will run scans and pulls. It will return when Stop()ed or on a
@@ -297,7 +289,7 @@ func (f *rwFolder) Serve() {
 		// same time.
 		case <-f.scan.timer.C:
 			err := f.scanSubdirsIfHealthy(nil)
-			f.scan.reschedule()
+			f.scan.Reschedule()
 			if err != nil {
 				continue
 			}
@@ -443,7 +435,7 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 		if !handleFile(file) {
 			// A new or changed file or symlink. This is the only case where we
 			// do stuff concurrently in the background
-			f.queue.Push(file.Name, file.Size(), file.Modified)
+			f.queue.Push(file.Name, file.Size, file.Modified)
 		}
 
 		changed++
@@ -577,7 +569,7 @@ func (f *rwFolder) handleDir(file protocol.FileInfo) {
 	}()
 
 	realName := filepath.Join(f.dir, file.Name)
-	mode := os.FileMode(file.Flags & 0777)
+	mode := os.FileMode(file.Permissions & 0777)
 	if f.ignorePermissions(file) {
 		mode = 0777
 	}
@@ -917,7 +909,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		// touching the file. If we can't stat the file we'll just pull it.
 		if info, err := osutil.Lstat(realName); err == nil {
 			mtime := f.virtualMtimeRepo.GetMtime(file.Name, info.ModTime())
-			if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size() {
+			if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size {
 				l.Debugln("file modified but not rescanned; not pulling:", realName)
 				// Scan() is synchronous (i.e. blocks until the scan is
 				// completed and returns an error), but a scan can't happen
@@ -973,7 +965,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 	} else {
 		// Copy the blocks, as we don't want to shuffle them on the FileInfo
 		blocks = append(blocks, file.Blocks...)
-		blocksSize = file.Size()
+		blocksSize = file.Size
 	}
 
 	if f.checkFreeSpace {
@@ -1029,7 +1021,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 func (f *rwFolder) shortcutFile(file protocol.FileInfo) error {
 	realName := filepath.Join(f.dir, file.Name)
 	if !f.ignorePermissions(file) {
-		if err := os.Chmod(realName, os.FileMode(file.Flags&0777)); err != nil {
+		if err := os.Chmod(realName, os.FileMode(file.Permissions&0777)); err != nil {
 			l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", f.folderID, file.Name, err)
 			f.newError(file.Name, err)
 			return err
@@ -1243,7 +1235,7 @@ func (f *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul
 func (f *rwFolder) performFinish(state *sharedPullerState) error {
 	// Set the correct permission bits on the new file
 	if !f.ignorePermissions(state.file) {
-		if err := os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777)); err != nil {
+		if err := os.Chmod(state.tempName, os.FileMode(state.file.Permissions&0777)); err != nil {
 			return err
 		}
 	}

lib/model/rwfolder_test.go

@@ -50,10 +50,8 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo {
 	}
 
 	return protocol.FileInfo{
-		Name:     filename,
-		Flags:    0,
-		Modified: 0,
-		Blocks:   existingBlocks,
+		Name:   filename,
+		Blocks: existingBlocks,
 	}
 }
 
@@ -69,10 +67,8 @@ func setUpModel(file protocol.FileInfo) *Model {
 func setUpRwFolder(model *Model) rwFolder {
 	return rwFolder{
 		folder: folder{
-			stateTracker: stateTracker{
-				folderID: "default",
-			},
-			model: model,
+			stateTracker: newStateTracker("default"),
+			model:        model,
 		},
 		dir:       "testdata",
 		queue:     newJobQueue(),

lib/model/sharedpullerstate.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 )
@@ -150,7 +149,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 	if s.sparse && !s.file.IsSymlink() {
 		// Truncate sets the size of the file. This creates a sparse file or a
 		// space reservation, depending on the underlying filesystem.
-		if err := fd.Truncate(s.file.Size()); err != nil {
+		if err := fd.Truncate(s.file.Size); err != nil {
 			s.failLocked("dst truncate", err)
 			return nil, err
 		}
@@ -293,8 +292,8 @@ func (s *sharedPullerState) Progress() *pullerProgress {
 		CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
 		Pulled:              s.pullTotal - s.pullNeeded,
 		Pulling:             s.pullNeeded,
-		BytesTotal:          db.BlocksToSize(total),
-		BytesDone:           db.BlocksToSize(done),
+		BytesTotal:          blocksToSize(total),
+		BytesDone:           blocksToSize(done),
 	}
 }
 
@@ -321,3 +320,10 @@ func (s *sharedPullerState) Available() []int32 {
 	s.mut.RUnlock()
 	return blocks
 }
+
+func blocksToSize(num int) int64 {
+	if num < 2 {
+		return protocol.BlockSize / 2
+	}
+	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
+}

lib/model/tempname.go

@@ -20,6 +20,11 @@ type tempNamer struct {
 
 var defTempNamer tempNamer
 
+// Real filesystems usually handle 255 bytes. encfs has varying and
+// confusing file name limits. We take a safe way out and switch to hashing
+// quite early.
+const maxFilenameLength = 160 - len(".syncthing.") - len(".tmp")
+
 func init() {
 	if runtime.GOOS == "windows" {
 		defTempNamer = tempNamer{"~syncthing~"}
@@ -35,7 +40,7 @@ func (t tempNamer) IsTemporary(name string) bool {
 func (t tempNamer) TempName(name string) string {
 	tdir := filepath.Dir(name)
 	tbase := filepath.Base(name)
-	if len(tbase) > 240 {
+	if len(tbase) > maxFilenameLength {
 		hash := md5.New()
 		hash.Write([]byte(name))
 		tbase = fmt.Sprintf("%x", hash.Sum(nil))

lib/protocol/benchmark_test.go

@@ -63,8 +63,8 @@ func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) {
 	c1.Start()
 
 	// Satisfy the assertions in the protocol by sending an initial cluster config
-	c0.ClusterConfig(ClusterConfigMessage{})
-	c1.ClusterConfig(ClusterConfigMessage{})
+	c0.ClusterConfig(ClusterConfig{})
+	c1.ClusterConfig(ClusterConfig{})
 
 	// Report some useful stats and reset the timer for the actual test
 	b.ReportAllocs()
@@ -164,13 +164,13 @@ func negotiateTLS(cert tls.Certificate, conn0, conn1 net.Conn) (net.Conn, net.Co
 
 type fakeModel struct{}
 
-func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	// We write the offset to the end of the buffer, so the receiver
 	// can verify that it did in fact get some data back over the
 	// connection.
@@ -178,11 +178,11 @@ func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offse
 	return nil
 }
 
-func (m *fakeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+func (m *fakeModel) ClusterConfig(deviceID DeviceID, config ClusterConfig) {
 }
 
 func (m *fakeModel) Close(deviceID DeviceID, err error) {
 }
 
-func (m *fakeModel) DownloadProgress(deviceID DeviceID, folder string, updates []FileDownloadProgressUpdate, flags uint32, options []Option) {
+func (m *fakeModel) DownloadProgress(deviceID DeviceID, folder string, updates []FileDownloadProgressUpdate) {
 }

lib/protocol/bep.proto

@@ -0,0 +1,189 @@
+// protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. message.proto
+
+syntax = "proto3";
+
+package protocol;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+option (gogoproto.goproto_enum_stringer_all) = true;
+option (gogoproto.goproto_enum_prefix_all) = false;
+
+// --- Pre-auth ---
+
+message Hello {
+    string device_name    = 1;
+    string client_name    = 2;
+    string client_version = 3;
+}
+
+// --- Header ---
+
+message Header {
+    MessageType        type        = 1;
+    MessageCompression compression = 2;
+}
+
+enum MessageType {
+    CLUSTER_CONFIG    = 0 [(gogoproto.enumvalue_customname) = "messageTypeClusterConfig"];
+    INDEX             = 1 [(gogoproto.enumvalue_customname) = "messageTypeIndex"];
+    INDEX_UPDATE      = 2 [(gogoproto.enumvalue_customname) = "messageTypeIndexUpdate"];
+    REQUEST           = 3 [(gogoproto.enumvalue_customname) = "messageTypeRequest"];
+    RESPONSE          = 4 [(gogoproto.enumvalue_customname) = "messageTypeResponse"];
+    DOWNLOAD_PROGRESS = 5 [(gogoproto.enumvalue_customname) = "messageTypeDownloadProgress"];
+    PING              = 6 [(gogoproto.enumvalue_customname) = "messageTypePing"];
+    CLOSE             = 7 [(gogoproto.enumvalue_customname) = "messageTypeClose"];
+}
+
+enum MessageCompression {
+    NONE = 0 [(gogoproto.enumvalue_customname) = "MessageCompressionNone"];
+    LZ4  = 1 [(gogoproto.enumvalue_customname) = "MessageCompressionLZ4"];
+}
+
+// --- Actual messages ---
+
+// Cluster Config
+
+message ClusterConfig {
+    repeated Folder folders = 1 [(gogoproto.nullable) = false];
+}
+
+message Folder {
+    string id                   = 1 [(gogoproto.customname) = "ID"];
+    string label                = 2;
+    bool   read_only            = 3;
+    bool   ignore_permissions   = 4;
+    bool   ignore_delete        = 5;
+    bool   disable_temp_indexes = 6;
+
+    repeated Device devices = 16 [(gogoproto.nullable) = false];
+}
+
+message Device {
+    bytes           id                = 1 [(gogoproto.customname) = "ID"];
+    string          name              = 2;
+    repeated string addresses         = 3;
+    Compression     compression       = 4;
+    string          cert_name         = 5;
+    int64           max_local_version = 6;
+    bool            introducer        = 7;
+}
+
+enum Compression {
+    METADATA = 0 [(gogoproto.enumvalue_customname) = "CompressMetadata"];
+    NEVER    = 1 [(gogoproto.enumvalue_customname) = "CompressNever"];
+    ALWAYS   = 2 [(gogoproto.enumvalue_customname) = "CompressAlways"];
+}
+
+// Index and Index Update
+
+message Index {
+    string            folder = 1;
+    repeated FileInfo files  = 2 [(gogoproto.nullable) = false];
+}
+
+message IndexUpdate {
+    string            folder = 1;
+    repeated FileInfo files  = 2 [(gogoproto.nullable) = false];
+}
+
+message FileInfo {
+    option (gogoproto.goproto_stringer) = false;
+    string       name           = 1;
+    FileInfoType type           = 2;
+    int64        size           = 3;
+    uint32       permissions    = 4;
+    int64        modified       = 5;
+    bool         deleted        = 6;
+    bool         invalid        = 7;
+    bool         no_permissions = 8;
+    Vector       version        = 9 [(gogoproto.nullable) = false];
+    int64        local_version  = 10;
+
+    repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false];
+}
+
+enum FileInfoType {
+    FILE              = 0 [(gogoproto.enumvalue_customname) = "FileInfoTypeFile"];
+    DIRECTORY         = 1 [(gogoproto.enumvalue_customname) = "FileInfoTypeDirectory"];
+    SYMLINK_FILE      = 2 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkFile"];
+    SYMLINK_DIRECTORY = 3 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkDirectory"];
+    SYMLINK_UNKNOWN   = 4 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkUnknown"];
+}
+
+message BlockInfo {
+    option (gogoproto.goproto_stringer) = false;
+    int64 offset = 1;
+    int32 size   = 2;
+    bytes hash   = 3;
+}
+
+message Vector {
+    repeated Counter counters = 1 [(gogoproto.nullable) = false];
+}
+
+message Counter {
+    uint64 id    = 1 [(gogoproto.customname) = "ID", (gogoproto.customtype) = "ShortID", (gogoproto.nullable) = false];
+    uint64 value = 2;
+}
+
+// Request
+
+message Request {
+    int32  id             = 1 [(gogoproto.customname) = "ID"];
+    string folder         = 2;
+    string name           = 3;
+    int64  offset         = 4;
+    int32  size           = 5;
+    bytes  hash           = 6;
+    bool   from_temporary = 7;
+}
+
+// Response
+
+message Response {
+    int32     id   = 1 [(gogoproto.customname) = "ID"];
+    bytes     data = 2;
+    ErrorCode code = 3;
+}
+
+enum ErrorCode {
+    NO_ERROR     = 0 [(gogoproto.enumvalue_customname) = "ErrorCodeNoError"];
+    GENERIC      = 1 [(gogoproto.enumvalue_customname) = "ErrorCodeGeneric"];
+    NO_SUCH_FILE = 2 [(gogoproto.enumvalue_customname) = "ErrorCodeNoSuchFile"];
+    INVALID_FILE = 3 [(gogoproto.enumvalue_customname) = "ErrorCodeInvalidFile"];
+}
+
+// DownloadProgress
+
+message DownloadProgress {
+    string                              folder  = 1;
+    repeated FileDownloadProgressUpdate updates = 2 [(gogoproto.nullable) = false];
+}
+
+message FileDownloadProgressUpdate {
+    FileDownloadProgressUpdateType update_type   = 1;
+    string                         name          = 2;
+    Vector                         version       = 3 [(gogoproto.nullable) = false];
+    repeated int32                 block_indexes = 4;
+}
+
+enum FileDownloadProgressUpdateType {
+    APPEND = 0 [(gogoproto.enumvalue_customname) = "UpdateTypeAppend"];
+    FORGET = 1 [(gogoproto.enumvalue_customname) = "UpdateTypeForget"];
+}
+
+// Ping
+
+message Ping {
+}
+
+// Close
+
+message Close {
+    string reason = 1;
+}
+

lib/protocol/bep_extensions.go

@@ -0,0 +1,96 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+//go:generate go run ../../script/protofmt.go bep.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. bep.proto
+
+package protocol
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+)
+
+var (
+	sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
+	HelloMessageMagic  = uint32(0x2EA7D90B)
+)
+
+func (m Hello) Magic() uint32 {
+	return HelloMessageMagic
+}
+
+func (f FileInfo) String() string {
+	return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, Blocks:%v}",
+		f.Name, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.Blocks)
+}
+
+func (f FileInfo) IsDeleted() bool {
+	return f.Deleted
+}
+
+func (f FileInfo) IsInvalid() bool {
+	return f.Invalid
+}
+
+func (f FileInfo) IsDirectory() bool {
+	return f.Type == FileInfoTypeDirectory
+}
+
+func (f FileInfo) IsSymlink() bool {
+	switch f.Type {
+	case FileInfoTypeSymlinkDirectory, FileInfoTypeSymlinkFile, FileInfoTypeSymlinkUnknown:
+		return true
+	default:
+		return false
+	}
+}
+
+func (f FileInfo) HasPermissionBits() bool {
+	return !f.NoPermissions
+}
+
+func (f FileInfo) FileSize() int64 {
+	if f.IsDirectory() || f.IsDeleted() {
+		return 128
+	}
+	return f.Size
+}
+
+func (f FileInfo) FileName() string {
+	return f.Name
+}
+
+// WinsConflict returns true if "f" is the one to choose when it is in
+// conflict with "other".
+func (f FileInfo) WinsConflict(other FileInfo) bool {
+	// If a modification is in conflict with a delete, we pick the
+	// modification.
+	if !f.IsDeleted() && other.IsDeleted() {
+		return true
+	}
+	if f.IsDeleted() && !other.IsDeleted() {
+		return false
+	}
+
+	// The one with the newer modification time wins.
+	if f.Modified > other.Modified {
+		return true
+	}
+	if f.Modified < other.Modified {
+		return false
+	}
+
+	// The modification times were equal. Use the device ID in the version
+	// vector as tie breaker.
+	return f.Version.Compare(other.Version) == ConcurrentGreater
+}
+
+func (b BlockInfo) String() string {
+	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
+}
+
+// IsEmpty returns true if the block is a full block of zeroes.
+func (b BlockInfo) IsEmpty() bool {
+	return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
+}

lib/protocol/bufferpool.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2016 The Protocol Authors.
+
+package protocol
+
+import "sync"
+
+type bufferPool struct {
+	minSize int
+	pool    sync.Pool
+}
+
+// get returns a new buffer of the requested size
+func (p *bufferPool) get(size int) []byte {
+	intf := p.pool.Get()
+	if intf == nil {
+		// Pool is empty, must allocate.
+		return p.new(size)
+	}
+
+	bs := intf.([]byte)
+	if cap(bs) < size {
+		// Buffer was too small, leave it for someone else and allocate.
+		p.put(bs)
+		return p.new(size)
+	}
+
+	return bs[:size]
+}
+
+// upgrade grows the buffer to the requested size, while attempting to reuse
+// it if possible.
+func (p *bufferPool) upgrade(bs []byte, size int) []byte {
+	if cap(bs) >= size {
+		// Reslicing is enough, lets go!
+		return bs[:size]
+	}
+
+	// It was too small. But it pack into the pool and try to get another
+	// buffer.
+	p.put(bs)
+	return p.get(size)
+}
+
+// put returns the buffer to the pool
+func (p *bufferPool) put(bs []byte) {
+	p.pool.Put(bs)
+}
+
+// new creates a new buffer of the requested size, taking the minimum
+// allocation count into account. For internal use only.
+func (p *bufferPool) new(size int) []byte {
+	allocSize := size
+	if allocSize < p.minSize {
+		// Avoid allocating tiny buffers that we won't be able to reuse for
+		// anything useful.
+		allocSize = p.minSize
+	}
+	return make([]byte, allocSize)[:size]
+}

lib/protocol/common_test.go

@@ -5,16 +5,15 @@ package protocol
 import "time"
 
 type TestModel struct {
-	data      []byte
-	folder    string
-	name      string
-	offset    int64
-	size      int
-	hash      []byte
-	flags     uint32
-	options   []Option
-	closedCh  chan struct{}
-	closedErr error
+	data          []byte
+	folder        string
+	name          string
+	offset        int64
+	size          int
+	hash          []byte
+	fromTemporary bool
+	closedCh      chan struct{}
+	closedErr     error
 }
 
 func newTestModel() *TestModel {
@@ -23,20 +22,19 @@ func newTestModel() *TestModel {
 	}
 }
 
-func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	t.folder = folder
 	t.name = name
 	t.offset = offset
 	t.size = len(buf)
 	t.hash = hash
-	t.flags = flags
-	t.options = options
+	t.fromTemporary = fromTemporary
 	copy(buf, t.data)
 	return nil
 }
@@ -46,10 +44,10 @@ func (t *TestModel) Close(deviceID DeviceID, err error) {
 	close(t.closedCh)
 }
 
-func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfig) {
 }
 
-func (t *TestModel) DownloadProgress(DeviceID, string, []FileDownloadProgressUpdate, uint32, []Option) {
+func (t *TestModel) DownloadProgress(DeviceID, string, []FileDownloadProgressUpdate) {
 }
 
 func (t *TestModel) closedError() error {

lib/protocol/compression.go

@@ -4,13 +4,7 @@ package protocol
 
 import "fmt"
 
-type Compression int
-
 const (
-	CompressMetadata Compression = iota // zero value is the default, default should be "metadata"
-	CompressNever
-	CompressAlways
-
 	compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes
 )
 
@@ -31,14 +25,6 @@ var compressionUnmarshal = map[string]Compression{
 	"always":   CompressAlways,
 }
 
-func (c Compression) String() string {
-	s, ok := compressionMarshal[c]
-	if !ok {
-		return fmt.Sprintf("unknown:%d", c)
-	}
-	return s
-}
-
 func (c Compression) GoString() string {
 	return fmt.Sprintf("%q", c.String())
 }

lib/protocol/conflict_test.go

@@ -8,8 +8,8 @@ func TestWinsConflict(t *testing.T) {
 	testcases := [][2]FileInfo{
 		// The first should always win over the second
 		{{Modified: 42}, {Modified: 41}},
-		{{Modified: 41}, {Modified: 42, Flags: FlagDeleted}},
-		{{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}},
+		{{Modified: 41}, {Modified: 42, Deleted: true}},
+		{{Modified: 41, Version: Vector{[]Counter{{42, 2}, {43, 1}}}}, {Modified: 41, Version: Vector{[]Counter{{42, 1}, {43, 2}}}}},
 	}
 
 	for _, tc := range testcases {

lib/protocol/debug.go

@@ -16,7 +16,3 @@ var (
 func init() {
 	l.SetDebug("protocol", strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all")
 }
-
-func shouldDebug() bool {
-	return l.ShouldDebug("protocol")
-}

lib/protocol/errors.go

@@ -6,13 +6,6 @@ import (
 	"errors"
 )
 
-const (
-	ecNoError int32 = iota
-	ecGeneric
-	ecNoSuchFile
-	ecInvalid
-)
-
 var (
 	ErrNoError    error
 	ErrGeneric    = errors.New("generic error")
@@ -20,32 +13,32 @@ var (
 	ErrInvalid    = errors.New("file is invalid")
 )
 
-var lookupError = map[int32]error{
-	ecNoError:    ErrNoError,
-	ecGeneric:    ErrGeneric,
-	ecNoSuchFile: ErrNoSuchFile,
-	ecInvalid:    ErrInvalid,
+var lookupError = map[ErrorCode]error{
+	ErrorCodeNoError:     ErrNoError,
+	ErrorCodeGeneric:     ErrGeneric,
+	ErrorCodeNoSuchFile:  ErrNoSuchFile,
+	ErrorCodeInvalidFile: ErrInvalid,
 }
 
-var lookupCode = map[error]int32{
-	ErrNoError:    ecNoError,
-	ErrGeneric:    ecGeneric,
-	ErrNoSuchFile: ecNoSuchFile,
-	ErrInvalid:    ecInvalid,
+var lookupCode = map[error]ErrorCode{
+	ErrNoError:    ErrorCodeNoError,
+	ErrGeneric:    ErrorCodeGeneric,
+	ErrNoSuchFile: ErrorCodeNoSuchFile,
+	ErrInvalid:    ErrorCodeInvalidFile,
 }
 
-func codeToError(errcode int32) error {
-	err, ok := lookupError[errcode]
+func codeToError(code ErrorCode) error {
+	err, ok := lookupError[code]
 	if !ok {
 		return ErrGeneric
 	}
 	return err
 }
 
-func errorToCode(err error) int32 {
+func errorToCode(err error) ErrorCode {
 	code, ok := lookupCode[err]
 	if !ok {
-		return ecGeneric
+		return ErrorCodeGeneric
 	}
 	return code
 }

lib/protocol/fuzz.go

@@ -1,70 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-// +build gofuzz
-
-package protocol
-
-import (
-	"bytes"
-	"encoding/binary"
-	"encoding/hex"
-	"fmt"
-	"reflect"
-	"sync"
-)
-
-func Fuzz(data []byte) int {
-	// Regenerate the length, or we'll most commonly exit quickly due to an
-	// unexpected eof which is unintestering.
-	if len(data) > 8 {
-		binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8)
-	}
-
-	// Setup a rawConnection we'll use to parse the message.
-	c := rawConnection{
-		cr:     &countingReader{Reader: bytes.NewReader(data)},
-		closed: make(chan struct{}),
-		pool: sync.Pool{
-			New: func() interface{} {
-				return make([]byte, BlockSize)
-			},
-		},
-	}
-
-	// Attempt to parse the message.
-	hdr, msg, err := c.readMessage()
-	if err != nil {
-		return 0
-	}
-
-	// If parsing worked, attempt to encode it again.
-	newBs, err := msg.AppendXDR(nil)
-	if err != nil {
-		panic("not encodable")
-	}
-
-	// Create an appriate header for the re-encoding.
-	newMsg := make([]byte, 8)
-	binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr))
-	binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs)))
-	newMsg = append(newMsg, newBs...)
-
-	// Use the rawConnection to parse the re-encoding.
-	c.cr = &countingReader{Reader: bytes.NewReader(newMsg)}
-	hdr2, msg2, err := c.readMessage()
-	if err != nil {
-		fmt.Println("Initial:\n" + hex.Dump(data))
-		fmt.Println("New:\n" + hex.Dump(newMsg))
-		panic("not parseable after re-encode: " + err.Error())
-	}
-
-	// Make sure the data is the same as it was before.
-	if hdr != hdr2 {
-		panic("headers differ")
-	}
-	if !reflect.DeepEqual(msg, msg2) {
-		panic("contents differ")
-	}
-
-	return 1
-}

lib/protocol/fuzz_test.go

@@ -1,89 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-// +build gofuzz
-
-package protocol
-
-import (
-	"encoding/binary"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
-	"testing/quick"
-)
-
-// This can be used to generate a corpus of valid messages as a starting point
-// for the fuzzer.
-func TestGenerateCorpus(t *testing.T) {
-	t.Skip("Use to generate initial corpus only")
-
-	n := 0
-	check := func(idx IndexMessage) bool {
-		for i := range idx.Options {
-			if len(idx.Options[i].Key) > 64 {
-				idx.Options[i].Key = idx.Options[i].Key[:64]
-			}
-		}
-		hdr := header{
-			version:     0,
-			msgID:       42,
-			msgType:     messageTypeIndex,
-			compression: false,
-		}
-
-		msgBs := idx.MustMarshalXDR()
-
-		buf := make([]byte, 8)
-		binary.BigEndian.PutUint32(buf, encodeHeader(hdr))
-		binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs)))
-		buf = append(buf, msgBs...)
-
-		ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644)
-		n++
-		return true
-	}
-
-	if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// Tests any crashers found by the fuzzer, for closer investigation.
-func TestCrashers(t *testing.T) {
-	testFiles(t, "testdata/crashers")
-}
-
-// Tests the entire corpus, which should PASS before the fuzzer starts
-// fuzzing.
-func TestCorpus(t *testing.T) {
-	testFiles(t, "testdata/corpus")
-}
-
-func testFiles(t *testing.T, dir string) {
-	fd, err := os.Open(dir)
-	if err != nil {
-		t.Fatal(err)
-	}
-	crashers, err := fd.Readdirnames(-1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, name := range crashers {
-		if strings.HasSuffix(name, ".output") {
-			continue
-		}
-		if strings.HasSuffix(name, ".quoted") {
-			continue
-		}
-
-		t.Log(name)
-		crasher, err := ioutil.ReadFile(dir + "/" + name)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		Fuzz(crasher)
-	}
-}

lib/protocol/header.go

@@ -1,44 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-package protocol
-
-import "github.com/calmh/xdr"
-
-type header struct {
-	version     int
-	msgID       int
-	msgType     int
-	compression bool
-}
-
-func (h header) MarshalXDRInto(m *xdr.Marshaller) error {
-	v := encodeHeader(h)
-	m.MarshalUint32(v)
-	return m.Error
-}
-
-func (h *header) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	v := u.UnmarshalUint32()
-	*h = decodeHeader(v)
-	return u.Error
-}
-
-func encodeHeader(h header) uint32 {
-	var isComp uint32
-	if h.compression {
-		isComp = 1 << 0 // the zeroth bit is the compression bit
-	}
-	return uint32(h.version&0xf)<<28 +
-		uint32(h.msgID&0xfff)<<16 +
-		uint32(h.msgType&0xff)<<8 +
-		isComp
-}
-
-func decodeHeader(u uint32) header {
-	return header{
-		version:     int(u>>28) & 0xf,
-		msgID:       int(u>>16) & 0xfff,
-		msgType:     int(u>>8) & 0xff,
-		compression: u&1 == 1,
-	}
-}

lib/protocol/hello.go

@@ -9,10 +9,10 @@ import (
 	"io"
 )
 
-// The HelloMessage interface is implemented by the version specific hello
+// The HelloIntf interface is implemented by the version specific hello
 // message. It knows its magic number and how to serialize itself to a byte
 // buffer.
-type HelloMessage interface {
+type HelloIntf interface {
 	Magic() uint32
 	Marshal() ([]byte, error)
 }
@@ -29,12 +29,15 @@ var (
 	// ErrTooOldVersion12 is returned by ExchangeHello when the other side
 	// speaks the older, incompatible version 0.12 of the protocol.
 	ErrTooOldVersion12 = errors.New("the remote device speaks an older version of the protocol (v0.12) not compatible with this version")
+	// ErrTooOldVersion13 is returned by ExchangeHello when the other side
+	// speaks the older, incompatible version 0.12 of the protocol.
+	ErrTooOldVersion13 = errors.New("the remote device speaks an older version of the protocol (v0.13) not compatible with this version")
 	// ErrUnknownMagic is returned by ExchangeHellow when the other side
 	// speaks something entirely unknown.
 	ErrUnknownMagic = errors.New("the remote device speaks an unknown (newer?) version of the protocol")
 )
 
-func ExchangeHello(c io.ReadWriter, h HelloMessage) (HelloResult, error) {
+func ExchangeHello(c io.ReadWriter, h HelloIntf) (HelloResult, error) {
 	if err := writeHello(c, h); err != nil {
 		return HelloResult{}, err
 	}
@@ -45,7 +48,7 @@ func ExchangeHello(c io.ReadWriter, h HelloMessage) (HelloResult, error) {
 // version mismatch that we might want to alert the user about.
 func IsVersionMismatch(err error) bool {
 	switch err {
-	case ErrTooOldVersion12, ErrUnknownMagic:
+	case ErrTooOldVersion12, ErrTooOldVersion13, ErrUnknownMagic:
 		return true
 	default:
 		return false
@@ -59,6 +62,28 @@ func readHello(c io.Reader) (HelloResult, error) {
 	}
 
 	switch binary.BigEndian.Uint32(header[:4]) {
+	case HelloMessageMagic:
+		// This is a v0.14 Hello message in proto format
+		msgSize := binary.BigEndian.Uint32(header[4:])
+		if msgSize > 1024 {
+			return HelloResult{}, fmt.Errorf("hello message too big")
+		}
+		buf := make([]byte, msgSize)
+		if _, err := io.ReadFull(c, buf); err != nil {
+			return HelloResult{}, err
+		}
+
+		var hello Hello
+		if err := hello.Unmarshal(buf); err != nil {
+			return HelloResult{}, err
+		}
+		res := HelloResult{
+			DeviceName:    hello.DeviceName,
+			ClientName:    hello.ClientName,
+			ClientVersion: hello.ClientVersion,
+		}
+		return res, nil
+
 	case Version13HelloMagic:
 		// This is a v0.13 Hello message in XDR format
 		msgSize := binary.BigEndian.Uint32(header[4:])
@@ -79,7 +104,7 @@ func readHello(c io.Reader) (HelloResult, error) {
 			ClientName:    hello.ClientName,
 			ClientVersion: hello.ClientVersion,
 		}
-		return res, nil
+		return res, ErrTooOldVersion13
 
 	case 0x00010001, 0x00010000:
 		// This is the first word of a v0.12 cluster config message.
@@ -90,7 +115,7 @@ func readHello(c io.Reader) (HelloResult, error) {
 	return HelloResult{}, ErrUnknownMagic
 }
 
-func writeHello(c io.Writer, h HelloMessage) error {
+func writeHello(c io.Writer, h HelloIntf) error {
 	msg, err := h.Marshal()
 	if err != nil {
 		return err

lib/protocol/hello_test.go

@@ -13,6 +13,53 @@ import (
 
 var spaceRe = regexp.MustCompile(`\s`)
 
+func TestVersion14Hello(t *testing.T) {
+	// Tests that we can send and receive a version 0.14 hello message.
+
+	expected := Hello{
+		DeviceName:    "test device",
+		ClientName:    "syncthing",
+		ClientVersion: "v0.14.5",
+	}
+	msgBuf, err := expected.Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hdrBuf := make([]byte, 8)
+	binary.BigEndian.PutUint32(hdrBuf, HelloMessageMagic)
+	binary.BigEndian.PutUint32(hdrBuf[4:], uint32(len(msgBuf)))
+
+	outBuf := new(bytes.Buffer)
+	outBuf.Write(hdrBuf)
+	outBuf.Write(msgBuf)
+
+	inBuf := new(bytes.Buffer)
+
+	conn := &readWriter{outBuf, inBuf}
+
+	send := &Hello{
+		DeviceName:    "this device",
+		ClientName:    "other client",
+		ClientVersion: "v0.14.6",
+	}
+
+	res, err := ExchangeHello(conn, send)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if res.ClientName != expected.ClientName {
+		t.Errorf("incorrect ClientName %q != expected %q", res.ClientName, expected.ClientName)
+	}
+	if res.ClientVersion != expected.ClientVersion {
+		t.Errorf("incorrect ClientVersion %q != expected %q", res.ClientVersion, expected.ClientVersion)
+	}
+	if res.DeviceName != expected.DeviceName {
+		t.Errorf("incorrect DeviceName %q != expected %q", res.DeviceName, expected.DeviceName)
+	}
+}
+
 func TestVersion13Hello(t *testing.T) {
 	// Tests that we can send and receive a version 0.13 hello message.
 
@@ -42,8 +89,8 @@ func TestVersion13Hello(t *testing.T) {
 	}
 
 	res, err := ExchangeHello(conn, send)
-	if err != nil {
-		t.Fatal(err)
+	if err != ErrTooOldVersion13 {
+		t.Errorf("unexpected error %v != ErrTooOldVersion13", err)
 	}
 
 	if res.ClientName != expected.ClientName {
@@ -94,7 +141,7 @@ func TestVersion12Hello(t *testing.T) {
 
 	_, err := ExchangeHello(conn, send)
 	if err != ErrTooOldVersion12 {
-		t.Errorf("unexpected error %v != ErrTooOld", err)
+		t.Errorf("unexpected error %v != ErrTooOldVersion12", err)
 	}
 }
 

lib/protocol/message.go

@@ -1,186 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o message_xdr.go message.go
-
-package protocol
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"fmt"
-)
-
-var (
-	sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
-)
-
-type IndexMessage struct {
-	Folder  string     // max:256
-	Files   []FileInfo // max:1000000
-	Flags   uint32
-	Options []Option // max:64
-}
-
-type FileInfo struct {
-	Name         string // max:8192
-	Flags        uint32
-	Modified     int64
-	Version      Vector
-	LocalVersion int64
-	CachedSize   int64       // noencode (cache only)
-	Blocks       []BlockInfo // max:10000000
-}
-
-func (f FileInfo) String() string {
-	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
-		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
-}
-
-func (f FileInfo) Size() (bytes int64) {
-	if f.IsDeleted() || f.IsDirectory() {
-		return 128
-	}
-	if f.CachedSize > 0 {
-		return f.CachedSize
-	}
-	for _, b := range f.Blocks {
-		bytes += int64(b.Size)
-	}
-	f.CachedSize = bytes
-	return
-}
-
-func (f FileInfo) IsDeleted() bool {
-	return f.Flags&FlagDeleted != 0
-}
-
-func (f FileInfo) IsInvalid() bool {
-	return f.Flags&FlagInvalid != 0
-}
-
-func (f FileInfo) IsDirectory() bool {
-	return f.Flags&FlagDirectory != 0
-}
-
-func (f FileInfo) IsSymlink() bool {
-	return f.Flags&FlagSymlink != 0
-}
-
-func (f FileInfo) HasPermissionBits() bool {
-	return f.Flags&FlagNoPermBits == 0
-}
-
-// WinsConflict returns true if "f" is the one to choose when it is in
-// conflict with "other".
-func (f FileInfo) WinsConflict(other FileInfo) bool {
-	// If a modification is in conflict with a delete, we pick the
-	// modification.
-	if !f.IsDeleted() && other.IsDeleted() {
-		return true
-	}
-	if f.IsDeleted() && !other.IsDeleted() {
-		return false
-	}
-
-	// The one with the newer modification time wins.
-	if f.Modified > other.Modified {
-		return true
-	}
-	if f.Modified < other.Modified {
-		return false
-	}
-
-	// The modification times were equal. Use the device ID in the version
-	// vector as tie breaker.
-	return f.Version.Compare(other.Version) == ConcurrentGreater
-}
-
-type BlockInfo struct {
-	Offset int64 // noencode (cache only)
-	Size   int32
-	Hash   []byte // max:64
-}
-
-func (b BlockInfo) String() string {
-	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
-}
-
-// IsEmpty returns true if the block is a full block of zeroes.
-func (b BlockInfo) IsEmpty() bool {
-	return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
-}
-
-type RequestMessage struct {
-	Folder  string // max:256
-	Name    string // max:8192
-	Offset  int64
-	Size    int32
-	Hash    []byte // max:64
-	Flags   uint32
-	Options []Option // max:64
-}
-
-type ResponseMessage struct {
-	Data []byte
-	Code int32
-}
-
-type ClusterConfigMessage struct {
-	Folders []Folder // max:1000000
-	Options []Option // max:64
-}
-
-type DownloadProgressMessage struct {
-	Folder  string                       // max:64
-	Updates []FileDownloadProgressUpdate // max:1000000
-	Flags   uint32
-	Options []Option // max:64
-}
-
-func (o *ClusterConfigMessage) GetOption(key string) string {
-	for _, option := range o.Options {
-		if option.Key == key {
-			return option.Value
-		}
-	}
-	return ""
-}
-
-type Folder struct {
-	ID      string   // max:256
-	Label   string   // max:256
-	Devices []Device // max:1000000
-	Flags   uint32
-	Options []Option // max:64
-}
-
-type Device struct {
-	ID              []byte   // max:32
-	Name            string   // max:64
-	Addresses       []string // max:64,2083
-	Compression     uint32
-	CertName        string // max:64
-	MaxLocalVersion int64
-	Flags           uint32
-	Options         []Option // max:64
-}
-
-type FileDownloadProgressUpdate struct {
-	UpdateType   uint32
-	Name         string // max:8192
-	Version      Vector
-	BlockIndexes []int32 // max:1000000
-}
-
-type Option struct {
-	Key   string // max:64
-	Value string // max:1024
-}
-
-type CloseMessage struct {
-	Reason string // max:1024
-	Code   int32
-}
-
-type EmptyMessage struct{}

lib/protocol/nativemodel_darwin.go

@@ -12,21 +12,21 @@ type nativeModel struct {
 	Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 	for i := range files {
 		files[i].Name = norm.NFD.String(files[i].Name)
 	}
-	m.Model.Index(deviceID, folder, files, flags, options)
+	m.Model.Index(deviceID, folder, files)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 	for i := range files {
 		files[i].Name = norm.NFD.String(files[i].Name)
 	}
-	m.Model.IndexUpdate(deviceID, folder, files, flags, options)
+	m.Model.IndexUpdate(deviceID, folder, files)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	name = norm.NFD.String(name)
-	return m.Model.Request(deviceID, folder, name, offset, hash, flags, options, buf)
+	return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
 }

lib/protocol/nativemodel_windows.go

@@ -24,19 +24,19 @@ type nativeModel struct {
 	Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 	fixupFiles(folder, files)
-	m.Model.Index(deviceID, folder, files, flags, options)
+	m.Model.Index(deviceID, folder, files)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 	fixupFiles(folder, files)
-	m.Model.IndexUpdate(deviceID, folder, files, flags, options)
+	m.Model.IndexUpdate(deviceID, folder, files)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	name = filepath.FromSlash(name)
-	return m.Model.Request(deviceID, folder, name, offset, hash, flags, options, buf)
+	return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
 }
 
 func fixupFiles(folder string, files []FileInfo) {
@@ -47,7 +47,7 @@ func fixupFiles(folder string, files []FileInfo) {
 				// can't possibly exist here anyway.
 				continue
 			}
-			files[i].Flags |= FlagInvalid
+			files[i].Invalid = true
 			l.Warnf("File name %q (folder %q) contains invalid characters; marked as invalid.", f.Name, folder)
 		}
 		files[i].Name = filepath.FromSlash(files[i].Name)

lib/protocol/protocol_test.go

@@ -3,21 +3,17 @@
 package protocol
 
 import (
-	"encoding/binary"
-	"encoding/hex"
+	"bytes"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
-	"reflect"
 	"strings"
 	"testing"
 	"testing/quick"
 	"time"
 
-	"github.com/calmh/xdr"
+	"github.com/syncthing/syncthing/lib/rand"
 )
 
 var (
@@ -26,64 +22,6 @@ var (
 	quickCfg = &quick.Config{}
 )
 
-func TestHeaderEncodeDecode(t *testing.T) {
-	f := func(ver, id, typ int) bool {
-		ver = int(uint(ver) % 16)
-		id = int(uint(id) % 4096)
-		typ = int(uint(typ) % 256)
-		h0 := header{version: ver, msgID: id, msgType: typ}
-		h1 := decodeHeader(encodeHeader(h0))
-		return h0 == h1
-	}
-	if err := quick.Check(f, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestHeaderMarshalUnmarshal(t *testing.T) {
-	f := func(ver, id, typ int) bool {
-		ver = int(uint(ver) % 16)
-		id = int(uint(id) % 4096)
-		typ = int(uint(typ) % 256)
-		buf := make([]byte, 4)
-
-		h0 := header{version: ver, msgID: id, msgType: typ}
-		h0.MarshalXDRInto(&xdr.Marshaller{Data: buf})
-
-		var h1 header
-		h1.UnmarshalXDRFrom(&xdr.Unmarshaller{Data: buf})
-		return h0 == h1
-	}
-	if err := quick.Check(f, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestHeaderLayout(t *testing.T) {
-	var e, a uint32
-
-	// Version are the first four bits
-	e = 0xf0000000
-	a = encodeHeader(header{version: 0xf})
-	if a != e {
-		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
-	}
-
-	// Message ID are the following 12 bits
-	e = 0x0fff0000
-	a = encodeHeader(header{msgID: 0xfff})
-	if a != e {
-		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
-	}
-
-	// Type are the last 8 bits before reserved
-	e = 0x0000ff00
-	a = encodeHeader(header{msgType: 0xff})
-	if a != e {
-		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
-	}
-}
-
 func TestPing(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
@@ -92,8 +30,8 @@ func TestPing(t *testing.T) {
 	c0.Start()
 	c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).Connection.(*rawConnection)
 	c1.Start()
-	c0.ClusterConfig(ClusterConfigMessage{})
-	c1.ClusterConfig(ClusterConfigMessage{})
+	c0.ClusterConfig(ClusterConfig{})
+	c1.ClusterConfig(ClusterConfig{})
 
 	if ok := c0.ping(); !ok {
 		t.Error("c0 ping failed")
@@ -103,56 +41,6 @@ func TestPing(t *testing.T) {
 	}
 }
 
-func TestVersionErr(t *testing.T) {
-	m0 := newTestModel()
-	m1 := newTestModel()
-
-	ar, aw := io.Pipe()
-	br, bw := io.Pipe()
-
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).Connection.(*rawConnection)
-	c0.Start()
-	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
-	c1.Start()
-	c0.ClusterConfig(ClusterConfigMessage{})
-	c1.ClusterConfig(ClusterConfigMessage{})
-
-	timeoutWriteHeader(c0.cw, header{
-		version: 2, // higher than supported
-		msgID:   0,
-		msgType: messageTypeIndex,
-	})
-
-	if err := m1.closedError(); err == nil || !strings.Contains(err.Error(), "unknown protocol version") {
-		t.Error("Connection should close due to unknown version, not", err)
-	}
-}
-
-func TestTypeErr(t *testing.T) {
-	m0 := newTestModel()
-	m1 := newTestModel()
-
-	ar, aw := io.Pipe()
-	br, bw := io.Pipe()
-
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).Connection.(*rawConnection)
-	c0.Start()
-	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
-	c1.Start()
-	c0.ClusterConfig(ClusterConfigMessage{})
-	c1.ClusterConfig(ClusterConfigMessage{})
-
-	timeoutWriteHeader(c0.cw, header{
-		version: 0,
-		msgID:   0,
-		msgType: 42, // unknown type
-	})
-
-	if err := m1.closedError(); err == nil || !strings.Contains(err.Error(), "unknown message type") {
-		t.Error("Connection should close due to unknown message type, not", err)
-	}
-}
-
 func TestClose(t *testing.T) {
 	m0 := newTestModel()
 	m1 := newTestModel()
@@ -164,8 +52,8 @@ func TestClose(t *testing.T) {
 	c0.Start()
 	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
 	c1.Start()
-	c0.ClusterConfig(ClusterConfigMessage{})
-	c1.ClusterConfig(ClusterConfigMessage{})
+	c0.ClusterConfig(ClusterConfig{})
+	c1.ClusterConfig(ClusterConfig{})
 
 	c0.close(errors.New("manual close"))
 
@@ -180,8 +68,8 @@ func TestClose(t *testing.T) {
 		t.Error("Ping should not return true")
 	}
 
-	c0.Index("default", nil, 0, nil)
-	c0.Index("default", nil, 0, nil)
+	c0.Index("default", nil)
+	c0.Index("default", nil)
 
 	if _, err := c0.Request("default", "foo", 0, 0, nil, false); err == nil {
 		t.Error("Request should return an error")
@@ -193,15 +81,11 @@ func TestMarshalIndexMessage(t *testing.T) {
 		quickCfg.MaxCount = 10
 	}
 
-	f := func(m1 IndexMessage) bool {
-		if len(m1.Options) == 0 {
-			m1.Options = nil
-		}
+	f := func(m1 Index) bool {
 		if len(m1.Files) == 0 {
 			m1.Files = nil
 		}
 		for i, f := range m1.Files {
-			m1.Files[i].CachedSize = 0
 			if len(f.Blocks) == 0 {
 				m1.Files[i].Blocks = nil
 			} else {
@@ -212,9 +96,12 @@ func TestMarshalIndexMessage(t *testing.T) {
 					}
 				}
 			}
+			if len(f.Version.Counters) == 0 {
+				m1.Files[i].Version.Counters = nil
+			}
 		}
 
-		return testMarshal(t, "index", &m1, &IndexMessage{})
+		return testMarshal(t, "index", &m1, &Index{})
 	}
 
 	if err := quick.Check(f, quickCfg); err != nil {
@@ -227,14 +114,11 @@ func TestMarshalRequestMessage(t *testing.T) {
 		quickCfg.MaxCount = 10
 	}
 
-	f := func(m1 RequestMessage) bool {
-		if len(m1.Options) == 0 {
-			m1.Options = nil
-		}
+	f := func(m1 Request) bool {
 		if len(m1.Hash) == 0 {
 			m1.Hash = nil
 		}
-		return testMarshal(t, "request", &m1, &RequestMessage{})
+		return testMarshal(t, "request", &m1, &Request{})
 	}
 
 	if err := quick.Check(f, quickCfg); err != nil {
@@ -247,11 +131,11 @@ func TestMarshalResponseMessage(t *testing.T) {
 		quickCfg.MaxCount = 10
 	}
 
-	f := func(m1 ResponseMessage) bool {
+	f := func(m1 Response) bool {
 		if len(m1.Data) == 0 {
 			m1.Data = nil
 		}
-		return testMarshal(t, "response", &m1, &ResponseMessage{})
+		return testMarshal(t, "response", &m1, &Response{})
 	}
 
 	if err := quick.Check(f, quickCfg); err != nil {
@@ -264,10 +148,7 @@ func TestMarshalClusterConfigMessage(t *testing.T) {
 		quickCfg.MaxCount = 10
 	}
 
-	f := func(m1 ClusterConfigMessage) bool {
-		if len(m1.Options) == 0 {
-			m1.Options = nil
-		}
+	f := func(m1 ClusterConfig) bool {
 		if len(m1.Folders) == 0 {
 			m1.Folders = nil
 		}
@@ -275,11 +156,8 @@ func TestMarshalClusterConfigMessage(t *testing.T) {
 			if len(m1.Folders[i].Devices) == 0 {
 				m1.Folders[i].Devices = nil
 			}
-			if len(m1.Folders[i].Options) == 0 {
-				m1.Folders[i].Options = nil
-			}
 		}
-		return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{})
+		return testMarshal(t, "clusterconfig", &m1, &ClusterConfig{})
 	}
 
 	if err := quick.Check(f, quickCfg); err != nil {
@@ -292,8 +170,8 @@ func TestMarshalCloseMessage(t *testing.T) {
 		quickCfg.MaxCount = 10
 	}
 
-	f := func(m1 CloseMessage) bool {
-		return testMarshal(t, "close", &m1, &CloseMessage{})
+	f := func(m1 Close) bool {
+		return testMarshal(t, "close", &m1, &Close{})
 	}
 
 	if err := quick.Check(f, quickCfg); err != nil {
@@ -301,108 +179,97 @@ func TestMarshalCloseMessage(t *testing.T) {
 	}
 }
 
-type message interface {
-	MarshalXDR() ([]byte, error)
-	UnmarshalXDR([]byte) error
-}
-
 func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
-	failed := func(bc []byte) {
-		bs, _ := json.MarshalIndent(m1, "", "  ")
-		ioutil.WriteFile(prefix+"-1.txt", bs, 0644)
-		bs, _ = json.MarshalIndent(m2, "", "  ")
-		ioutil.WriteFile(prefix+"-2.txt", bs, 0644)
-		if len(bc) > 0 {
-			f, _ := os.Create(prefix + "-data.txt")
-			fmt.Fprint(f, hex.Dump(bc))
-			f.Close()
+	buf, err := m1.Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = m2.Unmarshal(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs1, _ := json.MarshalIndent(m1, "", "  ")
+	bs2, _ := json.MarshalIndent(m2, "", "  ")
+	if !bytes.Equal(bs1, bs2) {
+		ioutil.WriteFile(prefix+"-1.txt", bs1, 0644)
+		ioutil.WriteFile(prefix+"-2.txt", bs1, 0644)
+		return false
+	}
+
+	return true
+}
+
+func TestMarshalledIndexMessageSize(t *testing.T) {
+	// We should be able to handle a 1 TiB file without
+	// blowing the default max message size.
+
+	if testing.Short() {
+		t.Skip("this test requires a lot of memory")
+		return
+	}
+
+	const (
+		maxMessageSize = MaxMessageLen
+		fileSize       = 1 << 40
+		blockSize      = BlockSize
+	)
+
+	f := FileInfo{
+		Name:        "a normal length file name withoout any weird stuff.txt",
+		Type:        FileInfoTypeFile,
+		Size:        fileSize,
+		Permissions: 0666,
+		Modified:    time.Now().Unix(),
+		Version:     Vector{Counters: []Counter{{ID: 1 << 60, Value: 1}, {ID: 2 << 60, Value: 1}}},
+		Blocks:      make([]BlockInfo, fileSize/blockSize),
+	}
+
+	for i := 0; i < fileSize/blockSize; i++ {
+		f.Blocks[i].Offset = int64(i) * blockSize
+		f.Blocks[i].Size = blockSize
+		f.Blocks[i].Hash = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 1, 2}
+	}
+
+	idx := Index{
+		Folder: "some folder ID",
+		Files:  []FileInfo{f},
+	}
+
+	msgSize := idx.ProtoSize()
+	if msgSize > maxMessageSize {
+		t.Errorf("Message size %d bytes is larger than max %d", msgSize, maxMessageSize)
+	}
+}
+
+func TestLZ4Compression(t *testing.T) {
+	c := new(rawConnection)
+
+	for i := 0; i < 10; i++ {
+		dataLen := 150 + rand.Intn(150)
+		data := make([]byte, dataLen)
+		_, err := io.ReadFull(rand.Reader, data[100:])
+		if err != nil {
+			t.Fatal(err)
+		}
+		comp, err := c.lz4Compress(data)
+		if err != nil {
+			t.Errorf("compressing %d bytes: %v", dataLen, err)
+			continue
 		}
-	}
 
-	buf, err := m1.MarshalXDR()
-	if err != nil && strings.Contains(err.Error(), "exceeds size") {
-		return true
-	}
-	if err != nil {
-		failed(nil)
-		t.Fatal(err)
-	}
-
-	err = m2.UnmarshalXDR(buf)
-	if err != nil {
-		failed(buf)
-		t.Fatal(err)
-	}
-
-	ok := reflect.DeepEqual(m1, m2)
-	if !ok {
-		failed(buf)
-	}
-	return ok
-}
-
-func timeoutWriteHeader(w io.Writer, hdr header) {
-	// This tries to write a message header to w, but times out after a while.
-	// This is useful because in testing, with a PipeWriter, it will block
-	// forever if the other side isn't reading any more. On the other hand we
-	// can't just "go" it into the background, because if the other side is
-	// still there we should wait for the write to complete. Yay.
-
-	var buf [8]byte // header and message length
-	binary.BigEndian.PutUint32(buf[:], encodeHeader(hdr))
-	binary.BigEndian.PutUint32(buf[4:], 0) // zero message length, explicitly
-
-	done := make(chan struct{})
-	go func() {
-		w.Write(buf[:])
-		close(done)
-	}()
-	select {
-	case <-done:
-	case <-time.After(250 * time.Millisecond):
-	}
-}
-
-func TestFileInfoSize(t *testing.T) {
-	fi := FileInfo{
-		Blocks: []BlockInfo{
-			{Size: 42},
-			{Offset: 42, Size: 23},
-			{Offset: 42 + 23, Size: 34},
-		},
-	}
-
-	size := fi.Size()
-	want := int64(42 + 23 + 34)
-	if size != want {
-		t.Errorf("Incorrect size reported, got %d, want %d", size, want)
-	}
-
-	size = fi.Size() // Cached, this time
-	if size != want {
-		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
-	}
-
-	fi.CachedSize = 8
-	want = 8
-	size = fi.Size() // Ensure it came from the cache
-	if size != want {
-		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
-	}
-
-	fi.CachedSize = 0
-	fi.Flags = FlagDirectory
-	want = 128
-	size = fi.Size() // Directories are 128 bytes large
-	if size != want {
-		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
-	}
-
-	fi.CachedSize = 0
-	fi.Flags = FlagDeleted
-	want = 128
-	size = fi.Size() // Also deleted files
-	if size != want {
-		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
+		res, err := c.lz4Decompress(comp)
+		if err != nil {
+			t.Errorf("decompressing %d bytes to %d: %v", len(comp), dataLen, err)
+			continue
+		}
+		if len(res) != len(data) {
+			t.Errorf("Incorrect len %d != expected %d", len(res), len(data))
+		}
+		if !bytes.Equal(data, res) {
+			t.Error("Incorrect decompressed data")
+		}
+		t.Logf("OK #%d, %d -> %d -> %d", i, dataLen, len(comp), dataLen)
 	}
 }

lib/protocol/vector.go

@@ -6,35 +6,30 @@ package protocol
 // version vector. The vector has slice semantics and some operations on it
 // are "append-like" in that they may return the same vector modified, or v
 // new allocated Vector with the modified contents.
-type Vector []Counter
 
 // Counter represents a single counter in the version vector.
-type Counter struct {
-	ID    ShortID
-	Value uint64
-}
 
 // Update returns a Vector with the index for the specific ID incremented by
 // one. If it is possible, the vector v is updated and returned. If it is not,
 // a copy will be created, updated and returned.
 func (v Vector) Update(id ShortID) Vector {
-	for i := range v {
-		if v[i].ID == id {
+	for i := range v.Counters {
+		if v.Counters[i].ID == id {
 			// Update an existing index
-			v[i].Value++
+			v.Counters[i].Value++
 			return v
-		} else if v[i].ID > id {
+		} else if v.Counters[i].ID > id {
 			// Insert a new index
-			nv := make(Vector, len(v)+1)
-			copy(nv, v[:i])
+			nv := make([]Counter, len(v.Counters)+1)
+			copy(nv, v.Counters[:i])
 			nv[i].ID = id
 			nv[i].Value = 1
-			copy(nv[i+1:], v[i:])
-			return nv
+			copy(nv[i+1:], v.Counters[i:])
+			return Vector{nv}
 		}
 	}
 	// Append a new index
-	return append(v, Counter{id, 1})
+	return Vector{append(v.Counters, Counter{id, 1})}
 }
 
 // Merge returns the vector containing the maximum indexes from v and b. If it
@@ -42,28 +37,28 @@ func (v Vector) Update(id ShortID) Vector {
 // will be created, updated and returned.
 func (v Vector) Merge(b Vector) Vector {
 	var vi, bi int
-	for bi < len(b) {
-		if vi == len(v) {
+	for bi < len(b.Counters) {
+		if vi == len(v.Counters) {
 			// We've reach the end of v, all that remains are appends
-			return append(v, b[bi:]...)
+			return Vector{append(v.Counters, b.Counters[bi:]...)}
 		}
 
-		if v[vi].ID > b[bi].ID {
+		if v.Counters[vi].ID > b.Counters[bi].ID {
 			// The index from b should be inserted here
-			n := make(Vector, len(v)+1)
-			copy(n, v[:vi])
-			n[vi] = b[bi]
-			copy(n[vi+1:], v[vi:])
-			v = n
+			n := make([]Counter, len(v.Counters)+1)
+			copy(n, v.Counters[:vi])
+			n[vi] = b.Counters[bi]
+			copy(n[vi+1:], v.Counters[vi:])
+			v.Counters = n
 		}
 
-		if v[vi].ID == b[bi].ID {
-			if val := b[bi].Value; val > v[vi].Value {
-				v[vi].Value = val
+		if v.Counters[vi].ID == b.Counters[bi].ID {
+			if val := b.Counters[bi].Value; val > v.Counters[vi].Value {
+				v.Counters[vi].Value = val
 			}
 		}
 
-		if bi < len(b) && v[vi].ID == b[bi].ID {
+		if bi < len(b.Counters) && v.Counters[vi].ID == b.Counters[bi].ID {
 			bi++
 		}
 		vi++
@@ -74,9 +69,9 @@ func (v Vector) Merge(b Vector) Vector {
 
 // Copy returns an identical vector that is not shared with v.
 func (v Vector) Copy() Vector {
-	nv := make(Vector, len(v))
-	copy(nv, v)
-	return nv
+	nv := make([]Counter, len(v.Counters))
+	copy(nv, v.Counters)
+	return Vector{nv}
 }
 
 // Equal returns true when the two vectors are equivalent.
@@ -106,10 +101,96 @@ func (v Vector) Concurrent(b Vector) bool {
 
 // Counter returns the current value of the given counter ID.
 func (v Vector) Counter(id ShortID) uint64 {
-	for _, c := range v {
+	for _, c := range v.Counters {
 		if c.ID == id {
 			return c.Value
 		}
 	}
 	return 0
 }
+
+// Ordering represents the relationship between two Vectors.
+type Ordering int
+
+const (
+	Equal Ordering = iota
+	Greater
+	Lesser
+	ConcurrentLesser
+	ConcurrentGreater
+)
+
+// There's really no such thing as "concurrent lesser" and "concurrent
+// greater" in version vectors, just "concurrent". But it's useful to be able
+// to get a strict ordering between versions for stable sorts and so on, so we
+// return both variants. The convenience method Concurrent() can be used to
+// check for either case.
+
+// Compare returns the Ordering that describes a's relation to b.
+func (v Vector) Compare(b Vector) Ordering {
+	var ai, bi int     // index into a and b
+	var av, bv Counter // value at current index
+
+	result := Equal
+
+	for ai < len(v.Counters) || bi < len(b.Counters) {
+		var aMissing, bMissing bool
+
+		if ai < len(v.Counters) {
+			av = v.Counters[ai]
+		} else {
+			av = Counter{}
+			aMissing = true
+		}
+
+		if bi < len(b.Counters) {
+			bv = b.Counters[bi]
+		} else {
+			bv = Counter{}
+			bMissing = true
+		}
+
+		switch {
+		case av.ID == bv.ID:
+			// We have a counter value for each side
+			if av.Value > bv.Value {
+				if result == Lesser {
+					return ConcurrentLesser
+				}
+				result = Greater
+			} else if av.Value < bv.Value {
+				if result == Greater {
+					return ConcurrentGreater
+				}
+				result = Lesser
+			}
+
+		case !aMissing && av.ID < bv.ID || bMissing:
+			// Value is missing on the b side
+			if av.Value > 0 {
+				if result == Lesser {
+					return ConcurrentLesser
+				}
+				result = Greater
+			}
+
+		case !bMissing && bv.ID < av.ID || aMissing:
+			// Value is missing on the a side
+			if bv.Value > 0 {
+				if result == Greater {
+					return ConcurrentGreater
+				}
+				result = Lesser
+			}
+		}
+
+		if ai < len(v.Counters) && (av.ID <= bv.ID || bMissing) {
+			ai++
+		}
+		if bi < len(b.Counters) && (bv.ID <= av.ID || aMissing) {
+			bi++
+		}
+	}
+
+	return result
+}

lib/protocol/vector_compare.go

@@ -1,89 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-package protocol
-
-// Ordering represents the relationship between two Vectors.
-type Ordering int
-
-const (
-	Equal Ordering = iota
-	Greater
-	Lesser
-	ConcurrentLesser
-	ConcurrentGreater
-)
-
-// There's really no such thing as "concurrent lesser" and "concurrent
-// greater" in version vectors, just "concurrent". But it's useful to be able
-// to get a strict ordering between versions for stable sorts and so on, so we
-// return both variants. The convenience method Concurrent() can be used to
-// check for either case.
-
-// Compare returns the Ordering that describes a's relation to b.
-func (a Vector) Compare(b Vector) Ordering {
-	var ai, bi int     // index into a and b
-	var av, bv Counter // value at current index
-
-	result := Equal
-
-	for ai < len(a) || bi < len(b) {
-		var aMissing, bMissing bool
-
-		if ai < len(a) {
-			av = a[ai]
-		} else {
-			av = Counter{}
-			aMissing = true
-		}
-
-		if bi < len(b) {
-			bv = b[bi]
-		} else {
-			bv = Counter{}
-			bMissing = true
-		}
-
-		switch {
-		case av.ID == bv.ID:
-			// We have a counter value for each side
-			if av.Value > bv.Value {
-				if result == Lesser {
-					return ConcurrentLesser
-				}
-				result = Greater
-			} else if av.Value < bv.Value {
-				if result == Greater {
-					return ConcurrentGreater
-				}
-				result = Lesser
-			}
-
-		case !aMissing && av.ID < bv.ID || bMissing:
-			// Value is missing on the b side
-			if av.Value > 0 {
-				if result == Lesser {
-					return ConcurrentLesser
-				}
-				result = Greater
-			}
-
-		case !bMissing && bv.ID < av.ID || aMissing:
-			// Value is missing on the a side
-			if bv.Value > 0 {
-				if result == Greater {
-					return ConcurrentGreater
-				}
-				result = Lesser
-			}
-		}
-
-		if ai < len(a) && (av.ID <= bv.ID || bMissing) {
-			ai++
-		}
-		if bi < len(b) && (bv.ID <= av.ID || aMissing) {
-			bi++
-		}
-	}
-
-	return result
-}

lib/protocol/vector_compare_test.go

@@ -1,249 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-package protocol
-
-import (
-	"math"
-	"testing"
-)
-
-func TestCompare(t *testing.T) {
-	testcases := []struct {
-		a, b Vector
-		r    Ordering
-	}{
-		// Empty vectors are identical
-		{Vector{}, Vector{}, Equal},
-		{Vector{}, nil, Equal},
-		{nil, Vector{}, Equal},
-		{nil, Vector{Counter{42, 0}}, Equal},
-		{Vector{}, Vector{Counter{42, 0}}, Equal},
-		{Vector{Counter{42, 0}}, nil, Equal},
-		{Vector{Counter{42, 0}}, Vector{}, Equal},
-
-		// Zero is the implied value for a missing Counter
-		{
-			Vector{Counter{42, 0}},
-			Vector{Counter{77, 0}},
-			Equal,
-		},
-
-		// Equal vectors are equal
-		{
-			Vector{Counter{42, 33}},
-			Vector{Counter{42, 33}},
-			Equal,
-		},
-		{
-			Vector{Counter{42, 33}, Counter{77, 24}},
-			Vector{Counter{42, 33}, Counter{77, 24}},
-			Equal,
-		},
-
-		// These a-vectors are all greater than the b-vector
-		{
-			Vector{Counter{42, 1}},
-			nil,
-			Greater,
-		},
-		{
-			Vector{Counter{42, 1}},
-			Vector{},
-			Greater,
-		},
-		{
-			Vector{Counter{0, 1}},
-			Vector{Counter{0, 0}},
-			Greater,
-		},
-		{
-			Vector{Counter{42, 1}},
-			Vector{Counter{42, 0}},
-			Greater,
-		},
-		{
-			Vector{Counter{math.MaxUint64, 1}},
-			Vector{Counter{math.MaxUint64, 0}},
-			Greater,
-		},
-		{
-			Vector{Counter{0, math.MaxUint64}},
-			Vector{Counter{0, 0}},
-			Greater,
-		},
-		{
-			Vector{Counter{42, math.MaxUint64}},
-			Vector{Counter{42, 0}},
-			Greater,
-		},
-		{
-			Vector{Counter{math.MaxUint64, math.MaxUint64}},
-			Vector{Counter{math.MaxUint64, 0}},
-			Greater,
-		},
-		{
-			Vector{Counter{0, math.MaxUint64}},
-			Vector{Counter{0, math.MaxUint64 - 1}},
-			Greater,
-		},
-		{
-			Vector{Counter{42, math.MaxUint64}},
-			Vector{Counter{42, math.MaxUint64 - 1}},
-			Greater,
-		},
-		{
-			Vector{Counter{math.MaxUint64, math.MaxUint64}},
-			Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}},
-			Greater,
-		},
-		{
-			Vector{Counter{42, 2}},
-			Vector{Counter{42, 1}},
-			Greater,
-		},
-		{
-			Vector{Counter{22, 22}, Counter{42, 2}},
-			Vector{Counter{22, 22}, Counter{42, 1}},
-			Greater,
-		},
-		{
-			Vector{Counter{42, 2}, Counter{77, 3}},
-			Vector{Counter{42, 1}, Counter{77, 3}},
-			Greater,
-		},
-		{
-			Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
-			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
-			Greater,
-		},
-		{
-			Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
-			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
-			Greater,
-		},
-
-		// These a-vectors are all lesser than the b-vector
-		{nil, Vector{Counter{42, 1}}, Lesser},
-		{Vector{}, Vector{Counter{42, 1}}, Lesser},
-		{
-			Vector{Counter{42, 0}},
-			Vector{Counter{42, 1}},
-			Lesser,
-		},
-		{
-			Vector{Counter{42, 1}},
-			Vector{Counter{42, 2}},
-			Lesser,
-		},
-		{
-			Vector{Counter{22, 22}, Counter{42, 1}},
-			Vector{Counter{22, 22}, Counter{42, 2}},
-			Lesser,
-		},
-		{
-			Vector{Counter{42, 1}, Counter{77, 3}},
-			Vector{Counter{42, 2}, Counter{77, 3}},
-			Lesser,
-		},
-		{
-			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
-			Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
-			Lesser,
-		},
-		{
-			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
-			Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
-			Lesser,
-		},
-
-		// These are all in conflict
-		{
-			Vector{Counter{42, 2}},
-			Vector{Counter{43, 1}},
-			ConcurrentGreater,
-		},
-		{
-			Vector{Counter{43, 1}},
-			Vector{Counter{42, 2}},
-			ConcurrentLesser,
-		},
-		{
-			Vector{Counter{22, 23}, Counter{42, 1}},
-			Vector{Counter{22, 22}, Counter{42, 2}},
-			ConcurrentGreater,
-		},
-		{
-			Vector{Counter{22, 21}, Counter{42, 2}},
-			Vector{Counter{22, 22}, Counter{42, 1}},
-			ConcurrentLesser,
-		},
-		{
-			Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}},
-			Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}},
-			ConcurrentLesser,
-		},
-	}
-
-	for i, tc := range testcases {
-		// Test real Compare
-		if r := tc.a.Compare(tc.b); r != tc.r {
-			t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r)
-		}
-
-		// Test convenience functions
-		switch tc.r {
-		case Greater:
-			if tc.a.Equal(tc.b) {
-				t.Errorf("%+v == %+v", tc.a, tc.b)
-			}
-			if tc.a.Concurrent(tc.b) {
-				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
-			}
-			if !tc.a.GreaterEqual(tc.b) {
-				t.Errorf("%+v not >= %+v", tc.a, tc.b)
-			}
-			if tc.a.LesserEqual(tc.b) {
-				t.Errorf("%+v <= %+v", tc.a, tc.b)
-			}
-		case Lesser:
-			if tc.a.Concurrent(tc.b) {
-				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
-			}
-			if tc.a.Equal(tc.b) {
-				t.Errorf("%+v == %+v", tc.a, tc.b)
-			}
-			if tc.a.GreaterEqual(tc.b) {
-				t.Errorf("%+v >= %+v", tc.a, tc.b)
-			}
-			if !tc.a.LesserEqual(tc.b) {
-				t.Errorf("%+v not <= %+v", tc.a, tc.b)
-			}
-		case Equal:
-			if tc.a.Concurrent(tc.b) {
-				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
-			}
-			if !tc.a.Equal(tc.b) {
-				t.Errorf("%+v not == %+v", tc.a, tc.b)
-			}
-			if !tc.a.GreaterEqual(tc.b) {
-				t.Errorf("%+v not <= %+v", tc.a, tc.b)
-			}
-			if !tc.a.LesserEqual(tc.b) {
-				t.Errorf("%+v not <= %+v", tc.a, tc.b)
-			}
-		case ConcurrentLesser, ConcurrentGreater:
-			if !tc.a.Concurrent(tc.b) {
-				t.Errorf("%+v not concurrent %+v", tc.a, tc.b)
-			}
-			if tc.a.Equal(tc.b) {
-				t.Errorf("%+v == %+v", tc.a, tc.b)
-			}
-			if tc.a.GreaterEqual(tc.b) {
-				t.Errorf("%+v >= %+v", tc.a, tc.b)
-			}
-			if tc.a.LesserEqual(tc.b) {
-				t.Errorf("%+v <= %+v", tc.a, tc.b)
-			}
-		}
-	}
-}

lib/protocol/vector_test.go

@@ -2,7 +2,10 @@
 
 package protocol
 
-import "testing"
+import (
+	"math"
+	"testing"
+)
 
 func TestUpdate(t *testing.T) {
 	var v Vector
@@ -10,7 +13,7 @@ func TestUpdate(t *testing.T) {
 	// Append
 
 	v = v.Update(42)
-	expected := Vector{Counter{42, 1}}
+	expected := Vector{[]Counter{{42, 1}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)
@@ -19,7 +22,7 @@ func TestUpdate(t *testing.T) {
 	// Insert at front
 
 	v = v.Update(36)
-	expected = Vector{Counter{36, 1}, Counter{42, 1}}
+	expected = Vector{[]Counter{{36, 1}, {42, 1}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)
@@ -28,7 +31,7 @@ func TestUpdate(t *testing.T) {
 	// Insert in moddle
 
 	v = v.Update(37)
-	expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}}
+	expected = Vector{[]Counter{{36, 1}, {37, 1}, {42, 1}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)
@@ -37,7 +40,7 @@ func TestUpdate(t *testing.T) {
 	// Update existing
 
 	v = v.Update(37)
-	expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}}
+	expected = Vector{[]Counter{{36, 1}, {37, 2}, {42, 1}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)
@@ -45,7 +48,7 @@ func TestUpdate(t *testing.T) {
 }
 
 func TestCopy(t *testing.T) {
-	v0 := Vector{Counter{42, 1}}
+	v0 := Vector{[]Counter{{42, 1}}}
 	v1 := v0.Copy()
 	v1.Update(42)
 	if v0.Compare(v1) != Lesser {
@@ -64,52 +67,52 @@ func TestMerge(t *testing.T) {
 			Vector{},
 		},
 		{
-			Vector{Counter{22, 1}, Counter{42, 1}},
-			Vector{Counter{22, 1}, Counter{42, 1}},
-			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
 		},
 
 		// Appends
 		{
 			Vector{},
-			Vector{Counter{22, 1}, Counter{42, 1}},
-			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
 		},
 		{
-			Vector{Counter{22, 1}},
-			Vector{Counter{42, 1}},
-			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{[]Counter{{22, 1}}},
+			Vector{[]Counter{{42, 1}}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
 		},
 		{
-			Vector{Counter{22, 1}},
-			Vector{Counter{22, 1}, Counter{42, 1}},
-			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{[]Counter{{22, 1}}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
 		},
 
 		// Insert
 		{
-			Vector{Counter{22, 1}, Counter{42, 1}},
-			Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
-			Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
+			Vector{[]Counter{{22, 1}, {23, 2}, {42, 1}}},
+			Vector{[]Counter{{22, 1}, {23, 2}, {42, 1}}},
 		},
 		{
-			Vector{Counter{42, 1}},
-			Vector{Counter{22, 1}},
-			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{[]Counter{{42, 1}}},
+			Vector{[]Counter{{22, 1}}},
+			Vector{[]Counter{{22, 1}, {42, 1}}},
 		},
 
 		// Update
 		{
-			Vector{Counter{22, 1}, Counter{42, 2}},
-			Vector{Counter{22, 2}, Counter{42, 1}},
-			Vector{Counter{22, 2}, Counter{42, 2}},
+			Vector{[]Counter{{22, 1}, {42, 2}}},
+			Vector{[]Counter{{22, 2}, {42, 1}}},
+			Vector{[]Counter{{22, 2}, {42, 2}}},
 		},
 
 		// All of the above
 		{
-			Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}},
-			Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}},
-			Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}},
+			Vector{[]Counter{{10, 1}, {20, 2}, {30, 1}}},
+			Vector{[]Counter{{5, 1}, {10, 2}, {15, 1}, {20, 1}, {25, 1}, {35, 1}}},
+			Vector{[]Counter{{5, 1}, {10, 2}, {15, 1}, {20, 2}, {25, 1}, {30, 1}, {35, 1}}},
 		},
 	}
 
@@ -121,7 +124,7 @@ func TestMerge(t *testing.T) {
 }
 
 func TestCounterValue(t *testing.T) {
-	v0 := Vector{Counter{42, 1}, Counter{64, 5}}
+	v0 := Vector{[]Counter{{42, 1}, {64, 5}}}
 	if v0.Counter(42) != 1 {
 		t.Errorf("Counter error, %d != %d", v0.Counter(42), 1)
 	}
@@ -132,3 +135,234 @@ func TestCounterValue(t *testing.T) {
 		t.Errorf("Counter error, %d != %d", v0.Counter(72), 0)
 	}
 }
+
+func TestCompare(t *testing.T) {
+	testcases := []struct {
+		a, b Vector
+		r    Ordering
+	}{
+		// Empty vectors are identical
+		{Vector{}, Vector{}, Equal},
+		{Vector{}, Vector{[]Counter{{42, 0}}}, Equal},
+		{Vector{[]Counter{Counter{42, 0}}}, Vector{}, Equal},
+
+		// Zero is the implied value for a missing Counter
+		{
+			Vector{[]Counter{{42, 0}}},
+			Vector{[]Counter{{77, 0}}},
+			Equal,
+		},
+
+		// Equal vectors are equal
+		{
+			Vector{[]Counter{{42, 33}}},
+			Vector{[]Counter{{42, 33}}},
+			Equal,
+		},
+		{
+			Vector{[]Counter{{42, 33}, {77, 24}}},
+			Vector{[]Counter{{42, 33}, {77, 24}}},
+			Equal,
+		},
+
+		// These a-vectors are all greater than the b-vector
+		{
+			Vector{[]Counter{{42, 1}}},
+			Vector{},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{0, 1}}},
+			Vector{[]Counter{{0, 0}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{42, 1}}},
+			Vector{[]Counter{{42, 0}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{math.MaxUint64, 1}}},
+			Vector{[]Counter{{math.MaxUint64, 0}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{0, math.MaxUint64}}},
+			Vector{[]Counter{{0, 0}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{42, math.MaxUint64}}},
+			Vector{[]Counter{{42, 0}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{math.MaxUint64, math.MaxUint64}}},
+			Vector{[]Counter{{math.MaxUint64, 0}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{0, math.MaxUint64}}},
+			Vector{[]Counter{{0, math.MaxUint64 - 1}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{42, math.MaxUint64}}},
+			Vector{[]Counter{{42, math.MaxUint64 - 1}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{math.MaxUint64, math.MaxUint64}}},
+			Vector{[]Counter{{math.MaxUint64, math.MaxUint64 - 1}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{42, 2}}},
+			Vector{[]Counter{{42, 1}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{22, 22}, {42, 2}}},
+			Vector{[]Counter{{22, 22}, {42, 1}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{42, 2}, {77, 3}}},
+			Vector{[]Counter{{42, 1}, {77, 3}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{22, 22}, {42, 2}, {77, 3}}},
+			Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
+			Greater,
+		},
+		{
+			Vector{[]Counter{{22, 23}, {42, 2}, {77, 4}}},
+			Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
+			Greater,
+		},
+
+		// These a-vectors are all lesser than the b-vector
+		{Vector{}, Vector{[]Counter{{42, 1}}}, Lesser},
+		{
+			Vector{[]Counter{{42, 0}}},
+			Vector{[]Counter{{42, 1}}},
+			Lesser,
+		},
+		{
+			Vector{[]Counter{{42, 1}}},
+			Vector{[]Counter{{42, 2}}},
+			Lesser,
+		},
+		{
+			Vector{[]Counter{{22, 22}, {42, 1}}},
+			Vector{[]Counter{{22, 22}, {42, 2}}},
+			Lesser,
+		},
+		{
+			Vector{[]Counter{{42, 1}, {77, 3}}},
+			Vector{[]Counter{{42, 2}, {77, 3}}},
+			Lesser,
+		},
+		{
+			Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
+			Vector{[]Counter{{22, 22}, {42, 2}, {77, 3}}},
+			Lesser,
+		},
+		{
+			Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
+			Vector{[]Counter{{22, 23}, {42, 2}, {77, 4}}},
+			Lesser,
+		},
+
+		// These are all in conflict
+		{
+			Vector{[]Counter{{42, 2}}},
+			Vector{[]Counter{{43, 1}}},
+			ConcurrentGreater,
+		},
+		{
+			Vector{[]Counter{{43, 1}}},
+			Vector{[]Counter{{42, 2}}},
+			ConcurrentLesser,
+		},
+		{
+			Vector{[]Counter{{22, 23}, {42, 1}}},
+			Vector{[]Counter{{22, 22}, {42, 2}}},
+			ConcurrentGreater,
+		},
+		{
+			Vector{[]Counter{{22, 21}, {42, 2}}},
+			Vector{[]Counter{{22, 22}, {42, 1}}},
+			ConcurrentLesser,
+		},
+		{
+			Vector{[]Counter{{22, 21}, {42, 2}, {43, 1}}},
+			Vector{[]Counter{{20, 1}, {22, 22}, {42, 1}}},
+			ConcurrentLesser,
+		},
+	}
+
+	for i, tc := range testcases {
+		// Test real Compare
+		if r := tc.a.Compare(tc.b); r != tc.r {
+			t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r)
+		}
+
+		// Test convenience functions
+		switch tc.r {
+		case Greater:
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if !tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v not >= %+v", tc.a, tc.b)
+			}
+			if tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v <= %+v", tc.a, tc.b)
+			}
+		case Lesser:
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v >= %+v", tc.a, tc.b)
+			}
+			if !tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+		case Equal:
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if !tc.a.Equal(tc.b) {
+				t.Errorf("%+v not == %+v", tc.a, tc.b)
+			}
+			if !tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+			if !tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+		case ConcurrentLesser, ConcurrentGreater:
+			if !tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v not concurrent %+v", tc.a, tc.b)
+			}
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v >= %+v", tc.a, tc.b)
+			}
+			if tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v <= %+v", tc.a, tc.b)
+			}
+		}
+	}
+}

lib/protocol/vector_xdr.go

@@ -1,35 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-package protocol
-
-import "github.com/calmh/xdr"
-
-// This stuff is hacked up manually because genxdr doesn't support 'type
-// Vector []Counter' declarations and it was tricky when I tried to add it...
-
-func (v Vector) MarshalXDRInto(m *xdr.Marshaller) error {
-	m.MarshalUint32(uint32(len(v)))
-	for i := range v {
-		m.MarshalUint64(uint64(v[i].ID))
-		m.MarshalUint64(v[i].Value)
-	}
-	return m.Error
-}
-
-func (v *Vector) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	l := int(u.UnmarshalUint32())
-	if l > 1e6 {
-		return xdr.ElementSizeExceeded("number of counters", l, 1e6)
-	}
-	n := make(Vector, l)
-	for i := range n {
-		n[i].ID = ShortID(u.UnmarshalUint64())
-		n[i].Value = u.UnmarshalUint64()
-	}
-	*v = n
-	return u.Error
-}
-
-func (v Vector) XDRSize() int {
-	return 4 + 16*len(v)
-}

lib/protocol/wireformat.go

@@ -12,7 +12,7 @@ type wireFormatConnection struct {
 	Connection
 }
 
-func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error {
+func (c wireFormatConnection) Index(folder string, fs []FileInfo) error {
 	var myFs = make([]FileInfo, len(fs))
 	copy(myFs, fs)
 
@@ -20,10 +20,10 @@ func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32,
 		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
 	}
 
-	return c.Connection.Index(folder, myFs, flags, options)
+	return c.Connection.Index(folder, myFs)
 }
 
-func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error {
+func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
 	var myFs = make([]FileInfo, len(fs))
 	copy(myFs, fs)
 
@@ -31,7 +31,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags ui
 		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
 	}
 
-	return c.Connection.IndexUpdate(folder, myFs, flags, options)
+	return c.Connection.IndexUpdate(folder, myFs)
 }
 
 func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {