Merge branch 'master' into v0.12

* master:
  Update docs and translations
  Model.internalScanFolder: Don't ignore special .stfolder and .stignore files.
  Model.internalScanFolderSubs: Scan only requested subs.
  A couple of protocol tests
  Humanize serialization of version vectors (again)
  FetchLatestReleases: fix the error log message
  Add postinst script to restart after upgrade
  Templatize Debian files
  Don't require restart for usage reporting changes (fixes #2704)
  RLimit comment typo

build.go

@@ -26,6 +26,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 )
 
@@ -331,33 +332,32 @@ func buildDeb() {
 		}
 	}
 
-	control := `Package: syncthing
-Architecture: {{arch}}
-Depends: libc6
-Version: {{version}}
-Maintainer: Syncthing Release Management <release@syncthing.net>
-Description: Open Source Continuous File Synchronization
-	Syncthing does bidirectional synchronization of files between two or
-	more computers.
-`
-	changelog := `syncthing ({{version}}); urgency=medium
-
-  * Packaging of {{version}}.
-
- -- Jakob Borg <jakob@nym.se>  {{date}}
-`
-
-	control = strings.Replace(control, "{{arch}}", debarch, -1)
-	control = strings.Replace(control, "{{version}}", version[1:], -1)
-	changelog = strings.Replace(changelog, "{{arch}}", debarch, -1)
-	changelog = strings.Replace(changelog, "{{version}}", version[1:], -1)
-	changelog = strings.Replace(changelog, "{{date}}", time.Now().Format(time.RFC1123), -1)
-
 	os.MkdirAll("deb/DEBIAN", 0755)
-	ioutil.WriteFile("deb/DEBIAN/control", []byte(control), 0644)
-	ioutil.WriteFile("deb/DEBIAN/compat", []byte("9\n"), 0644)
-	ioutil.WriteFile("deb/DEBIAN/changelog", []byte(changelog), 0644)
 
+	data := map[string]string{
+		"arch":    debarch,
+		"version": version[1:],
+		"date":    time.Now().Format(time.RFC1123),
+	}
+	for _, file := range listFiles("debian") {
+		tpl, err := template.New(filepath.Base(file)).ParseFiles(file)
+		if err != nil {
+			log.Fatal(err)
+		}
+		outFile := filepath.Join("deb/DEBIAN", filepath.Base(file))
+		out, err := os.Create(outFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if err := tpl.Execute(out, data); err != nil {
+			log.Fatal(err)
+		}
+		if err := out.Close(); err != nil {
+			log.Fatal(err)
+		}
+		info, _ := os.Lstat(file)
+		os.Chmod(outFile, info.Mode())
+	}
 }
 
 func copyFile(src, dst string, perm os.FileMode) error {

cmd/syncthing/gui.go

@@ -1220,7 +1220,7 @@ type jsonVersionVector protocol.Vector
 func (v jsonVersionVector) MarshalJSON() ([]byte, error) {
 	res := make([]string, len(v))
 	for i, c := range v {
-		res[i] = fmt.Sprintf("%d:%d", c.ID, c.Value)
+		res[i] = fmt.Sprintf("%v:%d", c.ID, c.Value)
 	}
 	return json.Marshal(res)
 }

cmd/syncthing/main.go

@@ -1206,7 +1206,7 @@ func cleanConfigDirectory() {
 // short ID:s; that is, that the devices in the cluster all have unique
 // initial 64 bits.
 func checkShortIDs(cfg *config.Wrapper) error {
-	exists := make(map[uint64]protocol.DeviceID)
+	exists := make(map[protocol.ShortID]protocol.DeviceID)
 	for deviceID := range cfg.Devices() {
 		shortID := deviceID.Short()
 		if otherID, ok := exists[shortID]; ok {

debian/changelog

@@ -0,0 +1,5 @@
+syncthing ({{.version}}); urgency=medium
+
+  * Packaging of {{.version}}.
+
+ -- Syncthing Release Management <release@syncthing.net>  {{.date}}

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,8 @@
+Package: syncthing
+Architecture: {{.arch}}
+Depends: libc6, procps
+Version: {{.version}}
+Maintainer: Syncthing Release Management <release@syncthing.net>
+Description: Open Source Continuous File Synchronization
+	Syncthing does bidirectional synchronization of files between two or
+	more computers.

debian/postinst

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ ${1:-} == configure ]]; then
+	pkill -x -HUP syncthing || true
+fi

gui/default/assets/lang/lang-pt-BR.json

@@ -119,7 +119,7 @@
     "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Caminho para a pasta na máquina local. Será criado caso não exista. O caractere til (~) pode ser usado como um atalho para",
     "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "O caminho onde as versões serão salvas (deixe vazio para usar a pasta padrão .stversions dentro desta pasta).",
     "Pause": "Pausar",
-    "Paused": "Pausada",
+    "Paused": "Em pausa",
     "Please consult the release notes before performing a major upgrade.": "Por favor, consulte as notas de lançamento antes de atualizar para uma versão \"major\".",
     "Please set a GUI Authentication User and Password in the Settings dialog.": "Por favor, defina um nome de usuário e senha para acesso à interface web, nas configurações.",
     "Please wait": "Aguarde",

lib/model/model.go

@@ -67,7 +67,7 @@ type Model struct {
 	finder            *db.BlockFinder
 	progressEmitter   *ProgressEmitter
 	id                protocol.DeviceID
-	shortID           uint64
+	shortID           protocol.ShortID
 	cacheIgnoredFiles bool
 	protectedFiles    []string
 
@@ -1323,15 +1323,14 @@ func (m *Model) internalScanFolderSubs(folder string, subs []string) error {
 	var unifySubs []string
 nextSub:
 	for _, sub := range subs {
-		for sub != "" {
+		for sub != "" && sub != ".stfolder" && sub != ".stignore" {
-			parent := filepath.Dir(sub)
+			if _, ok = fs.Get(protocol.LocalDeviceID, sub); ok {
-			if parent == "." || parent == string(filepath.Separator) {
-				parent = ""
-			}
-			if _, ok = fs.Get(protocol.LocalDeviceID, parent); ok {
 				break
 			}
-			sub = parent
+			sub = filepath.Dir(sub)
+			if sub == "." || sub == string(filepath.Separator) {
+				sub = ""
+			}
 		}
 		for _, us := range unifySubs {
 			if strings.HasPrefix(sub, us) {
@@ -1959,7 +1958,16 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 		}
 	}
 
-	// All of the generic options require restart
+	// Some options don't require restart as those components handle it fine
+	// by themselves.
+	from.Options.URAccepted = to.Options.URAccepted
+	from.Options.URUniqueID = to.Options.URUniqueID
+	// All of the other generic options require restart. Or at least they may;
+	// removing this check requires going through those options carefully and
+	// making sure there are individual services that handle them correctly.
+	// This code is the "original" requires-restart check and protects other
+	// components that haven't yet been converted to VerifyConfig/CommitConfig
+	// handling.
 	if !reflect.DeepEqual(from.Options, to.Options) {
 		l.Debugln(m, "requires restart, options differ")
 		return false

lib/model/rwfolder.go

@@ -87,7 +87,7 @@ type rwFolder struct {
 	ignorePerms    bool
 	copiers        int
 	pullers        int
-	shortID        uint64
+	shortID        protocol.ShortID
 	order          config.PullOrder
 	maxConflicts   int
 	sleep          time.Duration
@@ -108,7 +108,7 @@ type rwFolder struct {
 	errorsMut sync.Mutex
 }
 
-func newRWFolder(m *Model, shortID uint64, cfg config.FolderConfiguration) *rwFolder {
+func newRWFolder(m *Model, shortID protocol.ShortID, cfg config.FolderConfiguration) *rwFolder {
 	p := &rwFolder{
 		stateTracker: stateTracker{
 			folder: cfg.ID,

lib/osutil/rlimit_unix.go

@@ -13,7 +13,7 @@ import "syscall"
 // MaximizeOpenFileLimit tries to set the resoure limit RLIMIT_NOFILE (number
 // of open file descriptors) to the max (hard limit), if the current (soft
 // limit) is below the max. Returns the new (though possibly unchanged) limit,
-// or an error if it was could not be changed.
+// or an error if it could not be changed.
 func MaximizeOpenFileLimit() (int, error) {
 	// Get the current limit on number of open files.
 	var lim syscall.Rlimit

lib/protocol/deviceid.go

@@ -16,6 +16,7 @@ import (
 )
 
 type DeviceID [32]byte
+type ShortID uint64
 
 var LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
 
@@ -69,14 +70,20 @@ func (n DeviceID) Equals(other DeviceID) bool {
 }
 
 // Short returns an integer representing bits 0-63 of the device ID.
-func (n DeviceID) Short() uint64 {
+func (n DeviceID) Short() ShortID {
-	return binary.BigEndian.Uint64(n[:])
+	return ShortID(binary.BigEndian.Uint64(n[:]))
 }
 
 func (n *DeviceID) MarshalText() ([]byte, error) {
 	return []byte(n.String()), nil
 }
 
+func (s ShortID) String() string {
+	var bs [8]byte
+	binary.BigEndian.PutUint64(bs[:], uint64(s))
+	return base32.StdEncoding.EncodeToString(bs[:])[:7]
+}
+
 func (n *DeviceID) UnmarshalText(bs []byte) error {
 	id := string(bs)
 	id = strings.Trim(id, "=")

lib/protocol/deviceid_test.go

@@ -74,3 +74,25 @@ func TestMarshallingDeviceID(t *testing.T) {
 		t.Error("Compare error")
 	}
 }
+
+func TestShortIDString(t *testing.T) {
+	id, _ := DeviceIDFromString(formatted)
+
+	sid := id.Short().String()
+	if len(sid) != 7 {
+		t.Errorf("Wrong length for short ID: got %d, want 7", len(sid))
+	}
+
+	want := formatted[:len(sid)]
+	if sid != want {
+		t.Errorf("Wrong short ID: got %q, want %q", sid, want)
+	}
+}
+
+func TestDeviceIDFromBytes(t *testing.T) {
+	id0, _ := DeviceIDFromString(formatted)
+	id1 := DeviceIDFromBytes(id0[:])
+	if id1.String() != formatted {
+		t.Errorf("Wrong device ID, got %q, want %q", id1, formatted)
+	}
+}

lib/protocol/protocol_test.go

@@ -36,7 +36,7 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestHeaderFunctions(t *testing.T) {
+func TestHeaderEncodeDecode(t *testing.T) {
 	f := func(ver, id, typ int) bool {
 		ver = int(uint(ver) % 16)
 		id = int(uint(id) % 4096)
@@ -50,6 +50,26 @@ func TestHeaderFunctions(t *testing.T) {
 	}
 }
 
+func TestHeaderMarshalUnmarshal(t *testing.T) {
+	f := func(ver, id, typ int) bool {
+		ver = int(uint(ver) % 16)
+		id = int(uint(id) % 4096)
+		typ = int(uint(typ) % 256)
+		buf := new(bytes.Buffer)
+		xw := xdr.NewWriter(buf)
+		h0 := header{version: ver, msgID: id, msgType: typ}
+		h0.encodeXDR(xw)
+
+		xr := xdr.NewReader(buf)
+		var h1 header
+		h1.decodeXDR(xr)
+		return h0 == h1
+	}
+	if err := quick.Check(f, nil); err != nil {
+		t.Error(err)
+	}
+}
+
 func TestHeaderLayout(t *testing.T) {
 	var e, a uint32
 
@@ -321,3 +341,47 @@ func timeoutWriteHeader(w *xdr.Writer, hdr header) {
 	case <-time.After(250 * time.Millisecond):
 	}
 }
+
+func TestFileInfoSize(t *testing.T) {
+	fi := FileInfo{
+		Blocks: []BlockInfo{
+			{Size: 42},
+			{Offset: 42, Size: 23},
+			{Offset: 42 + 23, Size: 34},
+		},
+	}
+
+	size := fi.Size()
+	want := int64(42 + 23 + 34)
+	if size != want {
+		t.Errorf("Incorrect size reported, got %d, want %d", size, want)
+	}
+
+	size = fi.Size() // Cached, this time
+	if size != want {
+		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
+	}
+
+	fi.CachedSize = 8
+	want = 8
+	size = fi.Size() // Ensure it came from the cache
+	if size != want {
+		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
+	}
+
+	fi.CachedSize = 0
+	fi.Flags = FlagDirectory
+	want = 128
+	size = fi.Size() // Directories are 128 bytes large
+	if size != want {
+		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
+	}
+
+	fi.CachedSize = 0
+	fi.Flags = FlagDeleted
+	want = 128
+	size = fi.Size() // Also deleted files
+	if size != want {
+		t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
+	}
+}

lib/protocol/vector.go

@@ -10,31 +10,31 @@ type Vector []Counter
 
 // Counter represents a single counter in the version vector.
 type Counter struct {
-	ID    uint64
+	ID    ShortID
 	Value uint64
 }
 
 // Update returns a Vector with the index for the specific ID incremented by
 // one. If it is possible, the vector v is updated and returned. If it is not,
 // a copy will be created, updated and returned.
-func (v Vector) Update(ID uint64) Vector {
+func (v Vector) Update(id ShortID) Vector {
 	for i := range v {
-		if v[i].ID == ID {
+		if v[i].ID == id {
 			// Update an existing index
 			v[i].Value++
 			return v
-		} else if v[i].ID > ID {
+		} else if v[i].ID > id {
 			// Insert a new index
 			nv := make(Vector, len(v)+1)
 			copy(nv, v[:i])
-			nv[i].ID = ID
+			nv[i].ID = id
 			nv[i].Value = 1
 			copy(nv[i+1:], v[i:])
 			return nv
 		}
 	}
 	// Append a new index
-	return append(v, Counter{ID, 1})
+	return append(v, Counter{id, 1})
 }
 
 // Merge returns the vector containing the maximum indexes from v and b. If it
@@ -105,7 +105,7 @@ func (v Vector) Concurrent(b Vector) bool {
 }
 
 // Counter returns the current value of the given counter ID.
-func (v Vector) Counter(id uint64) uint64 {
+func (v Vector) Counter(id ShortID) uint64 {
 	for _, c := range v {
 		if c.ID == id {
 			return c.Value

lib/protocol/vector_xdr.go

@@ -21,7 +21,7 @@ type xdrReader interface {
 func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) {
 	w.WriteUint32(uint32(len(v)))
 	for i := range v {
-		w.WriteUint64(v[i].ID)
+		w.WriteUint64(uint64(v[i].ID))
 		w.WriteUint64(v[i].Value)
 	}
 	return 4 + 16*len(v), nil
@@ -35,7 +35,7 @@ func (v *Vector) DecodeXDRFrom(r xdrReader) error {
 	}
 	n := make(Vector, l)
 	for i := range n {
-		n[i].ID = r.ReadUint64()
+		n[i].ID = ShortID(r.ReadUint64())
 		n[i].Value = r.ReadUint64()
 	}
 	*v = n

lib/scanner/walk.go

@@ -69,7 +69,7 @@ type Walker struct {
 	// Number of routines to use for hashing
 	Hashers int
 	// Our vector clock id
-	ShortID uint64
+	ShortID protocol.ShortID
 	// Optional progress tick interval which defines how often FolderScanProgress
 	// events are emitted. Negative number means disabled.
 	ProgressTickIntervalS int

lib/upgrade/upgrade_supported.go

@@ -56,7 +56,7 @@ func FetchLatestReleases(releasesURL, version string) []Release {
 		return nil
 	}
 	if resp.StatusCode > 299 {
-		l.Infoln("API call returned HTTP error: %s", resp.Status)
+		l.Infoln("API call returned HTTP error:", resp.Status)
 		return nil
 	}
 

man/syncthing-bep.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-BEP" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-BEP" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-bep \- Block Exchange Protocol v1
 .

man/syncthing-config.5

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-CONFIG" "5" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-CONFIG" "5" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-config \- Syncthing Configuration
 .
@@ -348,6 +348,19 @@ square brackets.
 The word \fBdynamic\fP means to use local and global discovery to find the
 device.
 .UNINDENT
+.SH IGNOREDDEVICE ELEMENT
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+<ignoredDevice>5SYI2FS\-LW6YAXI\-JJDYETS\-NDBBPIO\-256MWBO\-XDPXWVG\-24QPUM4\-PDW4UQU</ignoredDevice>
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+This optional element lists device IDs that have been specifically ignored. One element must be present for each device ID. Connection attempts from these devices are logged to the console but never displayed in the web GUI.
 .SH GUI ELEMENT
 .INDENT 0.0
 .INDENT 3.5

man/syncthing-device-ids.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-DEVICE-IDS" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-DEVICE-IDS" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-device-ids \- Understanding Device IDs
 .

man/syncthing-event-api.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-EVENT-API" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-EVENT-API" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-event-api \- Event API
 .

man/syncthing-faq.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-FAQ" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-FAQ" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-faq \- Frequently Asked Questions
 .

man/syncthing-globaldisco.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-GLOBALDISCO" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-GLOBALDISCO" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-globaldisco \- Global Discovery Protocol v3
 .

man/syncthing-localdisco.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-LOCALDISCO" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-LOCALDISCO" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-localdisco \- Local Discovery Protocol v3
 .

man/syncthing-networking.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-NETWORKING" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-NETWORKING" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-networking \- Firewall Setup
 .

man/syncthing-relay.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-RELAY" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-RELAY" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-relay \- Relay Protocol v1
 .

man/syncthing-rest-api.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-REST-API" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-REST-API" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-rest-api \- REST API
 .

man/syncthing-security.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-SECURITY" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-SECURITY" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-security \- Security Principles
 .

man/syncthing-stignore.5

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING-STIGNORE" "5" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING-STIGNORE" "5" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing-stignore \- Prevent files from being synchronized to other nodes
 .

man/syncthing-versioning.7

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "TODO" "7" "January 17, 2016" "v0.12" "Syncthing"
+.TH "TODO" "7" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 Todo \- Keep automatic backups of deleted files by other nodes
 .

man/syncthing.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "SYNCTHING" "1" "January 17, 2016" "v0.12" "Syncthing"
+.TH "SYNCTHING" "1" "January 20, 2016" "v0.12" "Syncthing"
 .SH NAME
 syncthing \- Syncthing
 .