Translation update

Fix GUI breaking during restarts (fixes #577)

.gitignore

@@ -9,3 +9,4 @@ coverage.out
 files/pidx
 bin
 perfstats*.csv
+coverage.xml

.travis.yml

@@ -1,18 +0,0 @@
-language: go
-
-go:
-   - tip
-
-install:
-   - export PATH=$PATH:$HOME/gopath/bin
-   - ./build.sh setup
-
-script:
-   - ./build.sh test-cov
-
-after_success:
-   - goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/syncthing -repotoken="$COVERALS_TOKEN"
-
-env:
-   global:
-      secure: "zEV2h2XtKHNLVdXJjM4LA/VjMfLVydm6goF+ARit+nOSGxGoH7f7jIdzJzhxgh7shKG93q61eLO1Tug+WBMYB2EpBuYnTB5AIMYhCDwNI8C4uBV6c3brHfcrie7MASNao8TID2QScASKNFFWvjv/i1Ccn5ztxdcQuhSsNjGZp8A="

CONTRIBUTING.md

@@ -1,7 +1,43 @@
+## Reporting Bugs
+
+Please file bugs in the [Github Issue
+Tracker](https://github.com/syncthing/syncthing/issues). Include at
+least the following:
+
+ - What happened
+
+ - What did you expect to happen instead of what *did* happen, if it's
+   not crazy obvious
+
+ - What operating system, operating system version and version of
+   Syncthing you are running
+
+ - The same for other connected nodes, where relevant
+
+ - Screenshot if the issue concerns something visible in the GUI
+
+ - Console log entries, where possible and relevant
+
+If you're not sure whether something is relevant, erring on the side of
+too much information will never get you yelled at. :)
+
+## Contributing Translations
+
+All translations are done via
+[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
+wish to contribute to a translation, just head over there and sign up.
+Before every release, the language resources are updated from the
+latest info on Transifex.
+
+## Contributing Code
+
 Please do contribute! If you want to contribute but are unsure where to
 start, the [Contributions Needed
-topic](http://discourse.syncthing.net/t/contributions-needed/49)
-lists areas in need of attention.
+topic](http://discourse.syncthing.net/t/49) lists areas in need of
+attention. In general, any open issues are fair game! Be prepared for a
+[certain amount of
+review](https://discourse.syncthing.net/t/733); it's all in the name of
+quality. :)
 
 ## Licensing
 
@@ -16,8 +52,8 @@ to add yourself as a separate commit in your first pull request.
 
 ## Building
 
-[See the
-documentation](http://discourse.syncthing.net/t/building-syncthing/44)
+[See the documentation](http://discourse.syncthing.net/t/44) on how to
+get started with a build environment.
 
 ## Branches
 
@@ -44,7 +80,9 @@ Yes please!
 
 ## Style
 
-`go fmt`
+ - `go fmt`
+
+ - Unix line breaks
 
 ## Documentation
 
@@ -53,4 +91,3 @@ Yes please!
 ## License
 
 MIT
-

CONTRIBUTORS

@@ -1,11 +1,15 @@
 Aaron Bieber <qbit@deftly.net>
+Alexander Graf <register-github@alex-graf.de>
 Andrew Dunham <andrew@du.nham.ca>
 Audrius Butkevicius <audrius.butkevicius@gmail.com>
-Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com>
+Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Ben Sidhom <bsidhom@gmail.com>
 Brandon Philips <brandon@ifup.org>
-James Patterson <jamespatterson@operamail.com>
-Jens Diemer <github.com@jensdiemer.de>
+Gilli Sigurdsson <gilli@vx.is>
+James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
+Marcin Dziadus <dziadus.marcin@gmail.com>
 Philippe Schommers <philippe@schommers.be>
 Ryan Sullivan <kayoticsully@gmail.com>
+Tully Robinson <tully@tojr.org>
 Veeti Paananen <veeti.paananen@rojekti.fi>

Godeps/Godeps.json

@@ -1,6 +1,6 @@
 {
-	"ImportPath": "github.com/calmh/syncthing",
-	"GoVersion": "go1.3",
+	"ImportPath": "github.com/syncthing/syncthing",
+	"GoVersion": "go1.3.1",
 	"Packages": [
 		"./cmd/..."
 	],
@@ -12,23 +12,23 @@
 		},
 		{
 			"ImportPath": "code.google.com/p/go.crypto/bcrypt",
-			"Comment": "null-213",
-			"Rev": "aa2644fe4aa50e3b38d75187b4799b1f0c9ddcef"
+			"Comment": "null-216",
+			"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
 		},
 		{
 			"ImportPath": "code.google.com/p/go.crypto/blowfish",
-			"Comment": "null-213",
-			"Rev": "aa2644fe4aa50e3b38d75187b4799b1f0c9ddcef"
+			"Comment": "null-216",
+			"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
 		},
 		{
 			"ImportPath": "code.google.com/p/go.text/transform",
-			"Comment": "null-88",
-			"Rev": "1506dcc33592c369c3be7bd30b38f90445b86deb"
+			"Comment": "null-90",
+			"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
 		},
 		{
 			"ImportPath": "code.google.com/p/go.text/unicode/norm",
-			"Comment": "null-88",
-			"Rev": "1506dcc33592c369c3be7bd30b38f90445b86deb"
+			"Comment": "null-90",
+			"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
 		},
 		{
 			"ImportPath": "code.google.com/p/snappy-go/snappy",
@@ -41,7 +41,7 @@
 		},
 		{
 			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "89d756f35ba26bcdd47ca25cf9795b3b8d1e1852"
+			"Rev": "a597b63b87d6140f79084c8aab214b4d533833a1"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
@@ -49,7 +49,7 @@
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "c5955912e3287376475731c5bc59c79a5a799105"
+			"Rev": "59d87758aeaab5ab6ed289c773349500228a1557"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",

Godeps/_workspace/src/code.google.com/p/go.crypto/blowfish/block.go

@@ -4,6 +4,22 @@
 
 package blowfish
 
+// getNextWord returns the next big-endian uint32 value from the byte slice
+// at the given position in a circular manner, updating the position.
+func getNextWord(b []byte, pos *int) uint32 {
+	var w uint32
+	j := *pos
+	for i := 0; i < 4; i++ {
+		w = w<<8 | uint32(b[j])
+		j++
+		if j >= len(b) {
+			j = 0
+		}
+	}
+	*pos = j
+	return w
+}
+
 // ExpandKey performs a key expansion on the given *Cipher. Specifically, it
 // performs the Blowfish algorithm's key schedule which sets up the *Cipher's
 // pi and substitution tables for calls to Encrypt. This is used, primarily,
@@ -12,6 +28,7 @@ package blowfish
 func ExpandKey(key []byte, c *Cipher) {
 	j := 0
 	for i := 0; i < 18; i++ {
+		// Using inlined getNextWord for performance.
 		var d uint32
 		for k := 0; k < 4; k++ {
 			d = d<<8 | uint32(key[j])
@@ -54,86 +71,44 @@ func ExpandKey(key []byte, c *Cipher) {
 func expandKeyWithSalt(key []byte, salt []byte, c *Cipher) {
 	j := 0
 	for i := 0; i < 18; i++ {
-		var d uint32
-		for k := 0; k < 4; k++ {
-			d = d<<8 | uint32(key[j])
-			j++
-			if j >= len(key) {
-				j = 0
-			}
-		}
-		c.p[i] ^= d
+		c.p[i] ^= getNextWord(key, &j)
 	}
 
 	j = 0
-	var expandedSalt [4]uint32
-	for i := range expandedSalt {
-		var d uint32
-		for k := 0; k < 4; k++ {
-			d = d<<8 | uint32(salt[j])
-			j++
-			if j >= len(salt) {
-				j = 0
-			}
-		}
-		expandedSalt[i] = d
-	}
-
 	var l, r uint32
 	for i := 0; i < 18; i += 2 {
-		l ^= expandedSalt[i&2]
-		r ^= expandedSalt[(i&2)+1]
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.p[i], c.p[i+1] = l, r
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s0[i], c.s0[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s0[i+2], c.s0[i+3] = l, r
-
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s1[i], c.s1[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s1[i+2], c.s1[i+3] = l, r
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s2[i], c.s2[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s2[i+2], c.s2[i+3] = l, r
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s3[i], c.s3[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s3[i+2], c.s3[i+3] = l, r
 	}
 }
 
@@ -182,9 +157,3 @@ func decryptBlock(l, r uint32, c *Cipher) (uint32, uint32) {
 	xr ^= c.p[0]
 	return xr, xl
 }
-
-func zero(x []uint32) {
-	for i := range x {
-		x[i] = 0
-	}
-}

Godeps/_workspace/src/code.google.com/p/go.crypto/blowfish/blowfish_test.go

@@ -4,9 +4,7 @@
 
 package blowfish
 
-import (
-	"testing"
-)
+import "testing"
 
 type CryptTest struct {
 	key []byte
@@ -202,3 +200,75 @@ func TestSaltedCipherKeyLength(t *testing.T) {
 		t.Errorf("NewSaltedCipher with long key, gave error %#v", err)
 	}
 }
+
+// Test vectors generated with Blowfish from OpenSSH.
+var saltedVectors = [][8]byte{
+	{0x0c, 0x82, 0x3b, 0x7b, 0x8d, 0x01, 0x4b, 0x7e},
+	{0xd1, 0xe1, 0x93, 0xf0, 0x70, 0xa6, 0xdb, 0x12},
+	{0xfc, 0x5e, 0xba, 0xde, 0xcb, 0xf8, 0x59, 0xad},
+	{0x8a, 0x0c, 0x76, 0xe7, 0xdd, 0x2c, 0xd3, 0xa8},
+	{0x2c, 0xcb, 0x7b, 0xee, 0xac, 0x7b, 0x7f, 0xf8},
+	{0xbb, 0xf6, 0x30, 0x6f, 0xe1, 0x5d, 0x62, 0xbf},
+	{0x97, 0x1e, 0xc1, 0x3d, 0x3d, 0xe0, 0x11, 0xe9},
+	{0x06, 0xd7, 0x4d, 0xb1, 0x80, 0xa3, 0xb1, 0x38},
+	{0x67, 0xa1, 0xa9, 0x75, 0x0e, 0x5b, 0xc6, 0xb4},
+	{0x51, 0x0f, 0x33, 0x0e, 0x4f, 0x67, 0xd2, 0x0c},
+	{0xf1, 0x73, 0x7e, 0xd8, 0x44, 0xea, 0xdb, 0xe5},
+	{0x14, 0x0e, 0x16, 0xce, 0x7f, 0x4a, 0x9c, 0x7b},
+	{0x4b, 0xfe, 0x43, 0xfd, 0xbf, 0x36, 0x04, 0x47},
+	{0xb1, 0xeb, 0x3e, 0x15, 0x36, 0xa7, 0xbb, 0xe2},
+	{0x6d, 0x0b, 0x41, 0xdd, 0x00, 0x98, 0x0b, 0x19},
+	{0xd3, 0xce, 0x45, 0xce, 0x1d, 0x56, 0xb7, 0xfc},
+	{0xd9, 0xf0, 0xfd, 0xda, 0xc0, 0x23, 0xb7, 0x93},
+	{0x4c, 0x6f, 0xa1, 0xe4, 0x0c, 0xa8, 0xca, 0x57},
+	{0xe6, 0x2f, 0x28, 0xa7, 0x0c, 0x94, 0x0d, 0x08},
+	{0x8f, 0xe3, 0xf0, 0xb6, 0x29, 0xe3, 0x44, 0x03},
+	{0xff, 0x98, 0xdd, 0x04, 0x45, 0xb4, 0x6d, 0x1f},
+	{0x9e, 0x45, 0x4d, 0x18, 0x40, 0x53, 0xdb, 0xef},
+	{0xb7, 0x3b, 0xef, 0x29, 0xbe, 0xa8, 0x13, 0x71},
+	{0x02, 0x54, 0x55, 0x41, 0x8e, 0x04, 0xfc, 0xad},
+	{0x6a, 0x0a, 0xee, 0x7c, 0x10, 0xd9, 0x19, 0xfe},
+	{0x0a, 0x22, 0xd9, 0x41, 0xcc, 0x23, 0x87, 0x13},
+	{0x6e, 0xff, 0x1f, 0xff, 0x36, 0x17, 0x9c, 0xbe},
+	{0x79, 0xad, 0xb7, 0x40, 0xf4, 0x9f, 0x51, 0xa6},
+	{0x97, 0x81, 0x99, 0xa4, 0xde, 0x9e, 0x9f, 0xb6},
+	{0x12, 0x19, 0x7a, 0x28, 0xd0, 0xdc, 0xcc, 0x92},
+	{0x81, 0xda, 0x60, 0x1e, 0x0e, 0xdd, 0x65, 0x56},
+	{0x7d, 0x76, 0x20, 0xb2, 0x73, 0xc9, 0x9e, 0xee},
+}
+
+func TestSaltedCipher(t *testing.T) {
+	var key, salt [32]byte
+	for i := range key {
+		key[i] = byte(i)
+		salt[i] = byte(i + 32)
+	}
+	for i, v := range saltedVectors {
+		c, err := NewSaltedCipher(key[:], salt[:i])
+		if err != nil {
+			t.Fatal(err)
+		}
+		var buf [8]byte
+		c.Encrypt(buf[:], buf[:])
+		if v != buf {
+			t.Errorf("%d: expected %x, got %x", i, v, buf)
+		}
+	}
+}
+
+func BenchmarkExpandKeyWithSalt(b *testing.B) {
+	key := make([]byte, 32)
+	salt := make([]byte, 16)
+	c, _ := NewCipher(key)
+	for i := 0; i < b.N; i++ {
+		expandKeyWithSalt(key, salt, c)
+	}
+}
+
+func BenchmarkExpandKey(b *testing.B) {
+	key := make([]byte, 32)
+	c, _ := NewCipher(key)
+	for i := 0; i < b.N; i++ {
+		ExpandKey(key, c)
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.crypto/blowfish/cipher.go

@@ -40,8 +40,11 @@ func NewCipher(key []byte) (*Cipher, error) {
 // NewSaltedCipher creates a returns a Cipher that folds a salt into its key
 // schedule. For most purposes, NewCipher, instead of NewSaltedCipher, is
 // sufficient and desirable. For bcrypt compatiblity, the key can be over 56
-// bytes. Only the first 16 bytes of salt are used.
+// bytes.
 func NewSaltedCipher(key, salt []byte) (*Cipher, error) {
+	if len(salt) == 0 {
+		return NewCipher(key)
+	}
 	var result Cipher
 	if k := len(key); k < 1 {
 		return nil, KeySizeError(k)

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketables.go

@@ -11,7 +11,6 @@
 package main
 
 import (
-	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
@@ -24,6 +23,8 @@ import (
 	"strconv"
 	"strings"
 	"unicode"
+
+	"code.google.com/p/go.text/internal/ucd"
 )
 
 func main() {
@@ -63,31 +64,7 @@ var localFiles = flag.Bool("local",
 
 var logger = log.New(os.Stderr, "", log.Lshortfile)
 
-// UnicodeData.txt has form:
-//	0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
-//	007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
-// See http://unicode.org/reports/tr44/ for full explanation
-// The fields:
-const (
-	FCodePoint = iota
-	FName
-	FGeneralCategory
-	FCanonicalCombiningClass
-	FBidiClass
-	FDecompMapping
-	FDecimalValue
-	FDigitValue
-	FNumericValue
-	FBidiMirrored
-	FUnicode1Name
-	FISOComment
-	FSimpleUppercaseMapping
-	FSimpleLowercaseMapping
-	FSimpleTitlecaseMapping
-	NumField
-
-	MaxChar = 0x10FFFF // anything above this shouldn't exist
-)
+const MaxChar = 0x10FFFF // anything above this shouldn't exist
 
 // Quick Check properties of runes allow us to quickly
 // determine whether a rune may occur in a normal form.
@@ -232,7 +209,7 @@ func openReader(file string) (input io.ReadCloser) {
 	return
 }
 
-func parseDecomposition(s string, skipfirst bool) (a []rune, e error) {
+func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 	decomp := strings.Split(s, " ")
 	if len(decomp) > 0 && skipfirst {
 		decomp = decomp[1:]
@@ -247,56 +224,31 @@ func parseDecomposition(s string, skipfirst bool) (a []rune, e error) {
 	return a, nil
 }
 
-func parseCharacter(line string) {
-	field := strings.Split(line, ";")
-	if len(field) != NumField {
-		logger.Fatalf("%5s: %d fields (expected %d)\n", line, len(field), NumField)
-	}
-	x, err := strconv.ParseUint(field[FCodePoint], 16, 64)
-	point := int(x)
-	if err != nil {
-		logger.Fatalf("%.5s...: %s", line, err)
-	}
-	if point == 0 {
-		return // not interesting and we use 0 as unset
-	}
-	if point > MaxChar {
-		logger.Fatalf("%5s: Rune %X > MaxChar (%X)", line, point, MaxChar)
-		return
-	}
-	state := SNormal
-	switch {
-	case strings.Index(field[FName], ", First>") > 0:
-		state = SFirst
-	case strings.Index(field[FName], ", Last>") > 0:
-		state = SLast
-	}
-	firstChar := lastChar + 1
-	lastChar = rune(point)
-	if state != SLast {
-		firstChar = lastChar
-	}
-	x, err = strconv.ParseUint(field[FCanonicalCombiningClass], 10, 64)
-	if err != nil {
-		logger.Fatalf("%U: bad ccc field: %s", int(x), err)
-	}
-	ccc := uint8(x)
-	decmap := field[FDecompMapping]
-	exp, e := parseDecomposition(decmap, false)
-	isCompat := false
-	if e != nil {
-		if len(decmap) > 0 {
-			exp, e = parseDecomposition(decmap, true)
-			if e != nil {
-				logger.Fatalf(`%U: bad decomp |%v|: "%s"`, int(x), decmap, e)
+func loadUnicodeData() {
+	f := openReader("UnicodeData.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(ucd.CodePoint)
+		char := &chars[r]
+
+		char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass))
+		decmap := p.String(ucd.DecompMapping)
+
+		exp, err := parseDecomposition(decmap, false)
+		isCompat := false
+		if err != nil {
+			if len(decmap) > 0 {
+				exp, err = parseDecomposition(decmap, true)
+				if err != nil {
+					logger.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
+				}
+				isCompat = true
 			}
-			isCompat = true
 		}
-	}
-	for i := firstChar; i <= lastChar; i++ {
-		char := &chars[i]
-		char.name = field[FName]
-		char.codePoint = i
+
+		char.name = p.String(ucd.Name)
+		char.codePoint = r
 		char.forms[FCompatibility].decomp = exp
 		if !isCompat {
 			char.forms[FCanonical].decomp = exp
@@ -306,24 +258,9 @@ func parseCharacter(line string) {
 		if len(decmap) > 0 {
 			char.forms[FCompatibility].decomp = exp
 		}
-		char.ccc = ccc
-		char.state = SMissing
-		if i == lastChar {
-			char.state = state
-		}
 	}
-	return
-}
-
-func loadUnicodeData() {
-	f := openReader("UnicodeData.txt")
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		parseCharacter(scanner.Text())
-	}
-	if scanner.Err() != nil {
-		logger.Fatal(scanner.Err())
+	if err := p.Err(); err != nil {
+		logger.Fatal(err)
 	}
 }
 
@@ -354,47 +291,22 @@ func compactCCC() {
 	}
 }
 
-var singlePointRe = regexp.MustCompile(`^([0-9A-F]+) *$`)
-
 // CompositionExclusions.txt has form:
 // 0958    # ...
 // See http://unicode.org/reports/tr44/ for full explanation
-func parseExclusion(line string) int {
-	comment := strings.Index(line, "#")
-	if comment >= 0 {
-		line = line[0:comment]
-	}
-	if len(line) == 0 {
-		return 0
-	}
-	matches := singlePointRe.FindStringSubmatch(line)
-	if len(matches) != 2 {
-		logger.Fatalf("%s: %d matches (expected 1)\n", line, len(matches))
-	}
-	point, err := strconv.ParseUint(matches[1], 16, 64)
-	if err != nil {
-		logger.Fatalf("%.5s...: %s", line, err)
-	}
-	return int(point)
-}
-
 func loadCompositionExclusions() {
 	f := openReader("CompositionExclusions.txt")
 	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		point := parseExclusion(scanner.Text())
-		if point == 0 {
-			continue
-		}
-		c := &chars[point]
+	p := ucd.New(f)
+	for p.Next() {
+		c := &chars[p.Rune(0)]
 		if c.excludeInComp {
 			logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
 		}
 		c.excludeInComp = true
 	}
-	if scanner.Err() != nil {
-		log.Fatal(scanner.Err())
+	if e := p.Err(); e != nil {
+		logger.Fatal(e)
 	}
 }
 
@@ -988,8 +900,6 @@ func verifyComputed() {
 	}
 }
 
-var qcRe = regexp.MustCompile(`([0-9A-F\.]+) *; (NF.*_QC); ([YNM]) #.*`)
-
 // Use values in DerivedNormalizationProps.txt to compare against the
 // values we computed.
 // DerivedNormalizationProps.txt has form:
@@ -999,27 +909,13 @@ var qcRe = regexp.MustCompile(`([0-9A-F\.]+) *; (NF.*_QC); ([YNM]) #.*`)
 func testDerived() {
 	f := openReader("DerivedNormalizationProps.txt")
 	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := scanner.Text()
-		qc := qcRe.FindStringSubmatch(line)
-		if qc == nil {
-			continue
-		}
-		rng := strings.Split(qc[1], "..")
-		i, err := strconv.ParseUint(rng[0], 16, 64)
-		if err != nil {
-			log.Fatal(err)
-		}
-		j := i
-		if len(rng) > 1 {
-			j, err = strconv.ParseUint(rng[1], 16, 64)
-			if err != nil {
-				log.Fatal(err)
-			}
-		}
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(0)
+		c := &chars[r]
+
 		var ftype, mode int
-		qt := strings.TrimSpace(qc[2])
+		qt := p.String(1)
 		switch qt {
 		case "NFC_QC":
 			ftype, mode = FCanonical, MComposed
@@ -1030,10 +926,10 @@ func testDerived() {
 		case "NFKD_QC":
 			ftype, mode = FCompatibility, MDecomposed
 		default:
-			log.Fatalf(`Unexpected quick check type "%s"`, qt)
+			continue
 		}
 		var qr QCResult
-		switch qc[3] {
+		switch p.String(2) {
 		case "Y":
 			qr = QCYes
 		case "N":
@@ -1041,27 +937,15 @@ func testDerived() {
 		case "M":
 			qr = QCMaybe
 		default:
-			log.Fatalf(`Unexpected quick check value "%s"`, qc[3])
+			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
 		}
-		var lastFailed bool
-		// Verify current
-		for ; i <= j; i++ {
-			c := &chars[int(i)]
-			c.forms[ftype].verified[mode] = true
-			curqr := c.forms[ftype].quickCheck[mode]
-			if curqr != qr {
-				if !lastFailed {
-					logger.Printf("%s: %.4X..%.4X -- %s\n",
-						qt, int(i), int(j), line[0:50])
-				}
-				logger.Printf("%U: FAILED %s (was %v need %v)\n",
-					int(i), qt, curqr, qr)
-				lastFailed = true
-			}
+		if got := c.forms[ftype].quickCheck[mode]; got != qr {
+			logger.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
 		}
+		c.forms[ftype].verified[mode] = true
 	}
-	if scanner.Err() != nil {
-		logger.Fatal(scanner.Err())
+	if err := p.Err(); err != nil {
+		logger.Fatal(err)
 	}
 	// Any unspecified value must be QCYes. Verify this.
 	for i, c := range chars {

Godeps/_workspace/src/github.com/calmh/xdr/.gitignore

@@ -0,0 +1 @@
+coverage.out

Godeps/_workspace/src/github.com/calmh/xdr/.travis.yml

@@ -0,0 +1,19 @@
+language: go
+go:
+- tip
+
+install:
+- export PATH=$PATH:$HOME/gopath/bin
+- go get code.google.com/p/go.tools/cmd/cover
+- go get github.com/mattn/goveralls
+
+script:
+- ./generate.sh
+- go test -coverprofile=coverage.out
+
+after_success:
+- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
+
+env:
+  global:
+    secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=

Godeps/_workspace/src/github.com/calmh/xdr/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2014 Jakob Borg.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

Godeps/_workspace/src/github.com/calmh/xdr/README.md

@@ -0,0 +1,12 @@
+xdr
+===
+
+[![Build Status](https://img.shields.io/travis/calmh/xdr.svg?style=flat)](https://travis-ci.org/calmh/xdr)
+[![Coverage Status](https://img.shields.io/coveralls/calmh/xdr.svg?style=flat)](https://coveralls.io/r/calmh/xdr?branch=master)
+[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)
+
+This is an XDR encoding/decoding library. It uses code generation and
+not reflection. It supports the IPDR bastardized XDR format when built
+with `-tags ipdr`.
+

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"io/ioutil"
 	"testing"
+
+	"github.com/calmh/xdr"
 )
 
 type XDRBenchStruct struct {
@@ -58,6 +60,16 @@ func BenchmarkThisEncode(b *testing.B) {
 	}
 }
 
+func BenchmarkThisEncoder(b *testing.B) {
+	w := xdr.NewWriter(ioutil.Discard)
+	for i := 0; i < b.N; i++ {
+		_, err := s.encodeXDR(w)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 type repeatReader struct {
 	data []byte
 }
@@ -86,3 +98,16 @@ func BenchmarkThisDecode(b *testing.B) {
 		rr.Reset(e)
 	}
 }
+
+func BenchmarkThisDecoder(b *testing.B) {
+	rr := &repeatReader{e}
+	r := xdr.NewReader(rr)
+	var t XDRBenchStruct
+	for i := 0; i < b.N; i++ {
+		err := t.decodeXDR(r)
+		if err != nil {
+			b.Fatal(err)
+		}
+		rr.Reset(e)
+	}
+}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -5,8 +5,12 @@ package xdr_test
 
 import (
 	"bytes"
+	"math/rand"
+	"reflect"
 	"testing"
 	"testing/quick"
+
+	"github.com/calmh/xdr"
 )
 
 // Contains all supported types
@@ -22,6 +26,25 @@ type TestStruct struct {
 	UI64 uint64
 	BS   []byte
 	S    string
+	C    Opaque
+}
+
+type Opaque [32]byte
+
+func (u *Opaque) encodeXDR(w *xdr.Writer) (int, error) {
+	return w.WriteRaw(u[:])
+}
+
+func (u *Opaque) decodeXDR(r *xdr.Reader) (int, error) {
+	return r.ReadRaw(u[:])
+}
+
+func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
+	var u Opaque
+	for i := range u[:] {
+		u[i] = byte(rand.Int())
+	}
+	return reflect.ValueOf(u)
 }
 
 func TestEncDec(t *testing.T) {
@@ -39,7 +62,7 @@ func TestEncDec(t *testing.T) {
 			t0.I32 != t1.I32 || t0.UI32 != t1.UI32 ||
 			t0.I64 != t1.I64 || t0.UI64 != t1.UI64 ||
 			bytes.Compare(t0.BS, t1.BS) != 0 ||
-			t0.S != t1.S {
+			t0.S != t1.S || t0.C != t1.C {
 			t.Logf("%#v", t0)
 			t.Logf("%#v", t1)
 			return false

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -52,6 +52,8 @@ TestStruct Structure:
 \                      S (variable length)                      \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                            Opaque                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct TestStruct {
@@ -66,6 +68,7 @@ struct TestStruct {
 	unsigned hyper UI64;
 	opaque BS<>;
 	string S<>;
+	Opaque C;
 }
 
 */
@@ -98,6 +101,10 @@ func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint64(o.UI64)
 	xw.WriteBytes(o.BS)
 	xw.WriteString(o.S)
+	_, err := o.C.encodeXDR(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
 	return xw.Tot(), xw.Error()
 }
 
@@ -124,5 +131,6 @@ func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
 	o.UI64 = xr.ReadUint64()
 	o.BS = xr.ReadBytes()
 	o.S = xr.ReadString()
+	(&o.C).decodeXDR(xr)
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -7,6 +7,8 @@ package xdr
 import (
 	"errors"
 	"io"
+	"reflect"
+	"unsafe"
 )
 
 var ErrElementSizeExceeded = errors.New("element size exceeded")
@@ -15,7 +17,6 @@ type Reader struct {
 	r   io.Reader
 	err error
 	b   [8]byte
-	sb  []byte
 }
 
 func NewReader(r io.Reader) *Reader {
@@ -35,23 +36,17 @@ func (r *Reader) ReadRaw(bs []byte) (int, error) {
 }
 
 func (r *Reader) ReadString() string {
-	if r.sb == nil {
-		r.sb = make([]byte, 64)
-	} else {
-		r.sb = r.sb[:cap(r.sb)]
-	}
-	r.sb = r.ReadBytesInto(r.sb)
-	return string(r.sb)
+	return r.ReadStringMax(0)
 }
 
 func (r *Reader) ReadStringMax(max int) string {
-	if r.sb == nil {
-		r.sb = make([]byte, 64)
-	} else {
-		r.sb = r.sb[:cap(r.sb)]
+	buf := r.ReadBytesMaxInto(max, nil)
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
+	sh := reflect.StringHeader{
+		Data: bh.Data,
+		Len:  bh.Len,
 	}
-	r.sb = r.ReadBytesMaxInto(max, r.sb)
-	return string(r.sb)
+	return *((*string)(unsafe.Pointer(&sh)))
 }
 
 func (r *Reader) ReadBytes() []byte {
@@ -80,10 +75,10 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 		return nil
 	}
 
-	if l+pad(l) > len(dst) {
-		dst = make([]byte, l+pad(l))
+	if fullLen := l + pad(l); fullLen > len(dst) {
+		dst = make([]byte, fullLen)
 	} else {
-		dst = dst[:l+pad(l)]
+		dst = dst[:fullLen]
 	}
 
 	var n int

Godeps/_workspace/src/github.com/calmh/xdr/reader_ipdr.go

@@ -22,7 +22,7 @@ func (r *Reader) ReadUint8() uint8 {
 	}
 
 	if debug {
-		dl.Printf("rd uint8=%d (0x%08x)", r.b[0], r.b[0])
+		dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
 	}
 	return r.b[0]
 }
@@ -43,7 +43,7 @@ func (r *Reader) ReadUint16() uint16 {
 	v := uint16(r.b[1]) | uint16(r.b[0])<<8
 
 	if debug {
-		dl.Printf("rd uint16=%d (0x%08x)", v, v)
+		dl.Printf("rd uint16=%d (0x%04x)", v, v)
 	}
 	return v
 }

Godeps/_workspace/src/github.com/calmh/xdr/writer.go

@@ -3,7 +3,11 @@
 
 package xdr
 
-import "io"
+import (
+	"io"
+	"reflect"
+	"unsafe"
+)
 
 var padBytes = []byte{0, 0, 0}
 
@@ -38,7 +42,13 @@ func (w *Writer) WriteRaw(bs []byte) (int, error) {
 }
 
 func (w *Writer) WriteString(s string) (int, error) {
-	return w.WriteBytes([]byte(s))
+	sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
+	bh := reflect.SliceHeader{
+		Data: sh.Data,
+		Len:  sh.Len,
+		Cap:  sh.Len,
+	}
+	return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
 }
 
 func (w *Writer) WriteBytes(bs []byte) (int, error) {

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -67,7 +67,7 @@ func TestReadBytesMaxInto(t *testing.T) {
 	}
 }
 
-func TestReadBytesMaxIntoNil(t *testing.T) {
+func TestReadStringMax(t *testing.T) {
 	for tot := 42; tot < 72; tot++ {
 		for max := 0; max < 128; max++ {
 			var b = new(bytes.Buffer)
@@ -77,8 +77,8 @@ func TestReadBytesMaxIntoNil(t *testing.T) {
 			var toWrite = make([]byte, tot)
 			w.WriteBytes(toWrite)
 
-			var bs = r.ReadBytesMaxInto(max, nil)
-			var read = len(bs)
+			var str = r.ReadStringMax(max)
+			var read = len(str)
 
 			if max == 0 || tot <= max {
 				if read != tot {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go

@@ -170,7 +170,7 @@ func (p *dbBench) writes(perBatch int) {
 	b.SetBytes(116)
 }
 
-func (p *dbBench) drop() {
+func (p *dbBench) gc() {
 	p.keys, p.values = nil, nil
 	runtime.GC()
 }
@@ -249,6 +249,9 @@ func (p *dbBench) newIter() iterator.Iterator {
 }
 
 func (p *dbBench) close() {
+	if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
+		p.b.Log("Block pool stats: ", bp)
+	}
 	p.db.Close()
 	p.stor.Close()
 	os.RemoveAll(benchDB)
@@ -331,7 +334,7 @@ func BenchmarkDBRead(b *testing.B) {
 	p := openDBBench(b, false)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -362,7 +365,7 @@ func BenchmarkDBReadUncompressed(b *testing.B) {
 	p := openDBBench(b, true)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -379,7 +382,7 @@ func BenchmarkDBReadTable(b *testing.B) {
 	p.populate(b.N)
 	p.fill()
 	p.reopen()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -395,7 +398,7 @@ func BenchmarkDBReadReverse(b *testing.B) {
 	p := openDBBench(b, false)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -413,7 +416,7 @@ func BenchmarkDBReadReverseTable(b *testing.B) {
 	p.populate(b.N)
 	p.fill()
 	p.reopen()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -11,84 +11,106 @@ import (
 	"sync/atomic"
 )
 
-// SetFunc used by Namespace.Get method to create a cache object. SetFunc
-// may return ok false, in that case the cache object will not be created.
-type SetFunc func() (ok bool, value interface{}, charge int, fin SetFin)
+// SetFunc is the function that will be called by Namespace.Get to create
+// a cache object, if charge is less than one than the cache object will
+// not be registered to cache tree, if value is nil then the cache object
+// will not be created.
+type SetFunc func() (charge int, value interface{})
 
-// SetFin will be called when corresponding cache object are released.
-type SetFin func()
+// DelFin is the function that will be called as the result of a delete operation.
+// Exist == true is indication that the object is exist, and pending == true is
+// indication of deletion already happen but haven't done yet (wait for all handles
+// to be released). And exist == false means the object doesn't exist.
+type DelFin func(exist, pending bool)
 
-// DelFin will be called when corresponding cache object are released.
-// DelFin will be called after SetFin. The exist is true if the corresponding
-// cache object is actually exist in the cache tree.
-type DelFin func(exist bool)
-
-// PurgeFin will be called when corresponding cache object are released.
-// PurgeFin will be called after SetFin. If PurgeFin present DelFin will
-// not be executed but passed to the PurgeFin, it is up to the caller
-// to call it or not.
-type PurgeFin func(ns, key uint64, delfin DelFin)
+// PurgeFin is the function that will be called as the result of a purge operation.
+type PurgeFin func(ns, key uint64)
 
 // Cache is a cache tree. A cache instance must be goroutine-safe.
 type Cache interface {
-	// SetCapacity sets cache capacity.
+	// SetCapacity sets cache tree capacity.
 	SetCapacity(capacity int)
 
-	// GetNamespace gets or creates a cache namespace for the given id.
+	// Capacity returns cache tree capacity.
+	Capacity() int
+
+	// Used returns used cache tree capacity.
+	Used() int
+
+	// Size returns entire alive cache objects size.
+	Size() int
+
+	// GetNamespace gets cache namespace with the given id.
+	// GetNamespace is never return nil.
 	GetNamespace(id uint64) Namespace
 
-	// Purge purges all cache namespaces, read Namespace.Purge method documentation.
+	// Purge purges all cache namespace from this cache tree.
+	// This is behave the same as calling Namespace.Purge method on all cache namespace.
 	Purge(fin PurgeFin)
 
-	// Zap zaps all cache namespaces, read Namespace.Zap method documentation.
-	Zap(closed bool)
+	// Zap detaches all cache namespace from this cache tree.
+	// This is behave the same as calling Namespace.Zap method on all cache namespace.
+	Zap()
 }
 
 // Namespace is a cache namespace. A namespace instance must be goroutine-safe.
 type Namespace interface {
-	// Get gets cache object for the given key. The given SetFunc (if not nil) will
-	// be called if the given key does not exist.
-	// If the given key does not exist, SetFunc is nil or SetFunc return ok false, Get
-	// will return ok false.
-	Get(key uint64, setf SetFunc) (obj Object, ok bool)
-
-	// Get deletes cache object for the given key. If exist the cache object will
-	// be deleted later when all of its handles have been released (i.e. no one use
-	// it anymore) and the given DelFin (if not nil) will finally be  executed. If
-	// such cache object does not exist the given DelFin will be executed anyway.
+	// Get gets cache object with the given key.
+	// If cache object is not found and setf is not nil, Get will atomically creates
+	// the cache object by calling setf. Otherwise Get will returns nil.
 	//
-	// Delete returns true if such cache object exist.
+	// The returned cache handle should be released after use by calling Release
+	// method.
+	Get(key uint64, setf SetFunc) Handle
+
+	// Delete removes cache object with the given key from cache tree.
+	// A deleted cache object will be released as soon as all of its handles have
+	// been released.
+	// Delete only happen once, subsequent delete will consider cache object doesn't
+	// exist, even if the cache object ins't released yet.
+	//
+	// If not nil, fin will be called if the cache object doesn't exist or when
+	// finally be released.
+	//
+	// Delete returns true if such cache object exist and never been deleted.
 	Delete(key uint64, fin DelFin) bool
 
-	// Purge deletes all cache objects, read Delete method documentation.
+	// Purge removes all cache objects within this namespace from cache tree.
+	// This is the same as doing delete on all cache objects.
+	//
+	// If not nil, fin will be called on all cache objects when its finally be
+	// released.
 	Purge(fin PurgeFin)
 
-	// Zap detaches the namespace from the cache tree and delete all its cache
-	// objects. The cache objects deletion and finalizers execution are happen
-	// immediately, even if its existing handles haven't yet been released.
-	// A zapped namespace can't never be filled again.
-	// If closed is false then the Get function will always call the given SetFunc
-	// if it is not nil, but resultant of the SetFunc will not be cached.
-	Zap(closed bool)
+	// Zap detaches namespace from cache tree and release all its cache objects.
+	// A zapped namespace can never be filled again.
+	// Calling Get on zapped namespace will always return nil.
+	Zap()
 }
 
-// Object is a cache object.
-type Object interface {
-	// Release releases the cache object. Other methods should not be called
-	// after the cache object has been released.
+// Handle is a cache handle.
+type Handle interface {
+	// Release releases this cache handle. This method can be safely called mutiple
+	// times.
 	Release()
 
-	// Value returns value of the cache object.
+	// Value returns value of this cache handle.
+	// Value will returns nil after this cache handle have be released.
 	Value() interface{}
 }
 
+const (
+	DelNotExist = iota
+	DelExist
+	DelPendig
+)
+
 // Namespace state.
 type nsState int
 
 const (
 	nsEffective nsState = iota
 	nsZapped
-	nsClosed
 )
 
 // Node state.
@@ -97,29 +119,29 @@ type nodeState int
 const (
 	nodeEffective nodeState = iota
 	nodeEvicted
-	nodeRemoved
+	nodeDeleted
 )
 
-// Fake object.
-type fakeObject struct {
+// Fake handle.
+type fakeHandle struct {
 	value interface{}
 	fin   func()
 	once  uint32
 }
 
-func (o *fakeObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.value
+func (h *fakeHandle) Value() interface{} {
+	if atomic.LoadUint32(&h.once) == 0 {
+		return h.value
 	}
 	return nil
 }
 
-func (o *fakeObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+func (h *fakeHandle) Release() {
+	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
 		return
 	}
-	if o.fin != nil {
-		o.fin()
-		o.fin = nil
+	if h.fin != nil {
+		h.fin()
+		h.fin = nil
 	}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -7,15 +7,35 @@
 package cache
 
 import (
+	"fmt"
 	"math/rand"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 )
 
-func set(ns Namespace, key uint64, value interface{}, charge int, fin func()) Object {
-	obj, _ := ns.Get(key, func() (bool, interface{}, int, SetFin) {
-		return true, value, charge, fin
+type releaserFunc struct {
+	fn    func()
+	value interface{}
+}
+
+func (r releaserFunc) Release() {
+	if r.fn != nil {
+		r.fn()
+	}
+}
+
+func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) Handle {
+	return ns.Get(key, func() (int, interface{}) {
+		if relf != nil {
+			return charge, releaserFunc{relf, value}
+		} else {
+			return charge, value
+		}
 	})
-	return obj
 }
 
 func TestCache_HitMiss(t *testing.T) {
@@ -43,29 +63,31 @@ func TestCache_HitMiss(t *testing.T) {
 			setfin++
 		}).Release()
 		for j, y := range cases {
-			r, ok := ns.Get(y.key, nil)
+			h := ns.Get(y.key, nil)
 			if j <= i {
 				// should hit
-				if !ok {
+				if h == nil {
 					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else if r.Value().(string) != y.value {
-					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				} else {
+					if x := h.Value().(releaserFunc).value.(string); x != y.value {
+						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
+					}
 				}
 			} else {
 				// should miss
-				if ok {
-					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, r.Value().(string))
+				if h != nil {
+					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string))
 				}
 			}
-			if ok {
-				r.Release()
+			if h != nil {
+				h.Release()
 			}
 		}
 	}
 
 	for i, x := range cases {
 		finalizerOk := false
-		ns.Delete(x.key, func(exist bool) {
+		ns.Delete(x.key, func(exist, pending bool) {
 			finalizerOk = true
 		})
 
@@ -74,22 +96,24 @@ func TestCache_HitMiss(t *testing.T) {
 		}
 
 		for j, y := range cases {
-			r, ok := ns.Get(y.key, nil)
+			h := ns.Get(y.key, nil)
 			if j > i {
 				// should hit
-				if !ok {
+				if h == nil {
 					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else if r.Value().(string) != y.value {
-					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				} else {
+					if x := h.Value().(releaserFunc).value.(string); x != y.value {
+						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
+					}
 				}
 			} else {
 				// should miss
-				if ok {
-					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, r.Value().(string))
+				if h != nil {
+					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string))
 				}
 			}
-			if ok {
-				r.Release()
+			if h != nil {
+				h.Release()
 			}
 		}
 	}
@@ -107,42 +131,42 @@ func TestLRUCache_Eviction(t *testing.T) {
 	set(ns, 3, 3, 1, nil).Release()
 	set(ns, 4, 4, 1, nil).Release()
 	set(ns, 5, 5, 1, nil).Release()
-	if r, ok := ns.Get(2, nil); ok { // 1,3,4,5,2
-		r.Release()
+	if h := ns.Get(2, nil); h != nil { // 1,3,4,5,2
+		h.Release()
 	}
 	set(ns, 9, 9, 10, nil).Release() // 5,2,9
 
-	for _, x := range []uint64{9, 2, 5, 1} {
-		r, ok := ns.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{9, 2, 5, 1} {
+		h := ns.Get(key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 	o1.Release()
-	for _, x := range []uint64{1, 2, 5} {
-		r, ok := ns.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{1, 2, 5} {
+		h := ns.Get(key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
-	for _, x := range []uint64{3, 4, 9} {
-		r, ok := ns.Get(x, nil)
-		if ok {
-			t.Errorf("hit for key '%d'", x)
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+	for _, key := range []uint64{3, 4, 9} {
+		h := ns.Get(key, nil)
+		if h != nil {
+			t.Errorf("hit for key '%d'", key)
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 }
@@ -153,16 +177,15 @@ func TestLRUCache_SetGet(t *testing.T) {
 	for i := 0; i < 200; i++ {
 		n := uint64(rand.Intn(99999) % 20)
 		set(ns, n, n, 1, nil).Release()
-		if p, ok := ns.Get(n, nil); ok {
-			if p.Value() == nil {
+		if h := ns.Get(n, nil); h != nil {
+			if h.Value() == nil {
 				t.Errorf("key '%d' contains nil value", n)
 			} else {
-				got := p.Value().(uint64)
-				if got != n {
-					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, got)
+				if x := h.Value().(uint64); x != n {
+					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, x)
 				}
 			}
-			p.Release()
+			h.Release()
 		} else {
 			t.Errorf("key '%d' doesn't exist", n)
 		}
@@ -176,31 +199,319 @@ func TestLRUCache_Purge(t *testing.T) {
 	o2 := set(ns1, 2, 2, 1, nil)
 	ns1.Purge(nil)
 	set(ns1, 3, 3, 1, nil).Release()
-	for _, x := range []uint64{1, 2, 3} {
-		r, ok := ns1.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{1, 2, 3} {
+		h := ns1.Get(key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 	o1.Release()
 	o2.Release()
-	for _, x := range []uint64{1, 2} {
-		r, ok := ns1.Get(x, nil)
-		if ok {
-			t.Errorf("hit for key '%d'", x)
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+	for _, key := range []uint64{1, 2} {
+		h := ns1.Get(key, nil)
+		if h != nil {
+			t.Errorf("hit for key '%d'", key)
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 }
 
+type testingCacheObjectCounter struct {
+	created  uint32
+	released uint32
+}
+
+func (c *testingCacheObjectCounter) createOne() {
+	atomic.AddUint32(&c.created, 1)
+}
+
+func (c *testingCacheObjectCounter) releaseOne() {
+	atomic.AddUint32(&c.released, 1)
+}
+
+type testingCacheObject struct {
+	t   *testing.T
+	cnt *testingCacheObjectCounter
+
+	ns, key uint64
+
+	releaseCalled uint32
+}
+
+func (x *testingCacheObject) Release() {
+	if atomic.CompareAndSwapUint32(&x.releaseCalled, 0, 1) {
+		x.cnt.releaseOne()
+	} else {
+		x.t.Errorf("duplicate setfin NS#%d KEY#%s", x.ns, x.key)
+	}
+}
+
+func TestLRUCache_Finalizer(t *testing.T) {
+	const (
+		capacity   = 100
+		goroutines = 100
+		iterations = 10000
+		keymax     = 8000
+	)
+
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	defer runtime.GOMAXPROCS(1)
+
+	wg := &sync.WaitGroup{}
+	cnt := &testingCacheObjectCounter{}
+
+	c := NewLRUCache(capacity)
+
+	type instance struct {
+		seed       int64
+		rnd        *rand.Rand
+		ns         uint64
+		effective  int32
+		handles    []Handle
+		handlesMap map[uint64]int
+
+		delete          bool
+		purge           bool
+		zap             bool
+		wantDel         int32
+		delfinCalledAll int32
+		delfinCalledEff int32
+		purgefinCalled  int32
+	}
+
+	instanceGet := func(p *instance, ns Namespace, key uint64) {
+		h := ns.Get(key, func() (charge int, value interface{}) {
+			to := &testingCacheObject{
+				t: t, cnt: cnt,
+				ns:  p.ns,
+				key: key,
+			}
+			atomic.AddInt32(&p.effective, 1)
+			cnt.createOne()
+			return 1, releaserFunc{func() {
+				to.Release()
+				atomic.AddInt32(&p.effective, -1)
+			}, to}
+		})
+		p.handles = append(p.handles, h)
+		p.handlesMap[key] = p.handlesMap[key] + 1
+	}
+	instanceRelease := func(p *instance, ns Namespace, i int) {
+		h := p.handles[i]
+		key := h.Value().(releaserFunc).value.(*testingCacheObject).key
+		if n := p.handlesMap[key]; n == 0 {
+			t.Fatal("key ref == 0")
+		} else if n > 1 {
+			p.handlesMap[key] = n - 1
+		} else {
+			delete(p.handlesMap, key)
+		}
+		h.Release()
+		p.handles = append(p.handles[:i], p.handles[i+1:]...)
+		p.handles[len(p.handles) : len(p.handles)+1][0] = nil
+	}
+
+	seeds := make([]int64, goroutines)
+	instances := make([]instance, goroutines)
+	for i := range instances {
+		p := &instances[i]
+		p.handlesMap = make(map[uint64]int)
+		if seeds[i] == 0 {
+			seeds[i] = time.Now().UnixNano()
+		}
+		p.seed = seeds[i]
+		p.rnd = rand.New(rand.NewSource(p.seed))
+		p.ns = uint64(i)
+		p.delete = i%6 == 0
+		p.purge = i%8 == 0
+		p.zap = i%12 == 0 || i%3 == 0
+	}
+
+	seedsStr := make([]string, len(seeds))
+	for i, seed := range seeds {
+		seedsStr[i] = fmt.Sprint(seed)
+	}
+	t.Logf("seeds := []int64{%s}", strings.Join(seedsStr, ", "))
+
+	// Get and release.
+	for i := range instances {
+		p := &instances[i]
+
+		wg.Add(1)
+		go func(p *instance) {
+			defer wg.Done()
+
+			ns := c.GetNamespace(p.ns)
+			for i := 0; i < iterations; i++ {
+				if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
+					instanceGet(p, ns, uint64(p.rnd.Intn(keymax)))
+				} else {
+					instanceRelease(p, ns, p.rnd.Intn(len(p.handles)))
+				}
+			}
+		}(p)
+	}
+	wg.Wait()
+
+	if used, cap := c.Used(), c.Capacity(); used > cap {
+		t.Errorf("Used > capacity, used=%d cap=%d", used, cap)
+	}
+
+	// Check effective objects.
+	for i := range instances {
+		p := &instances[i]
+		if int(p.effective) < len(p.handlesMap) {
+			t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap))
+		}
+	}
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// Delete and purge.
+	for i := range instances {
+		p := &instances[i]
+		p.wantDel = p.effective
+
+		wg.Add(1)
+		go func(p *instance) {
+			defer wg.Done()
+
+			ns := c.GetNamespace(p.ns)
+
+			if p.delete {
+				for key := uint64(0); key < keymax; key++ {
+					_, wantExist := p.handlesMap[key]
+					gotExist := ns.Delete(key, func(exist, pending bool) {
+						atomic.AddInt32(&p.delfinCalledAll, 1)
+						if exist {
+							atomic.AddInt32(&p.delfinCalledEff, 1)
+						}
+					})
+					if !gotExist && wantExist {
+						t.Errorf("delete on NS#%d KEY#%d not found", p.ns, key)
+					}
+				}
+
+				var delfinCalled int
+				for key := uint64(0); key < keymax; key++ {
+					func(key uint64) {
+						gotExist := ns.Delete(key, func(exist, pending bool) {
+							if exist && !pending {
+								t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.ns, key)
+							}
+							delfinCalled++
+						})
+						if gotExist {
+							t.Errorf("delete on NS#%d KEY#%d found", p.ns, key)
+						}
+					}(key)
+				}
+				if delfinCalled != keymax {
+					t.Errorf("(2) #%d not all delete fin called, diff=%d", p.ns, keymax-delfinCalled)
+				}
+			}
+
+			if p.purge {
+				ns.Purge(func(ns, key uint64) {
+					atomic.AddInt32(&p.purgefinCalled, 1)
+				})
+			}
+		}(p)
+	}
+	wg.Wait()
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// Release.
+	for i := range instances {
+		p := &instances[i]
+
+		if !p.zap {
+			wg.Add(1)
+			go func(p *instance) {
+				defer wg.Done()
+
+				ns := c.GetNamespace(p.ns)
+				for i := len(p.handles) - 1; i >= 0; i-- {
+					instanceRelease(p, ns, i)
+				}
+			}(p)
+		}
+	}
+	wg.Wait()
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// Zap.
+	for i := range instances {
+		p := &instances[i]
+
+		if p.zap {
+			wg.Add(1)
+			go func(p *instance) {
+				defer wg.Done()
+
+				ns := c.GetNamespace(p.ns)
+				ns.Zap()
+
+				p.handles = nil
+				p.handlesMap = nil
+			}(p)
+		}
+	}
+	wg.Wait()
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	if notrel, used := int(cnt.created-cnt.released), c.Used(); notrel != used {
+		t.Errorf("Invalid used value, want=%d got=%d", notrel, used)
+	}
+
+	c.Purge(nil)
+
+	for i := range instances {
+		p := &instances[i]
+
+		if p.delete {
+			if p.delfinCalledAll != keymax {
+				t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.ns, p.purge, p.zap, keymax-p.delfinCalledAll)
+			}
+			if p.delfinCalledEff != p.wantDel {
+				t.Errorf("#%d not all effective delete fin called, diff=%d", p.ns, p.wantDel-p.delfinCalledEff)
+			}
+			if p.purge && p.purgefinCalled > 0 {
+				t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.ns, p.delete, p.zap, p.purgefinCalled)
+			}
+		} else {
+			if p.purge {
+				if p.purgefinCalled != p.wantDel {
+					t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.ns, p.delete, p.zap, p.wantDel-p.purgefinCalled)
+				}
+			}
+		}
+	}
+
+	if cnt.created != cnt.released {
+		t.Errorf("Some cache object weren't released, created=%d released=%d", cnt.created, cnt.released)
+	}
+}
+
 func BenchmarkLRUCache_SetRelease(b *testing.B) {
 	capacity := b.N / 100
 	if capacity <= 0 {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go

@@ -1,246 +0,0 @@
-// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package cache
-
-import (
-	"sync"
-	"sync/atomic"
-)
-
-type emptyCache struct {
-	sync.Mutex
-	table map[uint64]*emptyNS
-}
-
-// NewEmptyCache creates a new initialized empty cache.
-func NewEmptyCache() Cache {
-	return &emptyCache{
-		table: make(map[uint64]*emptyNS),
-	}
-}
-
-func (c *emptyCache) GetNamespace(id uint64) Namespace {
-	c.Lock()
-	defer c.Unlock()
-
-	if ns, ok := c.table[id]; ok {
-		return ns
-	}
-
-	ns := &emptyNS{
-		cache: c,
-		id:    id,
-		table: make(map[uint64]*emptyNode),
-	}
-	c.table[id] = ns
-	return ns
-}
-
-func (c *emptyCache) Purge(fin PurgeFin) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.purgeNB(fin)
-	}
-	c.Unlock()
-}
-
-func (c *emptyCache) Zap(closed bool) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.zapNB(closed)
-	}
-	c.table = make(map[uint64]*emptyNS)
-	c.Unlock()
-}
-
-func (*emptyCache) SetCapacity(capacity int) {}
-
-type emptyNS struct {
-	cache *emptyCache
-	id    uint64
-	table map[uint64]*emptyNode
-	state nsState
-}
-
-func (ns *emptyNS) Get(key uint64, setf SetFunc) (o Object, ok bool) {
-	ns.cache.Lock()
-
-	switch ns.state {
-	case nsZapped:
-		ns.cache.Unlock()
-		if setf == nil {
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if ok {
-			o = &fakeObject{
-				value: value,
-				fin:   fin,
-			}
-		}
-		return
-	case nsClosed:
-		ns.cache.Unlock()
-		return
-	}
-
-	n, ok := ns.table[key]
-	if ok {
-		n.ref++
-	} else {
-		if setf == nil {
-			ns.cache.Unlock()
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if !ok {
-			ns.cache.Unlock()
-			return
-		}
-
-		n = &emptyNode{
-			ns:     ns,
-			key:    key,
-			value:  value,
-			setfin: fin,
-			ref:    1,
-		}
-		ns.table[key] = n
-	}
-
-	ns.cache.Unlock()
-	o = &emptyObject{node: n}
-	return
-}
-
-func (ns *emptyNS) Delete(key uint64, fin DelFin) bool {
-	ns.cache.Lock()
-
-	if ns.state != nsEffective {
-		ns.cache.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-
-	n, ok := ns.table[key]
-	if !ok {
-		ns.cache.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-	n.delfin = fin
-	ns.cache.Unlock()
-	return true
-}
-
-func (ns *emptyNS) purgeNB(fin PurgeFin) {
-	if ns.state != nsEffective {
-		return
-	}
-	for _, n := range ns.table {
-		n.purgefin = fin
-	}
-}
-
-func (ns *emptyNS) Purge(fin PurgeFin) {
-	ns.cache.Lock()
-	ns.purgeNB(fin)
-	ns.cache.Unlock()
-}
-
-func (ns *emptyNS) zapNB(closed bool) {
-	if ns.state != nsEffective {
-		return
-	}
-	for _, n := range ns.table {
-		n.execFin()
-	}
-	if closed {
-		ns.state = nsClosed
-	} else {
-		ns.state = nsZapped
-	}
-	ns.table = nil
-}
-
-func (ns *emptyNS) Zap(closed bool) {
-	ns.cache.Lock()
-	ns.zapNB(closed)
-	delete(ns.cache.table, ns.id)
-	ns.cache.Unlock()
-}
-
-type emptyNode struct {
-	ns       *emptyNS
-	key      uint64
-	value    interface{}
-	ref      int
-	setfin   SetFin
-	delfin   DelFin
-	purgefin PurgeFin
-}
-
-func (n *emptyNode) execFin() {
-	if n.setfin != nil {
-		n.setfin()
-		n.setfin = nil
-	}
-	if n.purgefin != nil {
-		n.purgefin(n.ns.id, n.key, n.delfin)
-		n.delfin = nil
-		n.purgefin = nil
-	} else if n.delfin != nil {
-		n.delfin(true)
-		n.delfin = nil
-	}
-}
-
-func (n *emptyNode) evict() {
-	n.ns.cache.Lock()
-	n.ref--
-	if n.ref == 0 {
-		if n.ns.state == nsEffective {
-			// Remove elem.
-			delete(n.ns.table, n.key)
-			// Execute finalizer.
-			n.execFin()
-		}
-	} else if n.ref < 0 {
-		panic("leveldb/cache: emptyNode: negative node reference")
-	}
-	n.ns.cache.Unlock()
-}
-
-type emptyObject struct {
-	node *emptyNode
-	once uint32
-}
-
-func (o *emptyObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.node.value
-	}
-	return nil
-}
-
-func (o *emptyObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
-		return
-	}
-	o.node.evict()
-	o.node = nil
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go

@@ -9,16 +9,17 @@ package cache
 import (
 	"sync"
 	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 // lruCache represent a LRU cache state.
 type lruCache struct {
-	sync.Mutex
-
-	recent   lruNode
-	table    map[uint64]*lruNs
-	capacity int
-	size     int
+	mu         sync.Mutex
+	recent     lruNode
+	table      map[uint64]*lruNs
+	capacity   int
+	used, size int
 }
 
 // NewLRUCache creates a new initialized LRU cache with the given capacity.
@@ -32,57 +33,75 @@ func NewLRUCache(capacity int) Cache {
 	return c
 }
 
+func (c *lruCache) Capacity() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.capacity
+}
+
+func (c *lruCache) Used() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.used
+}
+
+func (c *lruCache) Size() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.size
+}
+
 // SetCapacity set cache capacity.
 func (c *lruCache) SetCapacity(capacity int) {
-	c.Lock()
+	c.mu.Lock()
 	c.capacity = capacity
 	c.evict()
-	c.Unlock()
+	c.mu.Unlock()
 }
 
 // GetNamespace return namespace object for given id.
 func (c *lruCache) GetNamespace(id uint64) Namespace {
-	c.Lock()
-	defer c.Unlock()
+	c.mu.Lock()
+	defer c.mu.Unlock()
 
-	if p, ok := c.table[id]; ok {
-		return p
+	if ns, ok := c.table[id]; ok {
+		return ns
 	}
 
-	p := &lruNs{
+	ns := &lruNs{
 		lru:   c,
 		id:    id,
 		table: make(map[uint64]*lruNode),
 	}
-	c.table[id] = p
-	return p
+	c.table[id] = ns
+	return ns
 }
 
 // Purge purge entire cache.
 func (c *lruCache) Purge(fin PurgeFin) {
-	c.Lock()
+	c.mu.Lock()
 	for _, ns := range c.table {
 		ns.purgeNB(fin)
 	}
-	c.Unlock()
+	c.mu.Unlock()
 }
 
-func (c *lruCache) Zap(closed bool) {
-	c.Lock()
+func (c *lruCache) Zap() {
+	c.mu.Lock()
 	for _, ns := range c.table {
-		ns.zapNB(closed)
+		ns.zapNB()
 	}
 	c.table = make(map[uint64]*lruNs)
-	c.Unlock()
+	c.mu.Unlock()
 }
 
 func (c *lruCache) evict() {
 	top := &c.recent
-	for n := c.recent.rPrev; c.size > c.capacity && n != top; {
+	for n := c.recent.rPrev; c.used > c.capacity && n != top; {
 		n.state = nodeEvicted
 		n.rRemove()
-		n.evictNB()
-		c.size -= n.charge
+		n.derefNB()
+		c.used -= n.charge
 		n = c.recent.rPrev
 	}
 }
@@ -94,170 +113,157 @@ type lruNs struct {
 	state nsState
 }
 
-func (ns *lruNs) Get(key uint64, setf SetFunc) (o Object, ok bool) {
-	lru := ns.lru
-	lru.Lock()
+func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
+	ns.lru.mu.Lock()
 
-	switch ns.state {
-	case nsZapped:
-		lru.Unlock()
-		if setf == nil {
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if ok {
-			o = &fakeObject{
-				value: value,
-				fin:   fin,
-			}
-		}
-		return
-	case nsClosed:
-		lru.Unlock()
-		return
+	if ns.state != nsEffective {
+		ns.lru.mu.Unlock()
+		return nil
 	}
 
-	n, ok := ns.table[key]
+	node, ok := ns.table[key]
 	if ok {
-		switch n.state {
+		switch node.state {
 		case nodeEvicted:
 			// Insert to recent list.
-			n.state = nodeEffective
-			n.ref++
-			lru.size += n.charge
-			lru.evict()
+			node.state = nodeEffective
+			node.ref++
+			ns.lru.used += node.charge
+			ns.lru.evict()
 			fallthrough
 		case nodeEffective:
-			// Bump to front
-			n.rRemove()
-			n.rInsert(&lru.recent)
+			// Bump to front.
+			node.rRemove()
+			node.rInsert(&ns.lru.recent)
 		}
-		n.ref++
+		node.ref++
 	} else {
 		if setf == nil {
-			lru.Unlock()
-			return
+			ns.lru.mu.Unlock()
+			return nil
 		}
 
-		var value interface{}
-		var charge int
-		var fin func()
-		ok, value, charge, fin = setf()
-		if !ok {
-			lru.Unlock()
-			return
+		charge, value := setf()
+		if value == nil {
+			ns.lru.mu.Unlock()
+			return nil
 		}
 
-		n = &lruNode{
+		node = &lruNode{
 			ns:     ns,
 			key:    key,
 			value:  value,
 			charge: charge,
-			setfin: fin,
-			ref:    2,
+			ref:    1,
 		}
-		ns.table[key] = n
-		n.rInsert(&lru.recent)
+		ns.table[key] = node
 
-		lru.size += charge
-		lru.evict()
+		if charge > 0 {
+			node.ref++
+			node.rInsert(&ns.lru.recent)
+			ns.lru.used += charge
+			ns.lru.size += charge
+			ns.lru.evict()
+		}
 	}
 
-	lru.Unlock()
-	o = &lruObject{node: n}
-	return
+	ns.lru.mu.Unlock()
+	return &lruHandle{node: node}
 }
 
 func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
-	lru := ns.lru
-	lru.Lock()
+	ns.lru.mu.Lock()
 
 	if ns.state != nsEffective {
-		lru.Unlock()
 		if fin != nil {
-			fin(false)
+			fin(false, false)
 		}
+		ns.lru.mu.Unlock()
 		return false
 	}
 
-	n, ok := ns.table[key]
-	if !ok {
-		lru.Unlock()
+	node, exist := ns.table[key]
+	if !exist {
 		if fin != nil {
-			fin(false)
+			fin(false, false)
 		}
+		ns.lru.mu.Unlock()
 		return false
 	}
 
-	n.delfin = fin
-	switch n.state {
-	case nodeRemoved:
-		lru.Unlock()
+	switch node.state {
+	case nodeDeleted:
+		if fin != nil {
+			fin(true, true)
+		}
+		ns.lru.mu.Unlock()
 		return false
 	case nodeEffective:
-		lru.size -= n.charge
-		n.rRemove()
-		n.evictNB()
+		ns.lru.used -= node.charge
+		node.state = nodeDeleted
+		node.delfin = fin
+		node.rRemove()
+		node.derefNB()
+	default:
+		node.state = nodeDeleted
+		node.delfin = fin
 	}
-	n.state = nodeRemoved
 
-	lru.Unlock()
+	ns.lru.mu.Unlock()
 	return true
 }
 
 func (ns *lruNs) purgeNB(fin PurgeFin) {
-	lru := ns.lru
 	if ns.state != nsEffective {
 		return
 	}
 
-	for _, n := range ns.table {
-		n.purgefin = fin
-		if n.state == nodeEffective {
-			lru.size -= n.charge
-			n.rRemove()
-			n.evictNB()
+	for _, node := range ns.table {
+		switch node.state {
+		case nodeDeleted:
+		case nodeEffective:
+			ns.lru.used -= node.charge
+			node.state = nodeDeleted
+			node.purgefin = fin
+			node.rRemove()
+			node.derefNB()
+		default:
+			node.state = nodeDeleted
+			node.purgefin = fin
 		}
-		n.state = nodeRemoved
 	}
 }
 
 func (ns *lruNs) Purge(fin PurgeFin) {
-	ns.lru.Lock()
+	ns.lru.mu.Lock()
 	ns.purgeNB(fin)
-	ns.lru.Unlock()
+	ns.lru.mu.Unlock()
 }
 
-func (ns *lruNs) zapNB(closed bool) {
-	lru := ns.lru
+func (ns *lruNs) zapNB() {
 	if ns.state != nsEffective {
 		return
 	}
 
-	if closed {
-		ns.state = nsClosed
-	} else {
-		ns.state = nsZapped
-	}
-	for _, n := range ns.table {
-		if n.state == nodeEffective {
-			lru.size -= n.charge
-			n.rRemove()
+	ns.state = nsZapped
+
+	for _, node := range ns.table {
+		if node.state == nodeEffective {
+			ns.lru.used -= node.charge
+			node.rRemove()
 		}
-		n.state = nodeRemoved
-		n.execFin()
+		ns.lru.size -= node.charge
+		node.state = nodeDeleted
+		node.fin()
 	}
 	ns.table = nil
 }
 
-func (ns *lruNs) Zap(closed bool) {
-	ns.lru.Lock()
-	ns.zapNB(closed)
+func (ns *lruNs) Zap() {
+	ns.lru.mu.Lock()
+	ns.zapNB()
 	delete(ns.lru.table, ns.id)
-	ns.lru.Unlock()
+	ns.lru.mu.Unlock()
 }
 
 type lruNode struct {
@@ -270,7 +276,6 @@ type lruNode struct {
 	charge   int
 	ref      int
 	state    nodeState
-	setfin   SetFin
 	delfin   DelFin
 	purgefin PurgeFin
 }
@@ -284,7 +289,6 @@ func (n *lruNode) rInsert(at *lruNode) {
 }
 
 func (n *lruNode) rRemove() bool {
-	// only remove if not already removed
 	if n.rPrev == nil {
 		return false
 	}
@@ -297,58 +301,56 @@ func (n *lruNode) rRemove() bool {
 	return true
 }
 
-func (n *lruNode) execFin() {
-	if n.setfin != nil {
-		n.setfin()
-		n.setfin = nil
+func (n *lruNode) fin() {
+	if r, ok := n.value.(util.Releaser); ok {
+		r.Release()
 	}
 	if n.purgefin != nil {
-		n.purgefin(n.ns.id, n.key, n.delfin)
+		n.purgefin(n.ns.id, n.key)
 		n.delfin = nil
 		n.purgefin = nil
 	} else if n.delfin != nil {
-		n.delfin(true)
+		n.delfin(true, false)
 		n.delfin = nil
 	}
 }
 
-func (n *lruNode) evictNB() {
+func (n *lruNode) derefNB() {
 	n.ref--
 	if n.ref == 0 {
 		if n.ns.state == nsEffective {
-			// remove elem
+			// Remove elemement.
 			delete(n.ns.table, n.key)
-			// execute finalizer
-			n.execFin()
+			n.ns.lru.size -= n.charge
+			n.fin()
 		}
 	} else if n.ref < 0 {
 		panic("leveldb/cache: lruCache: negative node reference")
 	}
 }
 
-func (n *lruNode) evict() {
-	n.ns.lru.Lock()
-	n.evictNB()
-	n.ns.lru.Unlock()
+func (n *lruNode) deref() {
+	n.ns.lru.mu.Lock()
+	n.derefNB()
+	n.ns.lru.mu.Unlock()
 }
 
-type lruObject struct {
+type lruHandle struct {
 	node *lruNode
 	once uint32
 }
 
-func (o *lruObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.node.value
+func (h *lruHandle) Value() interface{} {
+	if atomic.LoadUint32(&h.once) == 0 {
+		return h.node.value
 	}
 	return nil
 }
 
-func (o *lruObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+func (h *lruHandle) Release() {
+	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
 		return
 	}
-
-	o.node.evict()
-	o.node = nil
+	h.node.deref()
+	h.node = nil
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -35,8 +35,8 @@ type DB struct {
 
 	// MemDB.
 	memMu             sync.RWMutex
-	mem               *memdb.DB
-	frozenMem         *memdb.DB
+	memPool           *util.Pool
+	mem, frozenMem    *memDB
 	journal           *journal.Writer
 	journalWriter     storage.Writer
 	journalFile       storage.File
@@ -79,6 +79,8 @@ func openDB(s *session) (*DB, error) {
 		s: s,
 		// Initial sequence
 		seq: s.stSeq,
+		// MemDB
+		memPool: util.NewPool(1),
 		// Write
 		writeC:       make(chan *Batch),
 		writeMergedC: make(chan bool),
@@ -257,6 +259,7 @@ func recoverTable(s *session, o *opt.Options) error {
 	var mSeq uint64
 	var good, corrupted int
 	rec := new(sessionRecord)
+	bpool := util.NewBufferPool(o.GetBlockSize() + 5)
 	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
 		tmp = s.newTemp()
 		writer, err := tmp.Create()
@@ -314,7 +317,7 @@ func recoverTable(s *session, o *opt.Options) error {
 		var tSeq uint64
 		var tgood, tcorrupted, blockerr int
 		var imin, imax []byte
-		tr := table.NewReader(reader, size, nil, o)
+		tr := table.NewReader(reader, size, nil, bpool, o)
 		iter := tr.NewIterator(nil, nil)
 		iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
 			s.logf("table@recovery found error @%d %q", file.Num(), err)
@@ -481,10 +484,11 @@ func (db *DB) recoverJournal() error {
 
 			buf.Reset()
 			if _, err := buf.ReadFrom(r); err != nil {
-				if strict {
+				if err == io.ErrUnexpectedEOF {
+					continue
+				} else {
 					return err
 				}
-				continue
 			}
 			if err := batch.decode(buf.Bytes()); err != nil {
 				return err
@@ -558,19 +562,20 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
 	ikey := newIKey(key, seq, tSeek)
 
 	em, fm := db.getMems()
-	for _, m := range [...]*memdb.DB{em, fm} {
+	for _, m := range [...]*memDB{em, fm} {
 		if m == nil {
 			continue
 		}
+		defer m.decref()
 
-		mk, mv, me := m.Find(ikey)
+		mk, mv, me := m.db.Find(ikey)
 		if me == nil {
 			ukey, _, t, ok := parseIkey(mk)
 			if ok && db.s.icmp.uCompare(ukey, key) == 0 {
 				if t == tDel {
 					return nil, ErrNotFound
 				}
-				return mv, nil
+				return append([]byte{}, mv...), nil
 			}
 		} else if me != ErrNotFound {
 			return nil, me
@@ -590,8 +595,9 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
 // Get gets the value for the given key. It returns ErrNotFound if the
 // DB does not contain the key.
 //
-// The caller should not modify the contents of the returned slice, but
-// it is safe to modify the contents of the argument after Get returns.
+// The returned slice is its own copy, it is safe to modify the contents
+// of the returned slice.
+// It is safe to modify the contents of the argument after Get returns.
 func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
 	err = db.ok()
 	if err != nil {
@@ -649,6 +655,8 @@ func (db *DB) GetSnapshot() (*Snapshot, error) {
 //		Returns statistics of the underlying DB.
 //	leveldb.sstables
 //		Returns sstables list for each level.
+//	leveldb.blockpool
+//		Returns block pool stats.
 func (db *DB) GetProperty(name string) (value string, err error) {
 	err = db.ok()
 	if err != nil {
@@ -694,6 +702,16 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 				value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.imin, t.imax)
 			}
 		}
+	case p == "blockpool":
+		value = fmt.Sprintf("%v", db.s.tops.bpool)
+	case p == "cachedblock":
+		if bc := db.s.o.GetBlockCache(); bc != nil {
+			value = fmt.Sprintf("%d", bc.Size())
+		} else {
+			value = "<nil>"
+		}
+	case p == "openedtables":
+		value = fmt.Sprintf("%d", db.s.tops.cache.Size())
 	default:
 		err = errors.New("leveldb: GetProperty: unknown property: " + name)
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -216,14 +216,15 @@ func (db *DB) memCompaction() {
 	if mem == nil {
 		return
 	}
+	defer mem.decref()
 
 	c := newCMem(db.s)
 	stats := new(cStatsStaging)
 
-	db.logf("mem@flush N·%d S·%s", mem.Len(), shortenb(mem.Size()))
+	db.logf("mem@flush N·%d S·%s", mem.db.Len(), shortenb(mem.db.Size()))
 
 	// Don't compact empty memdb.
-	if mem.Len() == 0 {
+	if mem.db.Len() == 0 {
 		db.logf("mem@flush skipping")
 		// drop frozen mem
 		db.dropFrozenMem()
@@ -241,7 +242,7 @@ func (db *DB) memCompaction() {
 	db.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
 		defer stats.stopTimer()
-		return c.flush(mem, -1)
+		return c.flush(mem.db, -1)
 	}, func() error {
 		for _, r := range c.rec.addedTables {
 			db.logf("mem@flush rollback @%d", r.num)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -9,6 +9,7 @@ package leveldb
 import (
 	"errors"
 	"runtime"
+	"sync"
 
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -19,6 +20,17 @@ var (
 	errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
 )
 
+type memdbReleaser struct {
+	once sync.Once
+	m    *memDB
+}
+
+func (mr *memdbReleaser) Release() {
+	mr.once.Do(func() {
+		mr.m.decref()
+	})
+}
+
 func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
 	em, fm := db.getMems()
 	v := db.s.version()
@@ -26,9 +38,13 @@ func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It
 	ti := v.getIterators(slice, ro)
 	n := len(ti) + 2
 	i := make([]iterator.Iterator, 0, n)
-	i = append(i, em.NewIterator(slice))
+	emi := em.db.NewIterator(slice)
+	emi.SetReleaser(&memdbReleaser{m: em})
+	i = append(i, emi)
 	if fm != nil {
-		i = append(i, fm.NewIterator(slice))
+		fmi := fm.db.NewIterator(slice)
+		fmi.SetReleaser(&memdbReleaser{m: fm})
+		i = append(i, fmi)
 	}
 	i = append(i, ti...)
 	strict := db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go

@@ -11,8 +11,27 @@ import (
 
 	"github.com/syndtr/goleveldb/leveldb/journal"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+type memDB struct {
+	pool *util.Pool
+	db   *memdb.DB
+	ref  int32
+}
+
+func (m *memDB) incref() {
+	atomic.AddInt32(&m.ref, 1)
+}
+
+func (m *memDB) decref() {
+	if ref := atomic.AddInt32(&m.ref, -1); ref == 0 {
+		m.pool.Put(m)
+	} else if ref < 0 {
+		panic("negative memdb ref")
+	}
+}
+
 // Get latest sequence number.
 func (db *DB) getSeq() uint64 {
 	return atomic.LoadUint64(&db.seq)
@@ -25,7 +44,7 @@ func (db *DB) addSeq(delta uint64) {
 
 // Create new memdb and froze the old one; need external synchronization.
 // newMem only called synchronously by the writer.
-func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
+func (db *DB) newMem(n int) (mem *memDB, err error) {
 	num := db.s.allocFileNum()
 	file := db.s.getJournalFile(num)
 	w, err := file.Create()
@@ -37,6 +56,10 @@ func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
 	db.memMu.Lock()
 	defer db.memMu.Unlock()
 
+	if db.frozenMem != nil {
+		panic("still has frozen mem")
+	}
+
 	if db.journal == nil {
 		db.journal = journal.NewWriter(w)
 	} else {
@@ -47,8 +70,19 @@ func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
 	db.journalWriter = w
 	db.journalFile = file
 	db.frozenMem = db.mem
-	db.mem = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
-	mem = db.mem
+	mem, ok := db.memPool.Get().(*memDB)
+	if ok && mem.db.Capacity() >= n {
+		mem.db.Reset()
+		mem.incref()
+	} else {
+		mem = &memDB{
+			pool: db.memPool,
+			db:   memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n)),
+			ref:  1,
+		}
+	}
+	mem.incref()
+	db.mem = mem
 	// The seq only incremented by the writer. And whoever called newMem
 	// should hold write lock, so no need additional synchronization here.
 	db.frozenSeq = db.seq
@@ -56,16 +90,27 @@ func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
 }
 
 // Get all memdbs.
-func (db *DB) getMems() (e *memdb.DB, f *memdb.DB) {
+func (db *DB) getMems() (e, f *memDB) {
 	db.memMu.RLock()
 	defer db.memMu.RUnlock()
+	if db.mem == nil {
+		panic("nil effective mem")
+	}
+	db.mem.incref()
+	if db.frozenMem != nil {
+		db.frozenMem.incref()
+	}
 	return db.mem, db.frozenMem
 }
 
 // Get frozen memdb.
-func (db *DB) getEffectiveMem() *memdb.DB {
+func (db *DB) getEffectiveMem() *memDB {
 	db.memMu.RLock()
 	defer db.memMu.RUnlock()
+	if db.mem == nil {
+		panic("nil effective mem")
+	}
+	db.mem.incref()
 	return db.mem
 }
 
@@ -77,9 +122,12 @@ func (db *DB) hasFrozenMem() bool {
 }
 
 // Get frozen memdb.
-func (db *DB) getFrozenMem() *memdb.DB {
+func (db *DB) getFrozenMem() *memDB {
 	db.memMu.RLock()
 	defer db.memMu.RUnlock()
+	if db.frozenMem != nil {
+		db.frozenMem.incref()
+	}
 	return db.frozenMem
 }
 
@@ -92,6 +140,7 @@ func (db *DB) dropFrozenMem() {
 		db.logf("journal@remove removed @%d", db.frozenJournalFile.Num())
 	}
 	db.frozenJournalFile = nil
+	db.frozenMem.decref()
 	db.frozenMem = nil
 	db.memMu.Unlock()
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go

@@ -45,7 +45,7 @@ func (db *DB) jWriter() {
 	}
 }
 
-func (db *DB) rotateMem(n int) (mem *memdb.DB, err error) {
+func (db *DB) rotateMem(n int) (mem *memDB, err error) {
 	// Wait for pending memdb compaction.
 	err = db.compSendIdle(db.mcompCmdC)
 	if err != nil {
@@ -63,13 +63,19 @@ func (db *DB) rotateMem(n int) (mem *memdb.DB, err error) {
 	return
 }
 
-func (db *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
+func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
 	delayed := false
-	flush := func() bool {
+	flush := func() (retry bool) {
 		v := db.s.version()
 		defer v.release()
 		mem = db.getEffectiveMem()
-		nn = mem.Free()
+		defer func() {
+			if retry {
+				mem.decref()
+				mem = nil
+			}
+		}()
+		nn = mem.db.Free()
 		switch {
 		case v.tLen(0) >= kL0_SlowdownWritesTrigger && !delayed:
 			delayed = true
@@ -84,12 +90,17 @@ func (db *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
 			}
 		default:
 			// Allow memdb to grow if it has no entry.
-			if mem.Len() == 0 {
+			if mem.db.Len() == 0 {
 				nn = n
-				return false
+			} else {
+				mem.decref()
+				mem, err = db.rotateMem(n)
+				if err == nil {
+					nn = mem.db.Free()
+				} else {
+					nn = 0
+				}
 			}
-			mem, err = db.rotateMem(n)
-			nn = mem.Free()
 			return false
 		}
 		return true
@@ -140,6 +151,7 @@ retry:
 	if err != nil {
 		return
 	}
+	defer mem.decref()
 
 	// Calculate maximum size of the batch.
 	m := 1 << 20
@@ -178,7 +190,7 @@ drain:
 			return
 		case db.journalC <- b:
 			// Write into memdb
-			b.memReplay(mem)
+			b.memReplay(mem.db)
 		}
 		// Wait for journal writer
 		select {
@@ -188,7 +200,7 @@ drain:
 		case err = <-db.journalAckC:
 			if err != nil {
 				// Revert memdb if error detected
-				b.revertMemReplay(mem)
+				b.revertMemReplay(mem.db)
 				return
 			}
 		}
@@ -197,7 +209,7 @@ drain:
 		if err != nil {
 			return
 		}
-		b.memReplay(mem)
+		b.memReplay(mem.db)
 	}
 
 	// Set last seq number.
@@ -258,7 +270,8 @@ func (db *DB) CompactRange(r util.Range) error {
 
 	// Check for overlaps in memdb.
 	mem := db.getEffectiveMem()
-	if isMemOverlaps(db.s.icmp, mem, r.Start, r.Limit) {
+	defer mem.decref()
+	if isMemOverlaps(db.s.icmp, mem.db, r.Start, r.Limit) {
 		// Memdb compaction.
 		if _, err := db.rotateMem(0); err != nil {
 			<-db.writeLockC

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go

@@ -37,6 +37,16 @@
 //	err = iter.Error()
 //	...
 //
+// Iterate over subset of database content with a particular prefix:
+//	iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil)
+//	for iter.Next() {
+//		// Use key/value.
+//		...
+//	}
+//	iter.Release()
+//	err = iter.Error()
+//	...
+//
 // Seek-then-Iterate:
 //
 // 	iter := db.NewIterator(nil, nil)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/go13_bench_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build go1.3
+
+package leveldb
+
+import (
+	"sync/atomic"
+	"testing"
+)
+
+func BenchmarkDBReadConcurrent(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.gc()
+	defer p.close()
+
+	b.ResetTimer()
+	b.SetBytes(116)
+
+	b.RunParallel(func(pb *testing.PB) {
+		iter := p.newIter()
+		defer iter.Release()
+		for pb.Next() && iter.Next() {
+		}
+	})
+}
+
+func BenchmarkDBReadConcurrent2(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.gc()
+	defer p.close()
+
+	b.ResetTimer()
+	b.SetBytes(116)
+
+	var dir uint32
+	b.RunParallel(func(pb *testing.PB) {
+		iter := p.newIter()
+		defer iter.Release()
+		if atomic.AddUint32(&dir, 1)%2 == 0 {
+			for pb.Next() && iter.Next() {
+			}
+		} else {
+			if pb.Next() && iter.Last() {
+				for pb.Next() && iter.Prev() {
+				}
+			}
+		}
+	})
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go

@@ -103,18 +103,18 @@ type flusher interface {
 	Flush() error
 }
 
-// DroppedError is the error type that passed to Dropper.Drop method.
-type DroppedError struct {
+// ErrCorrupted is the error type that generated by corrupted block or chunk.
+type ErrCorrupted struct {
 	Size   int
 	Reason string
 }
 
-func (e DroppedError) Error() string {
-	return fmt.Sprintf("leveldb/journal: dropped %d bytes: %s", e.Size, e.Reason)
+func (e ErrCorrupted) Error() string {
+	return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size)
 }
 
 // Dropper is the interface that wrap simple Drop method. The Drop
-// method will be called when the journal reader dropping a chunk.
+// method will be called when the journal reader dropping a block or chunk.
 type Dropper interface {
 	Drop(err error)
 }
@@ -158,76 +158,78 @@ func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader {
 	}
 }
 
+var errSkip = errors.New("leveldb/journal: skipped")
+
+func (r *Reader) corrupt(n int, reason string, skip bool) error {
+	if r.dropper != nil {
+		r.dropper.Drop(ErrCorrupted{n, reason})
+	}
+	if r.strict && !skip {
+		r.err = ErrCorrupted{n, reason}
+		return r.err
+	}
+	return errSkip
+}
+
 // nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the
 // next block into the buffer if necessary.
-func (r *Reader) nextChunk(wantFirst, skip bool) error {
+func (r *Reader) nextChunk(first bool) error {
 	for {
 		if r.j+headerSize <= r.n {
 			checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4])
 			length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6])
 			chunkType := r.buf[r.j+6]
 
-			var err error
 			if checksum == 0 && length == 0 && chunkType == 0 {
 				// Drop entire block.
-				err = DroppedError{r.n - r.j, "zero header"}
+				m := r.n - r.j
 				r.i = r.n
 				r.j = r.n
+				return r.corrupt(m, "zero header", false)
 			} else {
 				m := r.n - r.j
 				r.i = r.j + headerSize
 				r.j = r.j + headerSize + int(length)
 				if r.j > r.n {
 					// Drop entire block.
-					err = DroppedError{m, "chunk length overflows block"}
 					r.i = r.n
 					r.j = r.n
+					return r.corrupt(m, "chunk length overflows block", false)
 				} else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() {
 					// Drop entire block.
-					err = DroppedError{m, "checksum mismatch"}
 					r.i = r.n
 					r.j = r.n
+					return r.corrupt(m, "checksum mismatch", false)
 				}
 			}
-			if wantFirst && err == nil && chunkType != fullChunkType && chunkType != firstChunkType {
-				if skip {
-					// The chunk are intentionally skipped.
-					if chunkType == lastChunkType {
-						skip = false
-					}
-					continue
-				} else {
-					// Drop the chunk.
-					err = DroppedError{r.j - r.i + headerSize, "orphan chunk"}
-				}
+			if first && chunkType != fullChunkType && chunkType != firstChunkType {
+				m := r.j - r.i
+				r.i = r.j
+				// Report the error, but skip it.
+				return r.corrupt(m+headerSize, "orphan chunk", true)
 			}
-			if err == nil {
-				r.last = chunkType == fullChunkType || chunkType == lastChunkType
-			} else {
-				if r.dropper != nil {
-					r.dropper.Drop(err)
-				}
-				if r.strict {
-					r.err = err
-				}
+			r.last = chunkType == fullChunkType || chunkType == lastChunkType
+			return nil
+		}
+
+		// The last block.
+		if r.n < blockSize && r.n > 0 {
+			if !first {
+				return r.corrupt(0, "missing chunk part", false)
 			}
+			r.err = io.EOF
+			return r.err
+		}
+
+		// Read block.
+		n, err := io.ReadFull(r.r, r.buf[:])
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			return err
 		}
-		if r.n < blockSize && r.n > 0 {
-			// This is the last block.
-			if r.j != r.n {
-				r.err = io.ErrUnexpectedEOF
-			} else {
-				r.err = io.EOF
-			}
-			return r.err
-		}
-		n, err := io.ReadFull(r.r, r.buf[:])
-		if err != nil && err != io.ErrUnexpectedEOF {
-			r.err = err
-			return r.err
-		}
 		if n == 0 {
+			if !first {
+				return r.corrupt(0, "missing chunk part", false)
+			}
 			r.err = io.EOF
 			return r.err
 		}
@@ -237,29 +239,26 @@ func (r *Reader) nextChunk(wantFirst, skip bool) error {
 
 // Next returns a reader for the next journal. It returns io.EOF if there are no
 // more journals. The reader returned becomes stale after the next Next call,
-// and should no longer be used.
+// and should no longer be used. If strict is false, the reader will returns
+// io.ErrUnexpectedEOF error when found corrupted journal.
 func (r *Reader) Next() (io.Reader, error) {
 	r.seq++
 	if r.err != nil {
 		return nil, r.err
 	}
-	skip := !r.last
+	r.i = r.j
 	for {
-		r.i = r.j
-		if r.nextChunk(true, skip) != nil {
-			// So that 'orphan chunk' drop will be reported.
-			skip = false
-		} else {
+		if err := r.nextChunk(true); err == nil {
 			break
-		}
-		if r.err != nil {
-			return nil, r.err
+		} else if err != errSkip {
+			return nil, err
 		}
 	}
 	return &singleReader{r, r.seq, nil}, nil
 }
 
-// Reset resets the journal reader, allows reuse of the journal reader.
+// Reset resets the journal reader, allows reuse of the journal reader. Reset returns
+// last accumulated error.
 func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error {
 	r.seq++
 	err := r.err
@@ -296,7 +295,11 @@ func (x *singleReader) Read(p []byte) (int, error) {
 		if r.last {
 			return 0, io.EOF
 		}
-		if x.err = r.nextChunk(false, false); x.err != nil {
+		x.err = r.nextChunk(false)
+		if x.err != nil {
+			if x.err == errSkip {
+				x.err = io.ErrUnexpectedEOF
+			}
 			return 0, x.err
 		}
 	}
@@ -320,7 +323,11 @@ func (x *singleReader) ReadByte() (byte, error) {
 		if r.last {
 			return 0, io.EOF
 		}
-		if x.err = r.nextChunk(false, false); x.err != nil {
+		x.err = r.nextChunk(false)
+		if x.err != nil {
+			if x.err == errSkip {
+				x.err = io.ErrUnexpectedEOF
+			}
 			return 0, x.err
 		}
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go

@@ -12,6 +12,7 @@ package journal
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -326,3 +327,492 @@ func TestStaleWriter(t *testing.T) {
 		t.Fatalf("stale write #1: unexpected error: %v", err)
 	}
 }
+
+func TestCorrupt_MissingLastBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-1024)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Cut the last block.
+	b := buf.Bytes()[:blockSize]
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read.
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if n != blockSize-1024 {
+		t.Fatalf("read #0: got %d bytes want %d", n, blockSize-1024)
+	}
+
+	// Second read.
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedFirstBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #0.
+	for i := 0; i < 1024; i++ {
+		b[i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (third record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 2; n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedMiddleBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #1.
+	for i := 0; i < 1024; i++ {
+		b[blockSize+i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	// Third read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #2: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 2; n != want {
+		t.Fatalf("read #2: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedLastBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #3.
+	for i := len(b) - 1; i > len(b)-1024; i-- {
+		b[i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize - headerSize); n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	// Third read (third record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #2: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #2: got %d bytes want %d", n, want)
+	}
+
+	// Fourth read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #3: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_FirstChuckLengthOverflow(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting record #1.
+	x := blockSize
+	binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_MiddleChuckLengthOverflow(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting record #1.
+	x := blockSize/2 + headerSize
+	binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (third record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -31,9 +31,12 @@ const (
 type noCache struct{}
 
 func (noCache) SetCapacity(capacity int)               {}
+func (noCache) Capacity() int                          { return 0 }
+func (noCache) Used() int                              { return 0 }
+func (noCache) Size() int                              { return 0 }
 func (noCache) GetNamespace(id uint64) cache.Namespace { return nil }
 func (noCache) Purge(fin cache.PurgeFin)               {}
-func (noCache) Zap(closed bool)                        {}
+func (noCache) Zap()                                   {}
 
 var NoCache cache.Cache = noCache{}
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go

@@ -22,7 +22,7 @@ type dropper struct {
 }
 
 func (d dropper) Drop(err error) {
-	if e, ok := err.(journal.DroppedError); ok {
+	if e, ok := err.(journal.ErrCorrupted); ok {
 		d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason)
 	} else {
 		d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go

@@ -7,6 +7,7 @@
 package leveldb
 
 import (
+	"io"
 	"sort"
 	"sync/atomic"
 
@@ -275,6 +276,7 @@ type tOps struct {
 	s       *session
 	cache   cache.Cache
 	cacheNS cache.Namespace
+	bpool   *util.BufferPool
 }
 
 // Creates an empty table and returns table writer.
@@ -296,7 +298,7 @@ func (t *tOps) create() (*tWriter, error) {
 func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	w, err := t.create()
 	if err != nil {
-		return f, n, err
+		return
 	}
 
 	defer func() {
@@ -321,33 +323,33 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	return
 }
 
-// Opens table. It returns a cache object, which should
+type tableWrapper struct {
+	*table.Reader
+	closer io.Closer
+}
+
+func (tr tableWrapper) Release() {
+	tr.closer.Close()
+}
+
+// Opens table. It returns a cache handle, which should
 // be released after use.
-func (t *tOps) open(f *tFile) (c cache.Object, err error) {
+func (t *tOps) open(f *tFile) (ch cache.Handle, err error) {
 	num := f.file.Num()
-	c, ok := t.cacheNS.Get(num, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
+	ch = t.cacheNS.Get(num, func() (charge int, value interface{}) {
 		var r storage.Reader
 		r, err = f.file.Open()
 		if err != nil {
-			return
+			return 0, nil
 		}
 
-		o := t.s.o
-
-		var cacheNS cache.Namespace
-		if bc := o.GetBlockCache(); bc != nil {
-			cacheNS = bc.GetNamespace(num)
+		var bcacheNS cache.Namespace
+		if bc := t.s.o.GetBlockCache(); bc != nil {
+			bcacheNS = bc.GetNamespace(num)
 		}
-
-		ok = true
-		value = table.NewReader(r, int64(f.size), cacheNS, o)
-		charge = 1
-		fin = func() {
-			r.Close()
-		}
-		return
+		return 1, tableWrapper{table.NewReader(r, int64(f.size), bcacheNS, t.bpool, t.s.o), r}
 	})
-	if !ok && err == nil {
+	if ch == nil && err == nil {
 		err = ErrClosed
 	}
 	return
@@ -356,34 +358,34 @@ func (t *tOps) open(f *tFile) (c cache.Object, err error) {
 // Finds key/value pair whose key is greater than or equal to the
 // given key.
 func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
-	c, err := t.open(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return nil, nil, err
 	}
-	defer c.Release()
-	return c.Value().(*table.Reader).Find(key, ro)
+	defer ch.Release()
+	return ch.Value().(tableWrapper).Find(key, ro)
 }
 
 // Returns approximate offset of the given key.
 func (t *tOps) offsetOf(f *tFile, key []byte) (offset uint64, err error) {
-	c, err := t.open(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return
 	}
-	_offset, err := c.Value().(*table.Reader).OffsetOf(key)
+	_offset, err := ch.Value().(tableWrapper).OffsetOf(key)
 	offset = uint64(_offset)
-	c.Release()
+	ch.Release()
 	return
 }
 
 // Creates an iterator from the given table.
 func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
-	c, err := t.open(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
-	iter := c.Value().(*table.Reader).NewIterator(slice, ro)
-	iter.SetReleaser(c)
+	iter := ch.Value().(tableWrapper).NewIterator(slice, ro)
+	iter.SetReleaser(ch)
 	return iter
 }
 
@@ -391,14 +393,16 @@ func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) ite
 // no one use the the table.
 func (t *tOps) remove(f *tFile) {
 	num := f.file.Num()
-	t.cacheNS.Delete(num, func(exist bool) {
-		if err := f.file.Remove(); err != nil {
-			t.s.logf("table@remove removing @%d %q", num, err)
-		} else {
-			t.s.logf("table@remove removed @%d", num)
-		}
-		if bc := t.s.o.GetBlockCache(); bc != nil {
-			bc.GetNamespace(num).Zap(false)
+	t.cacheNS.Delete(num, func(exist, pending bool) {
+		if !pending {
+			if err := f.file.Remove(); err != nil {
+				t.s.logf("table@remove removing @%d %q", num, err)
+			} else {
+				t.s.logf("table@remove removed @%d", num)
+			}
+			if bc := t.s.o.GetBlockCache(); bc != nil {
+				bc.GetNamespace(num).Zap()
+			}
 		}
 	})
 }
@@ -406,14 +410,18 @@ func (t *tOps) remove(f *tFile) {
 // Closes the table ops instance. It will close all tables,
 // regadless still used or not.
 func (t *tOps) close() {
-	t.cache.Zap(true)
+	t.cache.Zap()
 }
 
 // Creates new initialized table ops instance.
 func newTableOps(s *session, cacheCap int) *tOps {
 	c := cache.NewLRUCache(cacheCap)
-	ns := c.GetNamespace(0)
-	return &tOps{s, c, ns}
+	return &tOps{
+		s:       s,
+		cache:   c,
+		cacheNS: c.GetNamespace(0),
+		bpool:   util.NewBufferPool(s.o.GetBlockSize() + 5),
+	}
 }
 
 // tWriter wraps the table writer. It keep track of file descriptor

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go

@@ -37,6 +37,7 @@ func max(x, y int) int {
 }
 
 type block struct {
+	bpool          *util.BufferPool
 	cmp            comparer.BasicComparer
 	data           []byte
 	restartsLen    int
@@ -139,6 +140,14 @@ func (b *block) newIterator(slice *util.Range, inclLimit bool, cache util.Releas
 	return bi
 }
 
+func (b *block) Release() {
+	if b.bpool != nil {
+		b.bpool.Put(b.data)
+		b.bpool = nil
+	}
+	b.data = nil
+}
+
 type dir int
 
 const (
@@ -437,18 +446,21 @@ func (i *blockIter) Value() []byte {
 }
 
 func (i *blockIter) Release() {
-	i.prevNode = nil
-	i.prevKeys = nil
-	i.key = nil
-	i.value = nil
-	i.dir = dirReleased
-	if i.cache != nil {
-		i.cache.Release()
-		i.cache = nil
-	}
-	if i.releaser != nil {
-		i.releaser.Release()
-		i.releaser = nil
+	if i.dir > dirReleased {
+		i.block = nil
+		i.prevNode = nil
+		i.prevKeys = nil
+		i.key = nil
+		i.value = nil
+		i.dir = dirReleased
+		if i.cache != nil {
+			i.cache.Release()
+			i.cache = nil
+		}
+		if i.releaser != nil {
+			i.releaser.Release()
+			i.releaser = nil
+		}
 	}
 }
 
@@ -519,6 +531,7 @@ type Reader struct {
 	reader io.ReaderAt
 	cache  cache.Namespace
 	err    error
+	bpool  *util.BufferPool
 	// Options
 	cmp        comparer.Comparer
 	filter     filter.Filter
@@ -538,7 +551,7 @@ func verifyChecksum(data []byte) bool {
 }
 
 func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
-	data := make([]byte, bh.length+blockTrailerLen)
+	data := r.bpool.Get(int(bh.length + blockTrailerLen))
 	if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF {
 		return nil, err
 	}
@@ -551,8 +564,13 @@ func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
 	case blockTypeNoCompression:
 		data = data[:bh.length]
 	case blockTypeSnappyCompression:
-		var err error
-		data, err = snappy.Decode(nil, data[:bh.length])
+		decLen, err := snappy.DecodedLen(data[:bh.length])
+		if err != nil {
+			return nil, err
+		}
+		tmp := data
+		data, err = snappy.Decode(r.bpool.Get(decLen), tmp[:bh.length])
+		r.bpool.Put(tmp)
 		if err != nil {
 			return nil, err
 		}
@@ -606,23 +624,21 @@ func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fi
 	if r.cache != nil {
 		// Get/set block cache.
 		var err error
-		cache, ok := r.cache.Get(dataBH.offset, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
+		cache := r.cache.Get(dataBH.offset, func() (charge int, value interface{}) {
 			if !fillCache {
-				return
+				return 0, nil
 			}
 			var dataBlock *block
 			dataBlock, err = r.readBlock(dataBH, checksum)
-			if err == nil {
-				ok = true
-				value = dataBlock
-				charge = int(dataBH.length)
+			if err != nil {
+				return 0, nil
 			}
-			return
+			return int(dataBH.length), dataBlock
 		})
 		if err != nil {
 			return iterator.NewEmptyIterator(err)
 		}
-		if ok {
+		if cache != nil {
 			dataBlock := cache.Value().(*block)
 			if !dataBlock.checksum && (r.checksum || checksum) {
 				if !verifyChecksum(dataBlock.data) {
@@ -638,7 +654,7 @@ func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fi
 	if err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
-	iter := dataBlock.newIterator(slice, false, nil)
+	iter := dataBlock.newIterator(slice, false, dataBlock)
 	return iter
 }
 
@@ -708,8 +724,11 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
 		}
 		return
 	}
+	// Don't use block buffer, no need to copy the buffer.
 	rkey = data.Key()
-	value = data.Value()
+	// Use block buffer, and since the buffer will be recycled, the buffer
+	// need to be copied.
+	value = append([]byte{}, data.Value()...)
 	return
 }
 
@@ -760,13 +779,17 @@ func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
 }
 
 // NewReader creates a new initialized table reader for the file.
-// The cache is optional and can be nil.
+// The cache and bpool is optional and can be nil.
 //
 // The returned table reader instance is goroutine-safe.
-func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, o *opt.Options) *Reader {
+func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) *Reader {
+	if bpool == nil {
+		bpool = util.NewBufferPool(o.GetBlockSize() + blockTrailerLen)
+	}
 	r := &Reader{
 		reader:     f,
 		cache:      cache,
+		bpool:      bpool,
 		cmp:        o.GetComparer(),
 		checksum:   o.GetStrict(opt.StrictBlockChecksum),
 		strictIter: o.GetStrict(opt.StrictIterator),

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go

@@ -59,7 +59,7 @@ var _ = testutil.Defer(func() {
 			It("Should be able to approximate offset of a key correctly", func() {
 				Expect(err).ShouldNot(HaveOccurred())
 
-				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, o)
 				CheckOffset := func(key string, expect, threshold int) {
 					offset, err := tr.OffsetOf([]byte(key))
 					Expect(err).ShouldNot(HaveOccurred())
@@ -95,7 +95,7 @@ var _ = testutil.Defer(func() {
 				tw.Close()
 
 				// Opening the table.
-				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, o)
 				return tableWrapper{tr}
 			}
 			Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool_legacy.go

@@ -0,0 +1,162 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package util
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+)
+
+type buffer struct {
+	b    []byte
+	miss int
+}
+
+// BufferPool is a 'buffer pool'.
+type BufferPool struct {
+	pool      [4]chan []byte
+	size      [3]uint32
+	sizeMiss  [3]uint32
+	baseline0 int
+	baseline1 int
+	baseline2 int
+
+	get     uint32
+	put     uint32
+	less    uint32
+	equal   uint32
+	greater uint32
+	miss    uint32
+}
+
+func (p *BufferPool) poolNum(n int) int {
+	switch {
+	case n <= p.baseline0:
+		return 0
+	case n <= p.baseline1:
+		return 1
+	case n <= p.baseline2:
+		return 2
+	default:
+		return 3
+	}
+}
+
+// Get returns buffer with length of n.
+func (p *BufferPool) Get(n int) []byte {
+	atomic.AddUint32(&p.get, 1)
+
+	poolNum := p.poolNum(n)
+	pool := p.pool[poolNum]
+	if poolNum == 0 {
+		// Fast path.
+		select {
+		case b := <-pool:
+			switch {
+			case cap(b) > n:
+				atomic.AddUint32(&p.less, 1)
+				return b[:n]
+			case cap(b) == n:
+				atomic.AddUint32(&p.equal, 1)
+				return b[:n]
+			default:
+				panic("not reached")
+			}
+		default:
+			atomic.AddUint32(&p.miss, 1)
+		}
+
+		return make([]byte, n, p.baseline0)
+	} else {
+		sizePtr := &p.size[poolNum-1]
+
+		select {
+		case b := <-pool:
+			switch {
+			case cap(b) > n:
+				atomic.AddUint32(&p.less, 1)
+				return b[:n]
+			case cap(b) == n:
+				atomic.AddUint32(&p.equal, 1)
+				return b[:n]
+			default:
+				atomic.AddUint32(&p.greater, 1)
+				if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) {
+					select {
+					case pool <- b:
+					default:
+					}
+				}
+			}
+		default:
+			atomic.AddUint32(&p.miss, 1)
+		}
+
+		if size := atomic.LoadUint32(sizePtr); uint32(n) > size {
+			if size == 0 {
+				atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n))
+			} else {
+				sizeMissPtr := &p.sizeMiss[poolNum-1]
+				if atomic.AddUint32(sizeMissPtr, 1) == 20 {
+					atomic.StoreUint32(sizePtr, uint32(n))
+					atomic.StoreUint32(sizeMissPtr, 0)
+				}
+			}
+			return make([]byte, n)
+		} else {
+			return make([]byte, n, size)
+		}
+	}
+}
+
+// Put adds given buffer to the pool.
+func (p *BufferPool) Put(b []byte) {
+	atomic.AddUint32(&p.put, 1)
+
+	pool := p.pool[p.poolNum(cap(b))]
+	select {
+	case pool <- b:
+	default:
+	}
+
+}
+
+func (p *BufferPool) String() string {
+	return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v G·%d P·%d <·%d =·%d >·%d M·%d}",
+		p.baseline0, p.size, p.sizeMiss, p.get, p.put, p.less, p.equal, p.greater, p.miss)
+}
+
+func (p *BufferPool) drain() {
+	for {
+		time.Sleep(1 * time.Second)
+		select {
+		case <-p.pool[0]:
+		case <-p.pool[1]:
+		case <-p.pool[2]:
+		case <-p.pool[3]:
+		default:
+		}
+	}
+}
+
+// NewBufferPool creates a new initialized 'buffer pool'.
+func NewBufferPool(baseline int) *BufferPool {
+	if baseline <= 0 {
+		panic("baseline can't be <= 0")
+	}
+	p := &BufferPool{
+		baseline0: baseline,
+		baseline1: baseline * 2,
+		baseline2: baseline * 4,
+	}
+	for i, cap := range []int{6, 6, 3, 1} {
+		p.pool[i] = make(chan []byte, cap)
+	}
+	go p.drain()
+	return p
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build go1.3
+
+package util
+
+import (
+	"sync"
+)
+
+type Pool struct {
+	sync.Pool
+}
+
+func NewPool(cap int) *Pool {
+	return &Pool{}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool_legacy.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build !go1.3
+
+package util
+
+type Pool struct {
+	pool chan interface{}
+}
+
+func (p *Pool) Get() interface{} {
+	select {
+	case x := <-p.pool:
+		return x
+	default:
+		return nil
+	}
+}
+
+func (p *Pool) Put(x interface{}) {
+	select {
+	case p.pool <- x:
+	default:
+	}
+}
+
+func NewPool(cap int) *Pool {
+	return &Pool{pool: make(chan interface{}, cap)}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go

@@ -14,3 +14,18 @@ type Range struct {
 	// Limit of the key range, not include in the range.
 	Limit []byte
 }
+
+// BytesPrefix returns key range that satisfy the given prefix.
+// This only applicable for the standard 'bytes comparer'.
+func BytesPrefix(prefix []byte) *Range {
+	var limit []byte
+	for i := len(prefix) - 1; i >= 0; i-- {
+		c := prefix[i]
+		if c < 0xff {
+			limit = make([]byte, i+1)
+			copy(limit, prefix)
+			limit[i] = c + 1
+		}
+	}
+	return &Range{prefix, limit}
+}

README.md

@@ -1,10 +1,9 @@
 syncthing
 =========
 
-[![Build Status](https://img.shields.io/travis/calmh/syncthing.svg?style=flat)](https://travis-ci.org/calmh/syncthing)
-[![Coverage Status](https://img.shields.io/coveralls/calmh/syncthing.svg?style=flat)](https://coveralls.io/r/calmh/syncthing?branch=master)
-[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/syncthing)
-[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
+[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT)
 
 This is the `syncthing` project. The following are the project goals:
 
@@ -12,7 +11,7 @@ This is the `syncthing` project. The following are the project goals:
     number of collaborating nodes. The protocol should be well defined,
     unambiguous, easily understood, free to use, efficient, secure and
     language neutral. This is the [Block Exchange
-    Protocol](https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md).
+    Protocol](https://github.com/syncthing/syncthing/blob/master/protocol/PROTOCOL.md).
 
  2. Provide the reference implementation to demonstrate the usability of
     said protocol. This is the `syncthing` utility. It is the hope that
@@ -26,14 +25,22 @@ for incompatible changes.
 Getting Started
 ---------------
 
-Take a look at the [getting started guide](http://discourse.syncthing.net/t/getting-started/46).
+Take a look at the [getting started guide](http://discourse.syncthing.net/t/46).
+
+Building
+--------
+
+Building Syncthing from source is easy, and there's a
+[guide](http://discourse.syncthing.net/t/44)
+that describes it for both Unix and Windows.
 
 Signed Releases
 ---------------
 
 As of v0.7.0 and onwards, git tags and release binaries are GPG signed with
-the key BCE524C7 (http://nym.se/gpg.txt). The signature is included in the
-normal release bundle as `syncthing.asc` or `syncthing.exe.asc`.
+the key BCE524C7 (http://nym.se/gpg.txt). For release binaries, MD5 and
+SHA1 checksums are calculated and signed, available in the
+md5sum.txt.asc and sha1sum.txt.asc files.
 
 Documentation
 =============
@@ -50,4 +57,4 @@ under the [Creative Commons Attribution 4.0 International
 License](http://creativecommons.org/licenses/by/4.0/).
 
 All code is licensed under the [MIT
-License](https://github.com/calmh/syncthing/blob/master/LICENSE).
+License](https://github.com/syncthing/syncthing/blob/master/LICENSE).

auto/auto_test.go

@@ -0,0 +1,7 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package auto_test
+
+// Empty test file to generate 0% coverage rather than no coverage

beacon/beacon.go

@@ -16,47 +16,17 @@ type dst struct {
 	conn *net.UDPConn
 }
 
-type Beacon struct {
-	conn   *net.UDPConn
-	port   int
-	conns  []dst
-	inbox  chan []byte
-	outbox chan recv
+type Interface interface {
+	Send(data []byte)
+	Recv() ([]byte, net.Addr)
 }
 
-func New(port int) (*Beacon, error) {
-	conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: port})
-	if err != nil {
-		return nil, err
-	}
-	b := &Beacon{
-		conn:   conn,
-		port:   port,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-	}
-
-	go b.reader()
-	go b.writer()
-
-	return b, nil
-}
-
-func (b *Beacon) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Beacon) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Beacon) reader() {
+func genericReader(conn *net.UDPConn, outbox chan<- recv) {
 	bs := make([]byte, 65536)
 	for {
-		n, addr, err := b.conn.ReadFrom(bs)
+		n, addr, err := conn.ReadFrom(bs)
 		if err != nil {
-			l.Warnln("Beacon read:", err)
+			l.Warnln("multicast read:", err)
 			return
 		}
 		if debug {
@@ -66,7 +36,7 @@ func (b *Beacon) reader() {
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
-		case b.outbox <- recv{c, addr}:
+		case outbox <- recv{c, addr}:
 		default:
 			if debug {
 				l.Debugln("dropping message")
@@ -74,59 +44,3 @@ func (b *Beacon) reader() {
 		}
 	}
 }
-
-func (b *Beacon) writer() {
-	for bs := range b.inbox {
-
-		addrs, err := net.InterfaceAddrs()
-		if err != nil {
-			l.Warnln("Beacon: interface addresses:", err)
-			continue
-		}
-
-		var dsts []net.IP
-		for _, addr := range addrs {
-			if iaddr, ok := addr.(*net.IPNet); ok && iaddr.IP.IsGlobalUnicast() && iaddr.IP.To4() != nil {
-				baddr := bcast(iaddr)
-				dsts = append(dsts, baddr.IP)
-			}
-		}
-
-		if len(dsts) == 0 {
-			// Fall back to the general IPv4 broadcast address
-			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
-		}
-
-		if debug {
-			l.Debugln("addresses:", dsts)
-		}
-
-		for _, ip := range dsts {
-			dst := &net.UDPAddr{IP: ip, Port: b.port}
-
-			_, err := b.conn.WriteTo(bs, dst)
-			if err != nil {
-				if debug {
-					l.Debugln(err)
-				}
-			} else if debug {
-				l.Debugf("sent %d bytes to %s", len(bs), dst)
-			}
-		}
-	}
-}
-
-func bcast(ip *net.IPNet) *net.IPNet {
-	var bc = &net.IPNet{}
-	bc.IP = make([]byte, len(ip.IP))
-	copy(bc.IP, ip.IP)
-	bc.Mask = ip.Mask
-
-	offset := len(bc.IP) - len(bc.Mask)
-	for i := range bc.IP {
-		if i-offset >= 0 {
-			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
-		}
-	}
-	return bc
-}

beacon/broadcast.go

@@ -0,0 +1,98 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package beacon
+
+import "net"
+
+type Broadcast struct {
+	conn   *net.UDPConn
+	port   int
+	conns  []dst
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func NewBroadcast(port int) (*Broadcast, error) {
+	conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, err
+	}
+	b := &Broadcast{
+		conn:   conn,
+		port:   port,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	go genericReader(b.conn, b.outbox)
+	go b.writer()
+
+	return b, nil
+}
+
+func (b *Broadcast) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Broadcast) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+func (b *Broadcast) writer() {
+	for bs := range b.inbox {
+
+		addrs, err := net.InterfaceAddrs()
+		if err != nil {
+			l.Warnln("Broadcast: interface addresses:", err)
+			continue
+		}
+
+		var dsts []net.IP
+		for _, addr := range addrs {
+			if iaddr, ok := addr.(*net.IPNet); ok && iaddr.IP.IsGlobalUnicast() && iaddr.IP.To4() != nil {
+				baddr := bcast(iaddr)
+				dsts = append(dsts, baddr.IP)
+			}
+		}
+
+		if len(dsts) == 0 {
+			// Fall back to the general IPv4 broadcast address
+			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
+		}
+
+		if debug {
+			l.Debugln("addresses:", dsts)
+		}
+
+		for _, ip := range dsts {
+			dst := &net.UDPAddr{IP: ip, Port: b.port}
+
+			_, err := b.conn.WriteTo(bs, dst)
+			if err != nil {
+				if debug {
+					l.Debugln(err)
+				}
+			} else if debug {
+				l.Debugf("sent %d bytes to %s", len(bs), dst)
+			}
+		}
+	}
+}
+
+func bcast(ip *net.IPNet) *net.IPNet {
+	var bc = &net.IPNet{}
+	bc.IP = make([]byte, len(ip.IP))
+	copy(bc.IP, ip.IP)
+	bc.Mask = ip.Mask
+
+	offset := len(bc.IP) - len(bc.Mask)
+	for i := range bc.IP {
+		if i-offset >= 0 {
+			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
+		}
+	}
+	return bc
+}

beacon/debug.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/syncthing/logger"
+	"github.com/syncthing/syncthing/logger"
 )
 
 var (

beacon/multicast.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package beacon
+
+import "net"
+
+type Multicast struct {
+	conn   *net.UDPConn
+	addr   *net.UDPAddr
+	conns  []dst
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func NewMulticast(addr string) (*Multicast, error) {
+	gaddr, err := net.ResolveUDPAddr("udp", addr)
+	if err != nil {
+		return nil, err
+	}
+	conn, err := net.ListenMulticastUDP("udp", nil, gaddr)
+	if err != nil {
+		return nil, err
+	}
+	b := &Multicast{
+		conn:   conn,
+		addr:   gaddr,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	go genericReader(b.conn, b.outbox)
+	go b.writer()
+
+	return b, nil
+}
+
+func (b *Multicast) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Multicast) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+func (b *Multicast) writer() {
+	for bs := range b.inbox {
+		intfs, err := net.Interfaces()
+		if err != nil {
+			l.Warnln("multicast interfaces:", err)
+			continue
+		}
+		for _, intf := range intfs {
+			if intf.Flags&net.FlagUp != 0 && intf.Flags&net.FlagMulticast != 0 {
+				addr := *b.addr
+				addr.Zone = intf.Name
+				_, err = b.conn.WriteTo(bs, &addr)
+				if err != nil {
+					if debug {
+						l.Debugln(err, "on write to", addr)
+					}
+				} else if debug {
+					l.Debugf("sent %d bytes to %s", len(bs), addr.String())
+				}
+			}
+		}
+	}
+}

build.go

@@ -0,0 +1,500 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+var (
+	versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
+	goarch    string
+	goos      string
+	noupgrade bool
+)
+
+const minGoVersion = 1.3
+
+func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(0)
+
+	if os.Getenv("GOPATH") == "" {
+		cwd, err := os.Getwd()
+		if err != nil {
+			log.Fatal(err)
+		}
+		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
+		log.Println("GOPATH is", gopath)
+		os.Setenv("GOPATH", gopath)
+	}
+	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))
+
+	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
+	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
+	flag.BoolVar(&noupgrade, "no-upgrade", false, "Disable upgrade functionality")
+	flag.Parse()
+
+	switch goarch {
+	case "386", "amd64", "armv5", "armv6", "armv7":
+		break
+	case "arm":
+		log.Println("Invalid goarch \"arm\". Use one of \"armv5\", \"armv6\", \"armv7\".")
+		log.Fatalln("Note that producing a correct \"armv5\" binary requires a rebuilt stdlib.")
+	default:
+		log.Printf("Unknown goarch %q; proceed with caution!", goarch)
+	}
+
+	checkRequiredGoVersion()
+
+	if check() != nil {
+		setup()
+	}
+
+	if flag.NArg() == 0 {
+		install("./cmd/...")
+		return
+	}
+
+	switch flag.Arg(0) {
+	case "install":
+		pkg := "./cmd/..."
+		if flag.NArg() > 2 {
+			pkg = flag.Arg(1)
+		}
+		install(pkg)
+
+	case "build":
+		pkg := "./cmd/syncthing"
+		if flag.NArg() > 2 {
+			pkg = flag.Arg(1)
+		}
+		var tags []string
+		if noupgrade {
+			tags = []string{"noupgrade"}
+		}
+		build(pkg, tags)
+
+	case "test":
+		pkg := "./..."
+		if flag.NArg() > 2 {
+			pkg = flag.Arg(1)
+		}
+		test(pkg)
+
+	case "assets":
+		assets()
+
+	case "xdr":
+		xdr()
+
+	case "translate":
+		translate()
+
+	case "transifex":
+		transifex()
+
+	case "deps":
+		deps()
+
+	case "tar":
+		buildTar()
+
+	case "zip":
+		buildZip()
+
+	case "clean":
+		clean()
+
+	default:
+		log.Fatalf("Unknown command %q", flag.Arg(0))
+	}
+}
+
+func check() error {
+	_, err := exec.LookPath("godep")
+	return err
+}
+
+func checkRequiredGoVersion() {
+	ver := run("go", "version")
+	re := regexp.MustCompile(`go version go(\d+\.\d+)`)
+	if m := re.FindSubmatch(ver); len(m) == 2 {
+		vs := string(m[1])
+		// This is a standard go build. Verify that it's new enough.
+		f, err := strconv.ParseFloat(vs, 64)
+		if err != nil {
+			log.Printf("*** Could parse Go version out of %q.\n*** This isn't known to work, proceed on your own risk.", vs)
+			return
+		}
+		if f < minGoVersion {
+			log.Fatalf("*** Go version %.01f is less than required %.01f.\n*** This is known not to work, not proceeding.", f, minGoVersion)
+		}
+	} else {
+		log.Printf("*** Unknown Go version %q.\n*** This isn't known to work, proceed on your own risk.", ver)
+	}
+}
+
+func setup() {
+	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/cover")
+	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/vet")
+	runPrint("go", "get", "-v", "code.google.com/p/go.net/html")
+	runPrint("go", "get", "-v", "github.com/tools/godep")
+}
+
+func test(pkg string) {
+	runPrint("godep", "go", "test", pkg)
+}
+
+func install(pkg string) {
+	os.Setenv("GOBIN", "./bin")
+	setBuildEnv()
+	runPrint("godep", "go", "install", "-ldflags", ldflags(), pkg)
+}
+
+func build(pkg string, tags []string) {
+	rmr("syncthing", "syncthing.exe")
+	args := []string{"go", "build", "-ldflags", ldflags()}
+	if len(tags) > 0 {
+		args = append(args, "-tags", strings.Join(tags, ","))
+	}
+	args = append(args, pkg)
+	setBuildEnv()
+	runPrint("godep", args...)
+}
+
+func buildTar() {
+	name := archiveName()
+	var tags []string
+	if noupgrade {
+		tags = []string{"noupgrade"}
+		name += "-noupgrade"
+	}
+	build("./cmd/syncthing", tags)
+	filename := name + ".tar.gz"
+	tarGz(filename, []archiveFile{
+		{"README.md", name + "/README.txt"},
+		{"LICENSE", name + "/LICENSE.txt"},
+		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
+		{"syncthing", name + "/syncthing"},
+	})
+	log.Println(filename)
+}
+
+func buildZip() {
+	name := archiveName()
+	var tags []string
+	if noupgrade {
+		tags = []string{"noupgrade"}
+		name += "-noupgrade"
+	}
+	build("./cmd/syncthing", tags)
+	filename := name + ".zip"
+	zipFile(filename, []archiveFile{
+		{"README.md", name + "/README.txt"},
+		{"LICENSE", name + "/LICENSE.txt"},
+		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
+		{"syncthing.exe", name + "/syncthing.exe"},
+	})
+	log.Println(filename)
+}
+
+func setBuildEnv() {
+	os.Setenv("GOOS", goos)
+	if strings.HasPrefix(goarch, "arm") {
+		os.Setenv("GOARCH", "arm")
+		os.Setenv("GOARM", goarch[4:])
+	} else {
+		os.Setenv("GOARCH", goarch)
+	}
+	if goarch == "386" {
+		os.Setenv("GO386", "387")
+	}
+}
+
+func assets() {
+	runPipe("auto/gui.files.go", "godep", "go", "run", "cmd/genassets/main.go", "gui")
+}
+
+func xdr() {
+	for _, f := range []string{"discover/packets", "files/leveldb", "protocol/message"} {
+		runPipe(f+"_xdr.go", "go", "run", "./Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go", "--", f+".go")
+	}
+}
+
+func translate() {
+	os.Chdir("gui")
+	runPipe("lang-en-new.json", "go", "run", "../cmd/translate/main.go", "lang-en.json", "index.html")
+	os.Remove("lang-en.json")
+	err := os.Rename("lang-en-new.json", "lang-en.json")
+	if err != nil {
+		log.Fatal(err)
+	}
+	os.Chdir("..")
+}
+
+func transifex() {
+	os.Chdir("gui")
+	runPrint("go", "run", "../cmd/transifexdl/main.go")
+	os.Chdir("..")
+	assets()
+}
+
+func deps() {
+	rmr("Godeps")
+	runPrint("godep", "save", "./cmd/...")
+}
+
+func clean() {
+	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
+	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
+}
+
+func ldflags() string {
+	var b bytes.Buffer
+	b.WriteString("-w")
+	b.WriteString(fmt.Sprintf(" -X main.Version %s", version()))
+	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
+	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
+	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
+	b.WriteString(fmt.Sprintf(" -X main.BuildEnv %s", buildEnvironment()))
+	if strings.HasPrefix(goarch, "arm") {
+		b.WriteString(fmt.Sprintf(" -X main.GoArchExtra %s", goarch[3:]))
+	}
+	return b.String()
+}
+
+func rmr(paths ...string) {
+	for _, path := range paths {
+		log.Println("rm -r", path)
+		os.RemoveAll(path)
+	}
+}
+
+func version() string {
+	v := run("git", "describe", "--always", "--dirty")
+	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
+		s[0] = '+'
+		return s
+	})
+	return string(v)
+}
+
+func buildStamp() int64 {
+	bs := run("git", "show", "-s", "--format=%ct")
+	s, _ := strconv.ParseInt(string(bs), 10, 64)
+	return s
+}
+
+func buildUser() string {
+	u, err := user.Current()
+	if err != nil {
+		return "unknown-user"
+	}
+	return strings.Replace(u.Username, " ", "-", -1)
+}
+
+func buildHost() string {
+	h, err := os.Hostname()
+	if err != nil {
+		return "unknown-host"
+	}
+	return h
+}
+
+func buildEnvironment() string {
+	if v := os.Getenv("ENVIRONMENT"); len(v) > 0 {
+		return v
+	}
+	return "default"
+}
+
+func buildArch() string {
+	os := goos
+	if os == "darwin" {
+		os = "macosx"
+	}
+	return fmt.Sprintf("%s-%s", os, goarch)
+}
+
+func archiveName() string {
+	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version())
+}
+
+func run(cmd string, args ...string) []byte {
+	ecmd := exec.Command(cmd, args...)
+	bs, err := ecmd.CombinedOutput()
+	if err != nil {
+		log.Println(cmd, strings.Join(args, " "))
+		log.Println(string(bs))
+		log.Fatal(err)
+	}
+	return bytes.TrimSpace(bs)
+}
+
+func runPrint(cmd string, args ...string) {
+	log.Println(cmd, strings.Join(args, " "))
+	ecmd := exec.Command(cmd, args...)
+	ecmd.Stdout = os.Stdout
+	ecmd.Stderr = os.Stderr
+	err := ecmd.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func runPipe(file, cmd string, args ...string) {
+	log.Println(cmd, strings.Join(args, " "), ">", file)
+	fd, err := os.Create(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+	ecmd := exec.Command(cmd, args...)
+	ecmd.Stdout = fd
+	ecmd.Stderr = os.Stderr
+	err = ecmd.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fd.Close()
+}
+
+type archiveFile struct {
+	src string
+	dst string
+}
+
+func tarGz(out string, files []archiveFile) {
+	fd, err := os.Create(out)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	gw := gzip.NewWriter(fd)
+	tw := tar.NewWriter(gw)
+
+	for _, f := range files {
+		sf, err := os.Open(f.src)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		info, err := sf.Stat()
+		if err != nil {
+			log.Fatal(err)
+		}
+		h := &tar.Header{
+			Name:    f.dst,
+			Size:    info.Size(),
+			Mode:    int64(info.Mode()),
+			ModTime: info.ModTime(),
+		}
+
+		err = tw.WriteHeader(h)
+		if err != nil {
+			log.Fatal(err)
+		}
+		_, err = io.Copy(tw, sf)
+		if err != nil {
+			log.Fatal(err)
+		}
+		sf.Close()
+	}
+
+	err = tw.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = gw.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = fd.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func zipFile(out string, files []archiveFile) {
+	fd, err := os.Create(out)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	zw := zip.NewWriter(fd)
+
+	for _, f := range files {
+		sf, err := os.Open(f.src)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		info, err := sf.Stat()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fh, err := zip.FileInfoHeader(info)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fh.Name = f.dst
+		fh.Method = zip.Deflate
+
+		if strings.HasSuffix(f.dst, ".txt") {
+			// Text file. Read it and convert line endings.
+			bs, err := ioutil.ReadAll(sf)
+			if err != nil {
+				log.Fatal(err)
+			}
+			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
+			fh.UncompressedSize = uint32(len(bs))
+			fh.UncompressedSize64 = uint64(len(bs))
+
+			of, err := zw.CreateHeader(fh)
+			if err != nil {
+				log.Fatal(err)
+			}
+			of.Write(bs)
+		} else {
+			// Binary file. Copy verbatim.
+			of, err := zw.CreateHeader(fh)
+			if err != nil {
+				log.Fatal(err)
+			}
+			_, err = io.Copy(of, sf)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+	}
+
+	err = zw.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = fd.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

build.sh

@@ -1,236 +1,105 @@
 #!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
 
-export COPYFILE_DISABLE=true
-export GO386=387 # Don't use SSE on 32 bit builds
-
-distFiles=(README.md LICENSE CONTRIBUTORS) # apart from the binary itself
-version=$(git describe --always --dirty)
-date=$(git show -s --format=%ct)
-user=$(whoami)
-host=$(hostname)
-host=${host%%.*}
-bldenv=${ENVIRONMENT:-default}
-ldflags="-w -X main.Version $version -X main.BuildStamp $date -X main.BuildUser $user -X main.BuildHost $host -X main.BuildEnv $bldenv"
-
-check() {
-	if ! command -v godep >/dev/null ; then
-		echo "Error: no godep. Try \"$0 setup\"."
-		exit 1
-	fi
-}
-
-build() {
-	check
-	godep go build $* -ldflags "$ldflags" ./cmd/syncthing
-}
-
-assets() {
-	check
-	godep go run cmd/genassets/main.go gui > auto/gui.files.go
-}
-
-test-cov() {
-	echo "mode: set" > coverage.out
-	fail=0
-
-	for dir in $(go list ./...) ; do
-		godep go test -coverprofile=profile.out $dir || fail=1
-		if [ -f profile.out ] ; then
-			grep -v "mode: set" profile.out >> coverage.out
-			rm profile.out
-        fi
-    done
-
-   exit $fail
-}
-
-test() {
-	check
-	go vet ./...
-	godep go test -cpu=1,2,4 ./...
-}
-
-sign() {
-	if git describe --exact-match 2>/dev/null >/dev/null ; then
-		# HEAD is a tag
-		id=BCE524C7
-		if gpg --list-keys "$id" >/dev/null 2>&1 ; then
-			gpg -ab -u "$id" "$1"
-		fi
-	fi
-}
-
-tarDist() {
-	name="$1"
-	rm -rf "$name"
-	mkdir -p "$name"
-	cp syncthing "${distFiles[@]}" "$name"
-	sign "$name/syncthing"
-	tar zcvf "$name.tar.gz" "$name"
-	rm -rf "$name"
-}
-
-zipDist() {
-	name="$1"
-	rm -rf "$name"
-	mkdir -p "$name"
-	for f in "${distFiles[@]}" ; do
-		GOARCH="" GOOS="" go run cmd/todos/main.go < "$f" > "$name/$f.txt"
-	done
-	cp syncthing.exe "$name"
-	sign "$name/syncthing.exe"
-	zip -r "$name.zip" "$name"
-	rm -rf "$name"
-}
-
-deps() {
-	check
-	godep save ./cmd/...
-}
-
-setup() {
-	go get -v code.google.com/p/go.tools/cmd/cover
-	go get -v code.google.com/p/go.tools/cmd/vet
-	go get -v github.com/mattn/goveralls
-	go get -v github.com/tools/godep
-	GOPATH="$GOPATH:$(godep path)" go get -v -t ./...
-}
-
-xdr() {
-	for f in discover/packets files/leveldb protocol/message ; do
-		go run "$(godep path)/src/github.com/calmh/xdr/cmd/genxdr/main.go" -- "${f}.go" > "${f}_xdr.go"
-	done
-}
-
-translate() {
-	pushd gui
-	go run ../cmd/translate/main.go lang-en.json < index.html > lang-en-new.json
-	mv lang-en-new.json lang-en.json
-	popd
-}
-
-transifex() {
-	pushd gui
-	go run ../cmd/transifexdl/main.go > valid-langs.js
-	popd
-	assets
-}
-
-case "$1" in
-	"")
-		shift
-		export GOBIN=$(pwd)/bin
-		godep go install $* -ldflags "$ldflags" ./cmd/...
+case "${1:-default}" in
+	default)
+		go run build.go
 		;;
 
-	race)
-		build -race
-		;;
-
-	guidev)
-		echo "Syncthing is already built for GUI developments. Try:"
-		echo "    STGUIASSETS=~/someDir/gui syncthing"
+	clean)
+		go run build.go "$1"
 		;;
 
 	test)
-		test
-		;;
+		ulimit -t 60 || true
+		ulimit -d 512000 || true
+		ulimit -m 512000 || true
 
-	test-cov)
-		test-cov
+		go run build.go "$1"
 		;;
 
 	tar)
-		rm -f *.tar.gz *.zip
-		test || exit 1
-		assets
-		build
-
-		eval $(go env)
-		name="syncthing-$GOOS-$GOARCH-$version"
-
-		tarDist "$name"
-		;;
-
-	all)
-		rm -f *.tar.gz *.zip
-		test || exit 1
-		assets
-
-		for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 windows-386 solaris-amd64 ; do
-			export GOOS=${os%-*}
-			export GOARCH=${os#*-}
-
-			build
-
-			name="syncthing-$os-$version"
-			case $GOOS in
-				windows)
-					zipDist "$name"
-					rm -f syncthing.exe
-					;;
-				*)
-					tarDist "$name"
-					rm -f syncthing
-					;;
-			esac
-		done
-
-		export GOOS=linux
-		export GOARCH=arm
-
-		origldflags="$ldflags"
-
-		export GOARM=7
-		ldflags="$origldflags -X main.GoArchExtra v7"
-		build
-		tarDist "syncthing-linux-armv7-$version"
-
-		export GOARM=6
-		ldflags="$origldflags -X main.GoArchExtra v6"
-		build
-		tarDist "syncthing-linux-armv6-$version"
-
-		export GOARM=5
-		ldflags="$origldflags -X main.GoArchExtra v5"
-		build
-		tarDist "syncthing-linux-armv5-$version"
-
-		;;
-
-	upload)
-		tag=$(git describe)
-		shopt -s nullglob
-		for f in *.tar.gz *.zip *.asc ; do
-			relup calmh/syncthing "$tag" "$f"
-		done
+		go run build.go "$1"
 		;;
 
 	deps)
-		deps
+		go run build.go "$1"
 		;;
 
 	assets)
-		assets
-		;;
-
-	setup)
-		setup
+		go run build.go "$1"
 		;;
 
 	xdr)
-		xdr
+		go run build.go "$1"
 		;;
 
 	translate)
-		translate
+		go run build.go "$1"
 		;;
 
 	transifex)
-		transifex
+		go run build.go "$1"
+		;;
+
+	noupgrade)
+		go run build.go -no-upgrade tar
+		;;
+
+	all)
+		go run build.go -goos linux -goarch amd64 tar
+		go run build.go -goos linux -goarch 386 tar
+		go run build.go -goos linux -goarch armv5 tar
+		go run build.go -goos linux -goarch armv6 tar
+		go run build.go -goos linux -goarch armv7 tar
+
+		go run build.go -goos freebsd -goarch amd64 tar
+		go run build.go -goos freebsd -goarch 386 tar
+
+		go run build.go -goos darwin -goarch amd64 tar
+
+		go run build.go -goos windows -goarch amd64 zip
+		go run build.go -goos windows -goarch 386 zip
+		;;
+
+	setup)
+		echo "Don't worry, just build."
+		;;
+
+	test-cov)
+		ulimit -t 60 || true
+		ulimit -d 512000 || true
+		ulimit -m 512000 || true
+
+		go get github.com/axw/gocov/gocov
+		go get github.com/AlekSi/gocov-xml
+
+		echo "mode: set" > coverage.out
+		fail=0
+
+		# For every package in the repo
+		for dir in $(go list ./...) ; do
+			# run the tests
+			godep go test -coverprofile=profile.out $dir
+			if [ -f profile.out ] ; then
+				# and if there was test output, append it to coverage.out
+				grep -v "mode: set" profile.out >> coverage.out
+				rm profile.out
+			fi
+		done
+
+		gocov convert coverage.out | gocov-xml > coverage.xml
+
+		# This is usually run from within Jenkins. If it is, we need to
+		# tweak the paths in coverage.xml so cobertura finds the
+		# source.
+		if [[ "${WORKSPACE:-default}" != "default" ]] ; then
+			sed "s#$WORKSPACE##g" < coverage.xml > coverage.xml.new && mv coverage.xml.new coverage.xml
+		fi
 		;;
 
 	*)
-		echo "Unknown build parameter $1"
+		echo "Unknown build command $1"
 		;;
 esac

check-contrib.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+missing-contribs() {
+	for email in $(git log --format=%ae master | grep -v jakob@nym.se | sort | uniq) ; do
+		grep -q "$email" CONTRIBUTORS || echo $email
+	done
+}
+
+no-docs-typos() {
+	# Commits that are known to not change code
+	grep -v f2459ef3319b2f060dbcdacd0c35a1788a94b8bd |\
+	grep -v b61f418bf2d1f7d5a9d7088a20a2a448e5e66801 |\
+	grep -v f0621207e3953711f9ab86d99724f1d0faac45b1 |\
+	grep -v f1120d7aa936c0658429edef0037792520b46334
+}
+
+for email in $(missing-contribs) ; do
+	git log --author="$email" --format="%H %ae %s" | no-docs-typos
+done
+

cmd/stindex/main.go

@@ -10,8 +10,8 @@ import (
 	"log"
 	"os"
 
-	"github.com/calmh/syncthing/files"
-	"github.com/calmh/syncthing/protocol"
+	"github.com/syncthing/syncthing/files"
+	"github.com/syncthing/syncthing/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -32,7 +32,8 @@ func main() {
 
 	if *node == "" {
 		log.Printf("*** Global index for repo %q", *repo)
-		fs.WithGlobal(func(f protocol.FileInfo) bool {
+		fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
+			f := fi.(protocol.FileInfoTruncated)
 			fmt.Println(f)
 			fmt.Println("\t", fs.Availability(f.Name))
 			return true
@@ -43,7 +44,8 @@ func main() {
 			log.Fatal(err)
 		}
 		log.Printf("*** Have index for repo %q node %q", *repo, n)
-		fs.WithHave(n, func(f protocol.FileInfo) bool {
+		fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool {
+			f := fi.(protocol.FileInfoTruncated)
 			fmt.Println(f)
 			return true
 		})

cmd/syncthing/gui.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -24,15 +25,14 @@ import (
 	"sync"
 	"time"
 
-	"crypto/tls"
 	"code.google.com/p/go.crypto/bcrypt"
-	"github.com/calmh/syncthing/auto"
-	"github.com/calmh/syncthing/config"
-	"github.com/calmh/syncthing/events"
-	"github.com/calmh/syncthing/logger"
-	"github.com/calmh/syncthing/model"
-	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/upgrade"
+	"github.com/syncthing/syncthing/auto"
+	"github.com/syncthing/syncthing/config"
+	"github.com/syncthing/syncthing/events"
+	"github.com/syncthing/syncthing/logger"
+	"github.com/syncthing/syncthing/model"
+	"github.com/syncthing/syncthing/protocol"
+	"github.com/syncthing/syncthing/upgrade"
 	"github.com/vitrun/qart/qr"
 )
 
@@ -57,7 +57,7 @@ const (
 
 func init() {
 	l.AddHandler(logger.LevelWarn, showGuiError)
-	sub := events.Default.Subscribe(^events.EventType(events.ItemStarted | events.ItemCompleted))
+	sub := events.Default.Subscribe(events.AllEvents)
 	eventSub = events.NewBufferedSubscription(sub, 1000)
 }
 
@@ -111,6 +111,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	getRestMux.HandleFunc("/rest/system", restGetSystem)
 	getRestMux.HandleFunc("/rest/upgrade", restGetUpgrade)
 	getRestMux.HandleFunc("/rest/version", restGetVersion)
+	getRestMux.HandleFunc("/rest/stats/node", withModel(m, restGetNodeStats))
 
 	// Debug endpoints, not for general use
 	getRestMux.HandleFunc("/rest/debug/peerCompletion", withModel(m, restGetPeerCompletion))
@@ -126,6 +127,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	postRestMux.HandleFunc("/rest/restart", restPostRestart)
 	postRestMux.HandleFunc("/rest/shutdown", restPostShutdown)
 	postRestMux.HandleFunc("/rest/upgrade", restPostUpgrade)
+	postRestMux.HandleFunc("/rest/scan", withModel(m, restPostScan))
 
 	// A handler that splits requests between the two above and disables
 	// caching
@@ -143,6 +145,9 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	// protected, other requests will grant cookies.
 	handler := csrfMiddleware("/rest", mux)
 
+	// Add our version as a header to responses
+	handler = withVersionMiddleware(handler)
+
 	// Wrap everything in basic auth, if user/password is set.
 	if len(cfg.User) > 0 {
 		handler = basicAuthMiddleware(cfg.User, cfg.Password, handler)
@@ -172,6 +177,13 @@ func noCacheMiddleware(h http.Handler) http.Handler {
 	})
 }
 
+func withVersionMiddleware(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Syncthing-Version", Version)
+		h.ServeHTTP(w, r)
+	})
+}
+
 func withModel(m *model.Model, h func(m *model.Model, w http.ResponseWriter, r *http.Request)) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		h(m, w, r)
@@ -264,6 +276,12 @@ func restGetConnections(m *model.Model, w http.ResponseWriter, r *http.Request)
 	json.NewEncoder(w).Encode(res)
 }
 
+func restGetNodeStats(m *model.Model, w http.ResponseWriter, r *http.Request) {
+	var res = m.NodeStatistics()
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(res)
+}
+
 func restGetConfig(w http.ResponseWriter, r *http.Request) {
 	encCfg := cfg
 	if encCfg.GUI.Password != "" {
@@ -277,7 +295,9 @@ func restPostConfig(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var newCfg config.Configuration
 	err := json.NewDecoder(r.Body).Decode(&newCfg)
 	if err != nil {
-		l.Warnln(err)
+		l.Warnln("decoding posted config:", err)
+		http.Error(w, err.Error(), 500)
+		return
 	} else {
 		if newCfg.GUI.Password == "" {
 			// Leave it empty
@@ -286,7 +306,9 @@ func restPostConfig(m *model.Model, w http.ResponseWriter, r *http.Request) {
 		} else {
 			hash, err := bcrypt.GenerateFromPassword([]byte(newCfg.GUI.Password), 0)
 			if err != nil {
-				l.Warnln(err)
+				l.Warnln("bcrypting password:", err)
+				http.Error(w, err.Error(), 500)
+				return
 			} else {
 				newCfg.GUI.Password = string(hash)
 			}
@@ -384,7 +406,7 @@ func restGetSystem(w http.ResponseWriter, r *http.Request) {
 	res["myID"] = myID.String()
 	res["goroutines"] = runtime.NumGoroutine()
 	res["alloc"] = m.Alloc
-	res["sys"] = m.Sys
+	res["sys"] = m.Sys - m.HeapReleased
 	res["tilde"] = expandTilde("~")
 	if cfg.Options.GlobalAnnEnabled && discoverer != nil {
 		res["extAnnounceOK"] = discoverer.ExtAnnounceOK()
@@ -454,12 +476,18 @@ func restGetEvents(w http.ResponseWriter, r *http.Request) {
 	since, _ := strconv.Atoi(sinceStr)
 	limit, _ := strconv.Atoi(limitStr)
 
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+	// Flush before blocking, to indicate that we've received the request
+	// and that it should not be retried.
+	f := w.(http.Flusher)
+	f.Flush()
+
 	evs := eventSub.Since(since, nil)
 	if 0 < limit && limit < len(evs) {
 		evs = evs[len(evs)-limit:]
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(evs)
 }
 
@@ -498,25 +526,25 @@ func restGetLang(w http.ResponseWriter, r *http.Request) {
 	lang := r.Header.Get("Accept-Language")
 	var langs []string
 	for _, l := range strings.Split(lang, ",") {
-		if len(l) >= 2 {
-			langs = append(langs, l[:2])
-		}
+		parts := strings.SplitN(l, ";", 2)
+		langs = append(langs, strings.ToLower(strings.TrimSpace(parts[0])))
 	}
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(langs)
 }
 
 func restPostUpgrade(w http.ResponseWriter, r *http.Request) {
 	rel, err := upgrade.LatestRelease(strings.Contains(Version, "-beta"))
 	if err != nil {
-		l.Warnln(err)
+		l.Warnln("getting latest release:", err)
 		http.Error(w, err.Error(), 500)
 		return
 	}
 
 	if upgrade.CompareVersions(rel.Tag, Version) == 1 {
-		err = upgrade.UpgradeTo(rel)
+		err = upgrade.UpgradeTo(rel, GoArchExtra)
 		if err != nil {
-			l.Warnln(err)
+			l.Warnln("upgrading:", err)
 			http.Error(w, err.Error(), 500)
 			return
 		}
@@ -525,9 +553,19 @@ func restPostUpgrade(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func restPostScan(m *model.Model, w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+	repo := qs.Get("repo")
+	sub := qs.Get("sub")
+	err := m.ScanRepoSub(repo, sub)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+	}
+}
+
 func getQR(w http.ResponseWriter, r *http.Request) {
-	r.ParseForm()
-	text := r.FormValue("text")
+	var qs = r.URL.Query()
+	var text = qs.Get("text")
 	code, err := qr.Encode(text, qr.M)
 	if err != nil {
 		http.Error(w, "Invalid", 500)

cmd/syncthing/gui_csrf.go

@@ -12,7 +12,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/calmh/syncthing/osutil"
+	"github.com/syncthing/syncthing/osutil"
 )
 
 var csrfTokens []string
@@ -43,6 +43,12 @@ func csrfMiddleware(prefix string, next http.Handler) http.Handler {
 			return
 		}
 
+		if r.Method == "GET" {
+			// Allow GET requests unconditionally
+			next.ServeHTTP(w, r)
+			return
+		}
+
 		// Verify the CSRF token
 		token := r.Header.Get("X-CSRF-Token")
 		if !validCsrfToken(token) {

cmd/syncthing/gui_solaris.go

@@ -79,7 +79,7 @@ func trackCPUUsage() {
 	for _ = range time.NewTicker(time.Second).C {
 		err := solarisPrusage(pid, &rusage)
 		if err != nil {
-			l.Warnln(err)
+			l.Warnln("getting prusage:", err)
 			continue
 		}
 		curTime := time.Now().UnixNano()

cmd/syncthing/heapprof.go

@@ -0,0 +1,46 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"syscall"
+	"time"
+)
+
+func init() {
+	if os.Getenv("STHEAPPROFILE") != "" {
+		go saveHeapProfiles()
+	}
+}
+
+func saveHeapProfiles() {
+	runtime.MemProfileRate = 1
+	var memstats, prevMemstats runtime.MemStats
+
+	t0 := time.Now()
+	for t := range time.NewTicker(250 * time.Millisecond).C {
+		startms := int(t.Sub(t0).Seconds() * 1000)
+		runtime.ReadMemStats(&memstats)
+		if memstats.HeapInuse > prevMemstats.HeapInuse {
+			fd, err := os.Create(fmt.Sprintf("heap-%05d-%07d.pprof", syscall.Getpid(), startms))
+			if err != nil {
+				panic(err)
+			}
+			err = pprof.WriteHeapProfile(fd)
+			if err != nil {
+				panic(err)
+			}
+			err = fd.Close()
+			if err != nil {
+				panic(err)
+			}
+			prevMemstats = memstats
+		}
+	}
+}

cmd/syncthing/main.go

@@ -26,16 +26,18 @@ import (
 	"strings"
 	"time"
 
-	"github.com/calmh/syncthing/config"
-	"github.com/calmh/syncthing/discover"
-	"github.com/calmh/syncthing/events"
-	"github.com/calmh/syncthing/logger"
-	"github.com/calmh/syncthing/model"
-	"github.com/calmh/syncthing/osutil"
-	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/upgrade"
-	"github.com/calmh/syncthing/upnp"
+	"code.google.com/p/go.crypto/bcrypt"
 	"github.com/juju/ratelimit"
+	"github.com/syncthing/syncthing/config"
+	"github.com/syncthing/syncthing/discover"
+	"github.com/syncthing/syncthing/events"
+	"github.com/syncthing/syncthing/files"
+	"github.com/syncthing/syncthing/logger"
+	"github.com/syncthing/syncthing/model"
+	"github.com/syncthing/syncthing/osutil"
+	"github.com/syncthing/syncthing/protocol"
+	"github.com/syncthing/syncthing/upgrade"
+	"github.com/syncthing/syncthing/upnp"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -47,6 +49,7 @@ var (
 	BuildHost   = "unknown"
 	BuildUser   = "unknown"
 	LongVersion string
+	GoArchExtra string // "", "v5", "v6", "v7"
 )
 
 var l = logger.DefaultLogger
@@ -54,7 +57,7 @@ var l = logger.DefaultLogger
 func init() {
 	if Version != "unknown-dev" {
 		// If not a generic dev build, version string should come from git describe
-		exp := regexp.MustCompile(`^v\d+\.\d+\.\d+(-beta\d+)?(-\d+-g[0-9a-f]+)?(-dirty)?$`)
+		exp := regexp.MustCompile(`^v\d+\.\d+\.\d+(-[a-z0-9]+)*(\+\d+-g[0-9a-f]+)?(-dirty)?$`)
 		if !exp.MatchString(Version) {
 			l.Fatalf("Invalid version string %q;\n\tdoes not match regexp %v", Version, exp)
 		}
@@ -72,15 +75,17 @@ func init() {
 }
 
 var (
-	cfg        config.Configuration
-	myID       protocol.NodeID
-	confDir    string
-	logFlags   int = log.Ltime
-	rateBucket *ratelimit.Bucket
-	stop       = make(chan bool)
-	discoverer *discover.Discoverer
-	lockConn   *net.TCPListener
-	lockPort   int
+	cfg          config.Configuration
+	myID         protocol.NodeID
+	confDir      string
+	logFlags     int = log.Ltime
+	rateBucket   *ratelimit.Bucket
+	stop         = make(chan bool)
+	discoverer   *discover.Discoverer
+	lockConn     *net.TCPListener
+	lockPort     int
+	externalPort int
+	cert         tls.Certificate
 )
 
 const (
@@ -99,13 +104,19 @@ show time only (2).
 
 The following enviroment variables are interpreted by syncthing:
 
+ STGUIADDRESS  Override GUI listen address set in config. Expects protocol type
+               followed by hostname or an IP address, followed by a port, such
+               as "https://127.0.0.1:8888".
+
+ STGUIAUTH     Override GUI authentication credentials set in config. Expects
+               a colon separated username and password, such as "admin:secret".
+
+ STGUIAPIKEY   Override GUI API key set in config.
+
  STNORESTART   Do not attempt to restart when requested to, instead just exit.
                Set this variable when running under a service manager such as
                runit, launchd, etc.
 
- STPROFILER    Set to a listen address such as "127.0.0.1:9090" to start the
-               profiler with HTTP access.
-
  STTRACE       A comma separated string of facilities to trace. The valid
                facility strings:
                - "beacon"   (the beacon package)
@@ -115,15 +126,26 @@ The following enviroment variables are interpreted by syncthing:
                - "net"      (the main package; connections & network messages)
                - "model"    (the model package)
                - "scanner"  (the scanner package)
+               - "stats"    (the stats package)
                - "upnp"     (the upnp package)
                - "xdr"      (the xdr package)
                - "all"      (all of the above)
 
- STCPUPROFILE  Write CPU profile to the specified file.
-
  STGUIASSETS   Directory to load GUI assets from. Overrides compiled in assets.
 
- STDEADLOCKTIMEOUT  Alter deadlock detection timeout (seconds; default 1200).`
+ STPROFILER    Set to a listen address such as "127.0.0.1:9090" to start the
+               profiler with HTTP access.
+
+ STCPUPROFILE  Write a CPU profile to cpu-$pid.pprof on exit.
+
+ STHEAPPROFILE Write heap profiles to heap-$pid-$timestamp.pprof each time
+               heap usage increases.
+
+ STPERFSTATS   Write running performance statistics to perf-$pid.csv. Not
+               supported on Windows.
+
+ GOMAXPROCS    Set the maximum number of CPU cores to use. Defaults to all
+               available CPU cores.`
 )
 
 func init() {
@@ -135,11 +157,21 @@ func main() {
 	var showVersion bool
 	var doUpgrade bool
 	var doUpgradeCheck bool
+	var noBrowser bool
+	var generateDir string
+	var guiAddress string
+	var guiAuthentication string
+	var guiAPIKey string
 	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
 	flag.BoolVar(&reset, "reset", false, "Prepare to resync from cluster")
 	flag.BoolVar(&showVersion, "version", false, "Show version")
 	flag.BoolVar(&doUpgrade, "upgrade", false, "Perform upgrade")
 	flag.BoolVar(&doUpgradeCheck, "upgrade-check", false, "Check for available upgrade")
+	flag.BoolVar(&noBrowser, "no-browser", false, "Do not start browser")
+	flag.StringVar(&generateDir, "generate", "", "Generate key in specified dir")
+	flag.StringVar(&guiAddress, "gui-address", "", "Override GUI address")
+	flag.StringVar(&guiAuthentication, "gui-authentication", "", "Override GUI authentication. Expects 'username:password'")
+	flag.StringVar(&guiAPIKey, "gui-apikey", "", "Override GUI API key")
 	flag.IntVar(&logFlags, "logflags", logFlags, "Set log flags")
 	flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
 	flag.Parse()
@@ -151,10 +183,29 @@ func main() {
 
 	l.SetFlags(logFlags)
 
-	var err error
-	lockPort, err = getLockPort()
-	if err != nil {
-		l.Fatalln("Opening lock port:", err)
+	if generateDir != "" {
+		dir := expandTilde(generateDir)
+
+		info, err := os.Stat(dir)
+		l.FatalErr(err)
+		if !info.IsDir() {
+			l.Fatalln(dir, "is not a directory")
+		}
+
+		cert, err := loadCert(dir, "")
+		if err == nil {
+			l.Warnln("Key exists; will not overwrite.")
+			l.Infoln("Node ID:", protocol.NewNodeID(cert.Certificate[0]))
+			return
+		}
+
+		newCertificate(dir, "")
+		cert, err = loadCert(dir, "")
+		l.FatalErr(err)
+		if err == nil {
+			l.Infoln("Node ID:", protocol.NewNodeID(cert.Certificate[0]))
+		}
+		return
 	}
 
 	if doUpgrade || doUpgradeCheck {
@@ -171,7 +222,7 @@ func main() {
 		l.Infof("Upgrade available (current %q < latest %q)", Version, rel.Tag)
 
 		if doUpgrade {
-			err = upgrade.UpgradeTo(rel)
+			err = upgrade.UpgradeTo(rel, GoArchExtra)
 			if err != nil {
 				l.Fatalln("Upgrade:", err) // exits 1
 			}
@@ -182,6 +233,12 @@ func main() {
 		}
 	}
 
+	var err error
+	lockPort, err = getLockPort()
+	if err != nil {
+		l.Fatalln("Opening lock port:", err)
+	}
+
 	if len(os.Getenv("GOGC")) == 0 {
 		debug.SetGCPercent(25)
 	}
@@ -218,7 +275,7 @@ func main() {
 	// Ensure that our home directory exists and that we have a certificate and key.
 
 	ensureDir(confDir, 0700)
-	cert, err := loadCert(confDir, "")
+	cert, err = loadCert(confDir, "")
 	if err != nil {
 		newCertificate(confDir, "")
 		cert, err = loadCert(confDir, "")
@@ -236,6 +293,8 @@ func main() {
 	cfgFile := filepath.Join(confDir, "config.xml")
 	go saveConfigLoop(cfgFile)
 
+	var myName string
+
 	// Load the configuration file, if it exists.
 	// If it does not, create a template.
 
@@ -247,25 +306,31 @@ func main() {
 			l.Fatalln(err)
 		}
 		cf.Close()
+		myCfg := cfg.GetNodeConfiguration(myID)
+		if myCfg == nil || myCfg.Name == "" {
+			myName, _ = os.Hostname()
+		} else {
+			myName = myCfg.Name
+		}
 	} else {
 		l.Infoln("No config file; starting with empty defaults")
-		name, _ := os.Hostname()
+		myName, _ = os.Hostname()
 		defaultRepo := filepath.Join(getHomeDir(), "Sync")
-		ensureDir(defaultRepo, 0755)
 
 		cfg, err = config.Load(nil, myID)
 		cfg.Repositories = []config.RepositoryConfiguration{
 			{
-				ID:        "default",
-				Directory: defaultRepo,
-				Nodes:     []config.NodeConfiguration{{NodeID: myID}},
+				ID:              "default",
+				Directory:       defaultRepo,
+				RescanIntervalS: 60,
+				Nodes:           []config.RepositoryNodeConfiguration{{NodeID: myID}},
 			},
 		}
 		cfg.Nodes = []config.NodeConfiguration{
 			{
 				NodeID:    myID,
 				Addresses: []string{"dynamic"},
-				Name:      name,
+				Name:      myName,
 			},
 		}
 
@@ -326,9 +391,19 @@ func main() {
 
 	db, err := leveldb.OpenFile(filepath.Join(confDir, "index"), nil)
 	if err != nil {
-		l.Fatalln("leveldb.OpenFile():", err)
+		l.Fatalln("Cannot open database:", err, "- Is another copy of Syncthing already running?")
 	}
-	m := model.NewModel(confDir, &cfg, "syncthing", Version, db)
+
+	// Remove database entries for repos that no longer exist in the config
+	repoMap := cfg.RepoMap()
+	for _, repo := range files.ListRepos(db) {
+		if _, ok := repoMap[repo]; !ok {
+			l.Infof("Cleaning data for dropped repo %q", repo)
+			files.DropRepo(db, repo)
+		}
+	}
+
+	m := model.NewModel(confDir, &cfg, myName, "syncthing", Version, db)
 
 nextRepo:
 	for i, repo := range cfg.Repositories {
@@ -365,10 +440,13 @@ nextRepo:
 	}
 
 	// GUI
-	if cfg.GUI.Enabled && cfg.GUI.Address != "" {
-		addr, err := net.ResolveTCPAddr("tcp", cfg.GUI.Address)
+
+	guiCfg := overrideGUIConfig(cfg.GUI, guiAddress, guiAuthentication, guiAPIKey)
+
+	if guiCfg.Enabled && guiCfg.Address != "" {
+		addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address)
 		if err != nil {
-			l.Fatalf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
+			l.Fatalf("Cannot start GUI on %q: %v", guiCfg.Address, err)
 		} else {
 			var hostOpen, hostShow string
 			switch {
@@ -384,21 +462,30 @@ nextRepo:
 			}
 
 			var proto = "http"
-			if cfg.GUI.UseTLS {
+			if guiCfg.UseTLS {
 				proto = "https"
 			}
 
 			l.Infof("Starting web GUI on %s://%s:%d/", proto, hostShow, addr.Port)
-			err := startGUI(cfg.GUI, os.Getenv("STGUIASSETS"), m)
+			err := startGUI(guiCfg, os.Getenv("STGUIASSETS"), m)
 			if err != nil {
 				l.Fatalln("Cannot start GUI:", err)
 			}
-			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
+			if !noBrowser && cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
 				openURL(fmt.Sprintf("%s://%s:%d", proto, hostOpen, addr.Port))
 			}
 		}
 	}
 
+	// Clear out old indexes for other nodes. Otherwise we'll start up and
+	// start needing a bunch of files which are nowhere to be found. This
+	// needs to be changed when we correctly do persistent indexes.
+	for _, repoCfg := range cfg.Repositories {
+		for _, node := range repoCfg.NodeIDs() {
+			m.Index(node, repoCfg.ID, nil)
+		}
+	}
+
 	// Walk the repository and update the local model before establishing any
 	// connections to other nodes.
 
@@ -431,11 +518,8 @@ nextRepo:
 
 	// UPnP
 
-	var externalPort = 0
 	if cfg.Options.UPnPEnabled {
-		// We seed the random number generator with the node ID to get a
-		// repeatable sequence of random external ports.
-		externalPort = setupUPnP(rand.NewSource(certSeed(cert.Certificate[0])))
+		setupUPnP()
 	}
 
 	// Routine to connect out to configured nodes
@@ -459,7 +543,7 @@ nextRepo:
 	}
 
 	if cpuprof := os.Getenv("STCPUPROFILE"); len(cpuprof) > 0 {
-		f, err := os.Create(cpuprof)
+		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -488,6 +572,8 @@ nextRepo:
 		}()
 	}
 
+	go standbyMonitor()
+
 	events.Default.Log(events.StartupComplete, nil)
 	go generateEvents()
 
@@ -522,8 +608,7 @@ func waitForParentExit() {
 	l.Infoln("Continuing")
 }
 
-func setupUPnP(r rand.Source) int {
-	var externalPort = 0
+func setupUPnP() {
 	if len(cfg.Options.ListenAddress) == 1 {
 		_, portStr, err := net.SplitHostPort(cfg.Options.ListenAddress[0])
 		if err != nil {
@@ -533,17 +618,11 @@ func setupUPnP(r rand.Source) int {
 			port, _ := strconv.Atoi(portStr)
 			igd, err := upnp.Discover()
 			if err == nil {
-				for i := 0; i < 10; i++ {
-					r := 1024 + int(r.Int63()%(65535-1024))
-					err := igd.AddPortMapping(upnp.TCP, r, port, "syncthing", 0)
-					if err == nil {
-						externalPort = r
-						l.Infoln("Created UPnP port mapping - external port", externalPort)
-						break
-					}
-				}
+				externalPort = setupExternalPort(igd, port)
 				if externalPort == 0 {
 					l.Warnln("Failed to create UPnP port mapping")
+				} else {
+					l.Infoln("Created UPnP port mapping - external port", externalPort)
 				}
 			} else {
 				l.Infof("No UPnP gateway detected")
@@ -551,11 +630,57 @@ func setupUPnP(r rand.Source) int {
 					l.Debugf("UPnP: %v", err)
 				}
 			}
+			if cfg.Options.UPnPRenewal > 0 {
+				go renewUPnP(port)
+			}
 		}
 	} else {
 		l.Warnln("Multiple listening addresses; not attempting UPnP port mapping")
 	}
-	return externalPort
+}
+
+func setupExternalPort(igd *upnp.IGD, port int) int {
+	// We seed the random number generator with the node ID to get a
+	// repeatable sequence of random external ports.
+	rnd := rand.NewSource(certSeed(cert.Certificate[0]))
+	for i := 0; i < 10; i++ {
+		r := 1024 + int(rnd.Int63()%(65535-1024))
+		err := igd.AddPortMapping(upnp.TCP, r, port, "syncthing", cfg.Options.UPnPLease*60)
+		if err == nil {
+			return r
+		}
+	}
+	return 0
+}
+
+func renewUPnP(port int) {
+	for {
+		time.Sleep(time.Duration(cfg.Options.UPnPRenewal) * time.Minute)
+
+		igd, err := upnp.Discover()
+		if err != nil {
+			continue
+		}
+
+		// Just renew the same port that we already have
+		err = igd.AddPortMapping(upnp.TCP, externalPort, port, "syncthing", cfg.Options.UPnPLease*60)
+		if err == nil {
+			l.Infoln("Renewed UPnP port mapping - external port", externalPort)
+			continue
+		}
+
+		// Something strange has happened. Perhaps the gateway has changed?
+		// Retry the same port sequence from the beginning.
+		r := setupExternalPort(igd, port)
+		if r != 0 {
+			externalPort = r
+			l.Infoln("Updated UPnP port mapping - external port", externalPort)
+			discoverer.StopGlobal()
+			discoverer.StartGlobal(cfg.Options.GlobalAnnServer, uint16(r))
+			continue
+		}
+		l.Warnln("Failed to update UPnP port mapping - external port", externalPort)
+	}
 }
 
 func resetRepositories() {
@@ -756,6 +881,10 @@ next:
 			}
 		}
 
+		events.Default.Log(events.NodeRejected, map[string]string{
+			"node":    remoteID.String(),
+			"address": conn.RemoteAddr().String(),
+		})
 		l.Infof("Connection from %s with unknown node ID %s; ignoring", conn.RemoteAddr(), remoteID)
 		conn.Close()
 	}
@@ -895,19 +1024,15 @@ func setTCPOptions(conn *net.TCPConn) {
 }
 
 func discovery(extPort int) *discover.Discoverer {
-	disc, err := discover.NewDiscoverer(myID, cfg.Options.ListenAddress, cfg.Options.LocalAnnPort)
-	if err != nil {
-		l.Warnf("No discovery possible (%v)", err)
-		return nil
-	}
+	disc := discover.NewDiscoverer(myID, cfg.Options.ListenAddress)
 
 	if cfg.Options.LocalAnnEnabled {
-		l.Infoln("Sending local discovery announcements")
-		disc.StartLocal()
+		l.Infoln("Starting local discovery announcements")
+		disc.StartLocal(cfg.Options.LocalAnnPort, cfg.Options.LocalAnnMCAddr)
 	}
 
 	if cfg.Options.GlobalAnnEnabled {
-		l.Infoln("Sending global discovery announcements")
+		l.Infoln("Starting global discovery announcements")
 		disc.StartGlobal(cfg.Options.GlobalAnnServer, uint16(extPort))
 	}
 
@@ -1005,3 +1130,64 @@ func getLockPort() (int, error) {
 	addr := lockConn.Addr().(*net.TCPAddr)
 	return addr.Port, nil
 }
+
+func overrideGUIConfig(originalCfg config.GUIConfiguration, address, authentication, apikey string) config.GUIConfiguration {
+	// Make a copy of the config
+	cfg := originalCfg
+
+	if address == "" {
+		address = os.Getenv("STGUIADDRESS")
+	}
+
+	if address != "" {
+		cfg.Enabled = true
+
+		addressParts := strings.SplitN(address, "://", 2)
+		switch addressParts[0] {
+		case "http":
+			cfg.UseTLS = false
+		case "https":
+			cfg.UseTLS = true
+		default:
+			l.Fatalln("Unidentified protocol", addressParts[0])
+		}
+		cfg.Address = addressParts[1]
+	}
+
+	if authentication == "" {
+		authentication = os.Getenv("STGUIAUTH")
+	}
+
+	if authentication != "" {
+		authenticationParts := strings.SplitN(authentication, ":", 2)
+
+		hash, err := bcrypt.GenerateFromPassword([]byte(authenticationParts[1]), 0)
+		if err != nil {
+			l.Fatalln("Invalid GUI password:", err)
+		}
+
+		cfg.User = authenticationParts[0]
+		cfg.Password = string(hash)
+	}
+
+	if apikey == "" {
+		apikey = os.Getenv("STGUIAPIKEY")
+	}
+
+	if apikey != "" {
+		cfg.APIKey = apikey
+	}
+	return cfg
+}
+
+func standbyMonitor() {
+	now := time.Now()
+	for {
+		time.Sleep(10 * time.Second)
+		if time.Since(now) > 2*time.Minute {
+			l.Infoln("Paused state detected, possibly woke up from standby.")
+			restart()
+		}
+		now = time.Now()
+	}
+}

cmd/syncthing/main_test.go

@@ -0,0 +1,7 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package main_test
+
+// Empty test file to generate 0% coverage rather than no coverage

cmd/syncthing/perfstats_unix.go

@@ -1,4 +1,8 @@
-// +build perfstats
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !windows
 
 package main
 
@@ -11,7 +15,9 @@ import (
 )
 
 func init() {
-	go savePerfStats(fmt.Sprintf("perfstats-%d.csv", syscall.Getpid()))
+	if os.Getenv("STPERFSTATS") != "" {
+		go savePerfStats(fmt.Sprintf("perfstats-%d.csv", syscall.Getpid()))
+	}
 }
 
 func savePerfStats(file string) {
@@ -40,6 +46,6 @@ func savePerfStats(file string) {
 
 		startms := int(t.Sub(t0).Seconds() * 1000)
 
-		fmt.Fprintf(fd, "%d\t%f\t%d\t%d\n", startms, cpuUsagePercent, memstats.Alloc, memstats.Sys)
+		fmt.Fprintf(fd, "%d\t%f\t%d\t%d\n", startms, cpuUsagePercent, memstats.Alloc, memstats.Sys-memstats.HeapReleased)
 	}
 }

cmd/syncthing/tls.go

@@ -39,7 +39,7 @@ func certSeed(bs []byte) int64 {
 }
 
 func newCertificate(dir string, prefix string) {
-	l.Infoln("Generating RSA certificate and key...")
+	l.Infoln("Generating RSA key and certificate...")
 
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
 	l.FatalErr(err)
@@ -67,11 +67,9 @@ func newCertificate(dir string, prefix string) {
 	l.FatalErr(err)
 	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
 	certOut.Close()
-	l.Okln("Created RSA certificate file")
 
 	keyOut, err := os.OpenFile(filepath.Join(dir, prefix+"key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	l.FatalErr(err)
 	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
 	keyOut.Close()
-	l.Okln("Created RSA key file")
 }

cmd/syncthing/usage_report.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/calmh/syncthing/model"
+	"github.com/syncthing/syncthing/model"
 )
 
 // Current version number of the usage report, for acceptance purposes. If
@@ -51,7 +51,7 @@ func reportData(m *model.Model) map[string]interface{} {
 
 	var mem runtime.MemStats
 	runtime.ReadMemStats(&mem)
-	res["memoryUsageMiB"] = mem.Sys / 1024 / 1024
+	res["memoryUsageMiB"] = (mem.Sys - mem.HeapReleased) / 1024 / 1024
 
 	var perf float64
 	for i := 0; i < 5; i++ {

cmd/transifexdl/main.go

@@ -9,10 +9,13 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
+	"regexp"
 	"sort"
+	"strings"
 )
 
 type stat struct {
@@ -31,6 +34,12 @@ func main() {
 		log.Fatal("Need environment variables TRANSIFEX_USER and TRANSIFEX_PASS")
 	}
 
+	curValidLangs := map[string]bool{}
+	for _, lang := range loadValidLangs() {
+		curValidLangs[lang] = true
+	}
+	log.Println(curValidLangs)
+
 	resp := req("https://www.transifex.com/api/2/project/syncthing/resource/gui/stats")
 
 	var stats map[string]stat
@@ -42,19 +51,21 @@ func main() {
 
 	var langs []string
 	for code, stat := range stats {
-		shortCode := code[:2]
-		if pct := 100 * stat.Translated / (stat.Translated + stat.Untranslated); pct < 95 {
-			log.Printf("Skipping language %q (too low completion ratio %d%%)", shortCode, pct)
-			os.Remove("lang-" + shortCode + ".json")
+		code = strings.Replace(code, "_", "-", 1)
+		if !curValidLangs[code] {
+			if pct := 100 * stat.Translated / (stat.Translated + stat.Untranslated); pct < 95 {
+				log.Printf("Skipping language %q (too low completion ratio %d%%)", code, pct)
+				os.Remove("lang-" + code + ".json")
+				continue
+			}
+		}
+
+		langs = append(langs, code)
+		if code == "en" {
 			continue
 		}
 
-		langs = append(langs, shortCode)
-		if shortCode == "en" {
-			continue
-		}
-
-		log.Printf("Updating language %q", shortCode)
+		log.Printf("Updating language %q", code)
 
 		resp := req("https://www.transifex.com/api/2/project/syncthing/resource/gui/translation/" + code)
 		var t translation
@@ -64,7 +75,7 @@ func main() {
 		}
 		resp.Body.Close()
 
-		fd, err := os.Create("lang-" + shortCode + ".json")
+		fd, err := os.Create("lang-" + code + ".json")
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -72,9 +83,18 @@ func main() {
 		fd.Close()
 	}
 
+	saveValidLangs(langs)
+}
+
+func saveValidLangs(langs []string) {
 	sort.Strings(langs)
-	fmt.Print("var validLangs = ")
-	json.NewEncoder(os.Stdout).Encode(langs)
+	fd, err := os.Create("valid-langs.js")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Fprint(fd, "var validLangs = ")
+	json.NewEncoder(fd).Encode(langs)
+	fd.Close()
 }
 
 func userPass() (string, string) {
@@ -97,3 +117,27 @@ func req(url string) *http.Response {
 	}
 	return resp
 }
+
+func loadValidLangs() []string {
+	fd, err := os.Open("valid-langs.js")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fd.Close()
+	bs, err := ioutil.ReadAll(fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var langs []string
+	exp := regexp.MustCompile(`\[([a-zA-Z",-]+)\]`)
+	if matches := exp.FindSubmatch(bs); len(matches) == 2 {
+		langs = strings.Split(string(matches[1]), ",")
+		for i := range langs {
+			// Remove quotes
+			langs[i] = langs[i][1 : len(langs[i])-1]
+		}
+	}
+
+	return langs
+}

cmd/translate/main.go

@@ -10,12 +10,14 @@ import (
 	"encoding/json"
 	"log"
 	"os"
+	"regexp"
 	"strings"
 
 	"code.google.com/p/go.net/html"
 )
 
 var trans = make(map[string]string)
+var attrRe = regexp.MustCompile(`\{\{'([^']+)'\s+\|\s+translate\}\}`)
 
 func generalNode(n *html.Node) {
 	translate := false
@@ -24,6 +26,10 @@ func generalNode(n *html.Node) {
 			if a.Key == "translate" {
 				translate = true
 				break
+			} else {
+				if matches := attrRe.FindStringSubmatch(a.Val); len(matches) == 2 {
+					translation(matches[1])
+				}
 			}
 		}
 	} else if n.Type == html.TextNode {
@@ -44,12 +50,7 @@ func generalNode(n *html.Node) {
 
 func inTranslate(n *html.Node) {
 	if n.Type == html.TextNode {
-		v := strings.TrimSpace(n.Data)
-		if _, ok := trans[v]; !ok {
-			av := strings.Replace(v, "{%", "{{", -1)
-			av = strings.Replace(av, "%}", "}}", -1)
-			trans[v] = av
-		}
+		translation(n.Data)
 	} else {
 		log.Println("translate node with non-text child <")
 		log.Println(n)
@@ -60,6 +61,15 @@ func inTranslate(n *html.Node) {
 	}
 }
 
+func translation(v string) {
+	v = strings.TrimSpace(v)
+	if _, ok := trans[v]; !ok {
+		av := strings.Replace(v, "{%", "{{", -1)
+		av = strings.Replace(av, "%}", "}}", -1)
+		trans[v] = av
+	}
+}
+
 func main() {
 	fd, err := os.Open(os.Args[1])
 	if err != nil {
@@ -71,10 +81,16 @@ func main() {
 	}
 	fd.Close()
 
-	doc, err := html.Parse(os.Stdin)
+	fd, err = os.Open(os.Args[2])
 	if err != nil {
 		log.Fatal(err)
 	}
+	doc, err := html.Parse(fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fd.Close()
+
 	generalNode(doc)
 	bs, err := json.MarshalIndent(trans, "", "   ")
 	if err != nil {

config/config.go

@@ -15,14 +15,14 @@ import (
 	"strconv"
 
 	"code.google.com/p/go.crypto/bcrypt"
-	"github.com/calmh/syncthing/logger"
-	"github.com/calmh/syncthing/protocol"
+	"github.com/syncthing/syncthing/logger"
+	"github.com/syncthing/syncthing/protocol"
 )
 
 var l = logger.DefaultLogger
 
 type Configuration struct {
-	Version      int                       `xml:"version,attr" default:"2"`
+	Version      int                       `xml:"version,attr" default:"3"`
 	Repositories []RepositoryConfiguration `xml:"repository"`
 	Nodes        []NodeConfiguration       `xml:"node"`
 	GUI          GUIConfiguration          `xml:"gui"`
@@ -31,13 +31,14 @@ type Configuration struct {
 }
 
 type RepositoryConfiguration struct {
-	ID          string                  `xml:"id,attr"`
-	Directory   string                  `xml:"directory,attr"`
-	Nodes       []NodeConfiguration     `xml:"node"`
-	ReadOnly    bool                    `xml:"ro,attr"`
-	IgnorePerms bool                    `xml:"ignorePerms,attr"`
-	Invalid     string                  `xml:"-"` // Set at runtime when there is an error, not saved
-	Versioning  VersioningConfiguration `xml:"versioning"`
+	ID              string                        `xml:"id,attr"`
+	Directory       string                        `xml:"directory,attr"`
+	Nodes           []RepositoryNodeConfiguration `xml:"node"`
+	ReadOnly        bool                          `xml:"ro,attr"`
+	RescanIntervalS int                           `xml:"rescanIntervalS,attr" default:"60"`
+	IgnorePerms     bool                          `xml:"ignorePerms,attr"`
+	Invalid         string                        `xml:"-"` // Set at runtime when there is an error, not saved
+	Versioning      VersioningConfiguration       `xml:"versioning"`
 
 	nodeIDs []protocol.NodeID
 }
@@ -100,26 +101,35 @@ type NodeConfiguration struct {
 	CertName    string          `xml:"certName,attr,omitempty"`
 }
 
+type RepositoryNodeConfiguration struct {
+	NodeID protocol.NodeID `xml:"id,attr"`
+
+	Deprecated_Name      string   `xml:"name,attr,omitempty" json:"-"`
+	Deprecated_Addresses []string `xml:"address,omitempty" json:"-"`
+}
+
 type OptionsConfiguration struct {
 	ListenAddress      []string `xml:"listenAddress" default:"0.0.0.0:22000"`
 	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22026"`
 	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true"`
 	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true"`
 	LocalAnnPort       int      `xml:"localAnnouncePort" default:"21025"`
+	LocalAnnMCAddr     string   `xml:"localAnnounceMCAddr" default:"[ff32::5222]:21026"`
 	ParallelRequests   int      `xml:"parallelRequests" default:"16"`
 	MaxSendKbps        int      `xml:"maxSendKbps"`
-	RescanIntervalS    int      `xml:"rescanIntervalS" default:"60"`
 	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60"`
-	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"10000"`
 	StartBrowser       bool     `xml:"startBrowser" default:"true"`
 	UPnPEnabled        bool     `xml:"upnpEnabled" default:"true"`
+	UPnPLease          int      `xml:"upnpLeaseMinutes" default:"0"`
+	UPnPRenewal        int      `xml:"upnpRenewalMinutes" default:"30"`
 	URAccepted         int      `xml:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
 
-	Deprecated_UREnabled  bool   `xml:"urEnabled,omitempty" json:"-"`
-	Deprecated_URDeclined bool   `xml:"urDeclined,omitempty" json:"-"`
-	Deprecated_ReadOnly   bool   `xml:"readOnly,omitempty" json:"-"`
-	Deprecated_GUIEnabled bool   `xml:"guiEnabled,omitempty" json:"-"`
-	Deprecated_GUIAddress string `xml:"guiAddress,omitempty" json:"-"`
+	Deprecated_RescanIntervalS int    `xml:"rescanIntervalS,omitempty" json:"-"`
+	Deprecated_UREnabled       bool   `xml:"urEnabled,omitempty" json:"-"`
+	Deprecated_URDeclined      bool   `xml:"urDeclined,omitempty" json:"-"`
+	Deprecated_ReadOnly        bool   `xml:"readOnly,omitempty" json:"-"`
+	Deprecated_GUIEnabled      bool   `xml:"guiEnabled,omitempty" json:"-"`
+	Deprecated_GUIAddress      string `xml:"guiAddress,omitempty" json:"-"`
 }
 
 type GUIConfiguration struct {
@@ -139,6 +149,15 @@ func (cfg *Configuration) NodeMap() map[protocol.NodeID]NodeConfiguration {
 	return m
 }
 
+func (cfg *Configuration) GetNodeConfiguration(nodeid protocol.NodeID) *NodeConfiguration {
+	for i, node := range cfg.Nodes {
+		if node.NodeID == nodeid {
+			return &cfg.Nodes[i]
+		}
+	}
+	return nil
+}
+
 func (cfg *Configuration) RepoMap() map[string]RepositoryConfiguration {
 	m := make(map[string]RepositoryConfiguration, len(cfg.Repositories))
 	for _, r := range cfg.Repositories {
@@ -296,11 +315,21 @@ func Load(rd io.Reader, myID protocol.NodeID) (Configuration, error) {
 		convertV1V2(&cfg)
 	}
 
+	// Upgrade to v3 configuration if appropriate
+	if cfg.Version == 2 {
+		convertV2V3(&cfg)
+	}
+
+	// Upgrade to v4 configuration if appropriate
+	if cfg.Version == 3 {
+		convertV3V4(&cfg)
+	}
+
 	// Hash old cleartext passwords
 	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
 		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
 		if err != nil {
-			l.Warnln(err)
+			l.Warnln("bcrypting password:", err)
 		} else {
 			cfg.GUI.Password = string(hash)
 		}
@@ -314,15 +343,22 @@ func Load(rd io.Reader, myID protocol.NodeID) (Configuration, error) {
 	}
 
 	// Ensure this node is present in all relevant places
+	me := cfg.GetNodeConfiguration(myID)
+	if me == nil {
+		myName, _ := os.Hostname()
+		cfg.Nodes = append(cfg.Nodes, NodeConfiguration{
+			NodeID: myID,
+			Name:   myName,
+		})
+	}
+	sort.Sort(NodeConfigurationList(cfg.Nodes))
 	// Ensure that any loose nodes are not present in the wrong places
 	// Ensure that there are no duplicate nodes
-	cfg.Nodes = ensureNodePresent(cfg.Nodes, myID)
-	sort.Sort(NodeConfigurationList(cfg.Nodes))
 	for i := range cfg.Repositories {
 		cfg.Repositories[i].Nodes = ensureNodePresent(cfg.Repositories[i].Nodes, myID)
 		cfg.Repositories[i].Nodes = ensureExistingNodes(cfg.Repositories[i].Nodes, existingNodes)
 		cfg.Repositories[i].Nodes = ensureNoDuplicates(cfg.Repositories[i].Nodes)
-		sort.Sort(NodeConfigurationList(cfg.Repositories[i].Nodes))
+		sort.Sort(RepositoryNodeConfigurationList(cfg.Repositories[i].Nodes))
 	}
 
 	// An empty address list is equivalent to a single "dynamic" entry
@@ -333,20 +369,56 @@ func Load(rd io.Reader, myID protocol.NodeID) (Configuration, error) {
 		}
 	}
 
+	return cfg, err
+}
+
+func convertV3V4(cfg *Configuration) {
+	// In previous versions, rescan interval was common for each repository.
+	// From now, it can be set independently. We have to make sure, that after upgrade
+	// the individual rescan interval will be defined for every existing repository.
+	for i := range cfg.Repositories {
+		cfg.Repositories[i].RescanIntervalS = cfg.Options.Deprecated_RescanIntervalS
+	}
+
+	cfg.Options.Deprecated_RescanIntervalS = 0
+
+	// In previous versions, repositories held full node configurations.
+	// Since that's the only place where node configs were in V1, we still have
+	// to define the deprecated fields to be able to upgrade from V1 to V4.
+	for i, repo := range cfg.Repositories {
+
+		for j := range repo.Nodes {
+			rncfg := cfg.Repositories[i].Nodes[j]
+			rncfg.Deprecated_Name = ""
+			rncfg.Deprecated_Addresses = nil
+		}
+	}
+
+	cfg.Version = 4
+}
+
+func convertV2V3(cfg *Configuration) {
+	// In previous versions, compression was always on. When upgrading, enable
+	// compression on all existing new. New nodes will get compression on by
+	// default by the GUI.
+	for i := range cfg.Nodes {
+		cfg.Nodes[i].Compression = true
+	}
+
 	// The global discovery format and port number changed in v0.9. Having the
 	// default announce server but old port number is guaranteed to be legacy.
 	if cfg.Options.GlobalAnnServer == "announce.syncthing.net:22025" {
 		cfg.Options.GlobalAnnServer = "announce.syncthing.net:22026"
 	}
 
-	return cfg, err
+	cfg.Version = 3
 }
 
 func convertV1V2(cfg *Configuration) {
 	// Collect the list of nodes.
 	// Replace node configs inside repositories with only a reference to the nide ID.
 	// Set all repositories to read only if the global read only flag is set.
-	var nodes = map[string]NodeConfiguration{}
+	var nodes = map[string]RepositoryNodeConfiguration{}
 	for i, repo := range cfg.Repositories {
 		cfg.Repositories[i].ReadOnly = cfg.Options.Deprecated_ReadOnly
 		for j, node := range repo.Nodes {
@@ -354,14 +426,18 @@ func convertV1V2(cfg *Configuration) {
 			if _, ok := nodes[id]; !ok {
 				nodes[id] = node
 			}
-			cfg.Repositories[i].Nodes[j] = NodeConfiguration{NodeID: node.NodeID}
+			cfg.Repositories[i].Nodes[j] = RepositoryNodeConfiguration{NodeID: node.NodeID}
 		}
 	}
 	cfg.Options.Deprecated_ReadOnly = false
 
 	// Set and sort the list of nodes.
 	for _, node := range nodes {
-		cfg.Nodes = append(cfg.Nodes, node)
+		cfg.Nodes = append(cfg.Nodes, NodeConfiguration{
+			NodeID:    node.NodeID,
+			Name:      node.Deprecated_Name,
+			Addresses: node.Deprecated_Addresses,
+		})
 	}
 	sort.Sort(NodeConfigurationList(cfg.Nodes))
 
@@ -386,23 +462,33 @@ func (l NodeConfigurationList) Len() int {
 	return len(l)
 }
 
-func ensureNodePresent(nodes []NodeConfiguration, myID protocol.NodeID) []NodeConfiguration {
+type RepositoryNodeConfigurationList []RepositoryNodeConfiguration
+
+func (l RepositoryNodeConfigurationList) Less(a, b int) bool {
+	return l[a].NodeID.Compare(l[b].NodeID) == -1
+}
+func (l RepositoryNodeConfigurationList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+func (l RepositoryNodeConfigurationList) Len() int {
+	return len(l)
+}
+
+func ensureNodePresent(nodes []RepositoryNodeConfiguration, myID protocol.NodeID) []RepositoryNodeConfiguration {
 	for _, node := range nodes {
 		if node.NodeID.Equals(myID) {
 			return nodes
 		}
 	}
 
-	name, _ := os.Hostname()
-	nodes = append(nodes, NodeConfiguration{
+	nodes = append(nodes, RepositoryNodeConfiguration{
 		NodeID: myID,
-		Name:   name,
 	})
 
 	return nodes
 }
 
-func ensureExistingNodes(nodes []NodeConfiguration, existingNodes map[protocol.NodeID]bool) []NodeConfiguration {
+func ensureExistingNodes(nodes []RepositoryNodeConfiguration, existingNodes map[protocol.NodeID]bool) []RepositoryNodeConfiguration {
 	count := len(nodes)
 	i := 0
 loop:
@@ -417,7 +503,7 @@ loop:
 	return nodes[0:count]
 }
 
-func ensureNoDuplicates(nodes []NodeConfiguration) []NodeConfiguration {
+func ensureNoDuplicates(nodes []RepositoryNodeConfiguration) []RepositoryNodeConfiguration {
 	count := len(nodes)
 	i := 0
 	seenNodes := make(map[protocol.NodeID]bool)

config/config_test.go

@@ -11,7 +11,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/calmh/syncthing/protocol"
+	"github.com/syncthing/syncthing/protocol"
 )
 
 var node1, node2, node3, node4 protocol.NodeID
@@ -30,13 +30,14 @@ func TestDefaultValues(t *testing.T) {
 		GlobalAnnEnabled:   true,
 		LocalAnnEnabled:    true,
 		LocalAnnPort:       21025,
+		LocalAnnMCAddr:     "[ff32::5222]:21026",
 		ParallelRequests:   16,
 		MaxSendKbps:        0,
-		RescanIntervalS:    60,
 		ReconnectIntervalS: 60,
-		MaxChangeKbps:      10000,
 		StartBrowser:       true,
 		UPnPEnabled:        true,
+		UPnPLease:          0,
+		UPnPRenewal:        30,
 	}
 
 	cfg, err := Load(bytes.NewReader(nil), node1)
@@ -68,6 +69,7 @@ func TestNodeConfig(t *testing.T) {
     </repository>
     <options>
         <readOnly>true</readOnly>
+        <rescanIntervalS>600</rescanIntervalS>
     </options>
 </configuration>
 `)
@@ -88,10 +90,44 @@ func TestNodeConfig(t *testing.T) {
     <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ" name="node two">
         <address>b</address>
     </node>
+    <options>
+        <rescanIntervalS>600</rescanIntervalS>
+    </options>
 </configuration>
 `)
 
-	for i, data := range [][]byte{v1data, v2data} {
+	v3data := []byte(`
+<configuration version="3">
+    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false">
+        <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" compression="false"></node>
+        <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" compression="false"></node>
+    </repository>
+    <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </node>
+    <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </node>
+    <options>
+        <rescanIntervalS>600</rescanIntervalS>
+    </options>
+</configuration>`)
+
+	v4data := []byte(`
+<configuration version="4">
+    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></node>
+        <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></node>
+    </repository>
+    <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </node>
+    <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </node>
+</configuration>`)
+
+	for i, data := range [][]byte{v1data, v2data, v3data, v4data} {
 		cfg, err := Load(bytes.NewReader(data), node1)
 		if err != nil {
 			t.Error(err)
@@ -99,28 +135,31 @@ func TestNodeConfig(t *testing.T) {
 
 		expectedRepos := []RepositoryConfiguration{
 			{
-				ID:        "test",
-				Directory: "~/Sync",
-				Nodes:     []NodeConfiguration{{NodeID: node1}, {NodeID: node4}},
-				ReadOnly:  true,
+				ID:              "test",
+				Directory:       "~/Sync",
+				Nodes:           []RepositoryNodeConfiguration{{NodeID: node1}, {NodeID: node4}},
+				ReadOnly:        true,
+				RescanIntervalS: 600,
 			},
 		}
 		expectedNodes := []NodeConfiguration{
 			{
-				NodeID:    node1,
-				Name:      "node one",
-				Addresses: []string{"a"},
+				NodeID:      node1,
+				Name:        "node one",
+				Addresses:   []string{"a"},
+				Compression: true,
 			},
 			{
-				NodeID:    node4,
-				Name:      "node two",
-				Addresses: []string{"b"},
+				NodeID:      node4,
+				Name:        "node two",
+				Addresses:   []string{"b"},
+				Compression: true,
 			},
 		}
 		expectedNodeIDs := []protocol.NodeID{node1, node4}
 
-		if cfg.Version != 2 {
-			t.Errorf("%d: Incorrect version %d != 2", i, cfg.Version)
+		if cfg.Version != 4 {
+			t.Errorf("%d: Incorrect version %d != 3", i, cfg.Version)
 		}
 		if !reflect.DeepEqual(cfg.Repositories, expectedRepos) {
 			t.Errorf("%d: Incorrect Repositories\n  A: %#v\n  E: %#v", i, cfg.Repositories, expectedRepos)
@@ -169,13 +208,14 @@ func TestOverriddenValues(t *testing.T) {
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>42123</localAnnouncePort>
+        <localAnnounceMCAddr>quux:3232</localAnnounceMCAddr>
         <parallelRequests>32</parallelRequests>
         <maxSendKbps>1234</maxSendKbps>
-        <rescanIntervalS>600</rescanIntervalS>
         <reconnectionIntervalS>6000</reconnectionIntervalS>
-        <maxChangeKbps>2345</maxChangeKbps>
         <startBrowser>false</startBrowser>
         <upnpEnabled>false</upnpEnabled>
+        <upnpLeaseMinutes>60</upnpLeaseMinutes>
+        <upnpRenewalMinutes>15</upnpRenewalMinutes>
     </options>
 </configuration>
 `)
@@ -186,13 +226,14 @@ func TestOverriddenValues(t *testing.T) {
 		GlobalAnnEnabled:   false,
 		LocalAnnEnabled:    false,
 		LocalAnnPort:       42123,
+		LocalAnnMCAddr:     "quux:3232",
 		ParallelRequests:   32,
 		MaxSendKbps:        1234,
-		RescanIntervalS:    600,
 		ReconnectIntervalS: 6000,
-		MaxChangeKbps:      2345,
 		StartBrowser:       false,
 		UPnPEnabled:        false,
+		UPnPLease:          60,
+		UPnPRenewal:        15,
 	}
 
 	cfg, err := Load(bytes.NewReader(data), node1)
@@ -222,16 +263,19 @@ func TestNodeAddressesDynamic(t *testing.T) {
 	name, _ := os.Hostname()
 	expected := []NodeConfiguration{
 		{
-			NodeID:    node1,
-			Addresses: []string{"dynamic"},
+			NodeID:      node1,
+			Addresses:   []string{"dynamic"},
+			Compression: true,
 		},
 		{
-			NodeID:    node2,
-			Addresses: []string{"dynamic"},
+			NodeID:      node2,
+			Addresses:   []string{"dynamic"},
+			Compression: true,
 		},
 		{
-			NodeID:    node3,
-			Addresses: []string{"dynamic"},
+			NodeID:      node3,
+			Addresses:   []string{"dynamic"},
+			Compression: true,
 		},
 		{
 			NodeID:    node4,
@@ -252,7 +296,7 @@ func TestNodeAddressesDynamic(t *testing.T) {
 
 func TestNodeAddressesStatic(t *testing.T) {
 	data := []byte(`
-<configuration version="2">
+<configuration version="3">
     <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
         <address>192.0.2.1</address>
         <address>192.0.2.2</address>

discover/debug.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/syncthing/logger"
+	"github.com/syncthing/syncthing/logger"
 )
 
 var (

discover/discover.go

@@ -8,15 +8,15 @@ import (
 	"bytes"
 	"encoding/hex"
 	"errors"
-	"fmt"
 	"io"
 	"net"
+	"strconv"
 	"sync"
 	"time"
 
-	"github.com/calmh/syncthing/beacon"
-	"github.com/calmh/syncthing/events"
-	"github.com/calmh/syncthing/protocol"
+	"github.com/syncthing/syncthing/beacon"
+	"github.com/syncthing/syncthing/events"
+	"github.com/syncthing/syncthing/protocol"
 )
 
 type Discoverer struct {
@@ -24,15 +24,26 @@ type Discoverer struct {
 	listenAddrs      []string
 	localBcastIntv   time.Duration
 	globalBcastIntv  time.Duration
-	beacon           *beacon.Beacon
-	registry         map[protocol.NodeID][]string
+	errorRetryIntv   time.Duration
+	cacheLifetime    time.Duration
+	broadcastBeacon  beacon.Interface
+	multicastBeacon  beacon.Interface
+	registry         map[protocol.NodeID][]cacheEntry
 	registryLock     sync.RWMutex
 	extServer        string
 	extPort          uint16
 	localBcastTick   <-chan time.Time
+	stopGlobal       chan struct{}
+	globalWG         sync.WaitGroup
 	forcedBcastTick  chan time.Time
 	extAnnounceOK    bool
 	extAnnounceOKmut sync.Mutex
+	globalBcastStop  chan bool
+}
+
+type cacheEntry struct {
+	addr string
+	seen time.Time
 }
 
 var (
@@ -44,37 +55,63 @@ var (
 // When we hit this many errors in succession, we stop.
 const maxErrors = 30
 
-func NewDiscoverer(id protocol.NodeID, addresses []string, localPort int) (*Discoverer, error) {
-	b, err := beacon.New(localPort)
-	if err != nil {
-		return nil, err
-	}
-	disc := &Discoverer{
+func NewDiscoverer(id protocol.NodeID, addresses []string) *Discoverer {
+	return &Discoverer{
 		myID:            id,
 		listenAddrs:     addresses,
 		localBcastIntv:  30 * time.Second,
 		globalBcastIntv: 1800 * time.Second,
-		beacon:          b,
-		registry:        make(map[protocol.NodeID][]string),
+		errorRetryIntv:  60 * time.Second,
+		cacheLifetime:   5 * time.Minute,
+		registry:        make(map[protocol.NodeID][]cacheEntry),
 	}
-
-	go disc.recvAnnouncements()
-
-	return disc, nil
 }
 
-func (d *Discoverer) StartLocal() {
-	d.localBcastTick = time.Tick(d.localBcastIntv)
-	d.forcedBcastTick = make(chan time.Time)
-	go d.sendLocalAnnouncements()
+func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
+	if localPort > 0 {
+		bb, err := beacon.NewBroadcast(localPort)
+		if err != nil {
+			l.Infof("No IPv4 discovery possible (%v)", err)
+		} else {
+			d.broadcastBeacon = bb
+			go d.recvAnnouncements(bb)
+		}
+	}
+
+	if len(localMCAddr) > 0 {
+		mb, err := beacon.NewMulticast(localMCAddr)
+		if err != nil {
+			l.Infof("No IPv6 discovery possible (%v)", err)
+		} else {
+			d.multicastBeacon = mb
+			go d.recvAnnouncements(mb)
+		}
+	}
+
+	if d.broadcastBeacon == nil && d.multicastBeacon == nil {
+		l.Warnln("No local discovery method available")
+	} else {
+		d.localBcastTick = time.Tick(d.localBcastIntv)
+		d.forcedBcastTick = make(chan time.Time)
+		go d.sendLocalAnnouncements()
+	}
 }
 
 func (d *Discoverer) StartGlobal(server string, extPort uint16) {
+	// Wait for any previous announcer to stop before starting a new one.
+	d.globalWG.Wait()
 	d.extServer = server
 	d.extPort = extPort
+	d.stopGlobal = make(chan struct{})
+	d.globalWG.Add(1)
 	go d.sendExternalAnnouncements()
 }
 
+func (d *Discoverer) StopGlobal() {
+	close(d.stopGlobal)
+	d.globalWG.Wait()
+}
+
 func (d *Discoverer) ExtAnnounceOK() bool {
 	d.extAnnounceOKmut.Lock()
 	defer d.extAnnounceOKmut.Unlock()
@@ -83,14 +120,28 @@ func (d *Discoverer) ExtAnnounceOK() bool {
 
 func (d *Discoverer) Lookup(node protocol.NodeID) []string {
 	d.registryLock.Lock()
-	addr, ok := d.registry[node]
+	cached := d.filterCached(d.registry[node])
 	d.registryLock.Unlock()
 
-	if ok {
-		return addr
+	if len(cached) > 0 {
+		addrs := make([]string, len(cached))
+		for i := range cached {
+			addrs[i] = cached[i].addr
+		}
+		return addrs
 	} else if len(d.extServer) != 0 {
-		// We might want to cache this, but not permanently so it needs some intelligence
-		return d.externalLookup(node)
+		addrs := d.externalLookup(node)
+		cached = make([]cacheEntry, len(addrs))
+		for i := range addrs {
+			cached[i] = cacheEntry{
+				addr: addrs[i],
+				seen: time.Now(),
+			}
+		}
+
+		d.registryLock.Lock()
+		d.registry[node] = cached
+		d.registryLock.Unlock()
 	}
 	return nil
 }
@@ -105,11 +156,11 @@ func (d *Discoverer) Hint(node string, addrs []string) {
 	})
 }
 
-func (d *Discoverer) All() map[protocol.NodeID][]string {
+func (d *Discoverer) All() map[protocol.NodeID][]cacheEntry {
 	d.registryLock.RLock()
-	nodes := make(map[protocol.NodeID][]string, len(d.registry))
+	nodes := make(map[protocol.NodeID][]cacheEntry, len(d.registry))
 	for node, addrs := range d.registry {
-		addrsCopy := make([]string, len(addrs))
+		addrsCopy := make([]cacheEntry, len(addrs))
 		copy(addrsCopy, addrs)
 		nodes[node] = addrsCopy
 	}
@@ -149,21 +200,15 @@ func (d *Discoverer) sendLocalAnnouncements() {
 		Magic: AnnouncementMagic,
 		This:  Node{d.myID[:], addrs},
 	}
+	msg := pkt.MarshalXDR()
 
 	for {
-		pkt.Extra = nil
-		d.registryLock.RLock()
-		for node, addrs := range d.registry {
-			if len(pkt.Extra) == 16 {
-				break
-			}
-
-			anode := Node{node[:], resolveAddrs(addrs)}
-			pkt.Extra = append(pkt.Extra, anode)
+		if d.multicastBeacon != nil {
+			d.multicastBeacon.Send(msg)
+		}
+		if d.broadcastBeacon != nil {
+			d.broadcastBeacon.Send(msg)
 		}
-		d.registryLock.RUnlock()
-
-		d.beacon.Send(pkt.MarshalXDR())
 
 		select {
 		case <-d.localBcastTick:
@@ -173,20 +218,19 @@ func (d *Discoverer) sendLocalAnnouncements() {
 }
 
 func (d *Discoverer) sendExternalAnnouncements() {
-	// this should go in the Discoverer struct
-	errorRetryIntv := 60 * time.Second
+	defer d.globalWG.Done()
 
 	remote, err := net.ResolveUDPAddr("udp", d.extServer)
 	for err != nil {
-		l.Warnf("Global discovery: %v; trying again in %v", err, errorRetryIntv)
-		time.Sleep(errorRetryIntv)
+		l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
+		time.Sleep(d.errorRetryIntv)
 		remote, err = net.ResolveUDPAddr("udp", d.extServer)
 	}
 
 	conn, err := net.ListenUDP("udp", nil)
 	for err != nil {
-		l.Warnf("Global discovery: %v; trying again in %v", err, errorRetryIntv)
-		time.Sleep(errorRetryIntv)
+		l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
+		time.Sleep(d.errorRetryIntv)
 		conn, err = net.ListenUDP("udp", nil)
 	}
 
@@ -201,7 +245,10 @@ func (d *Discoverer) sendExternalAnnouncements() {
 		buf = d.announcementPkt()
 	}
 
-	for {
+	var bcastTick = time.Tick(d.globalBcastIntv)
+	var errTick <-chan time.Time
+
+	sendOneAnnouncement := func() {
 		var ok bool
 
 		if debug {
@@ -230,19 +277,40 @@ func (d *Discoverer) sendExternalAnnouncements() {
 		d.extAnnounceOKmut.Unlock()
 
 		if ok {
-			time.Sleep(d.globalBcastIntv)
-		} else {
-			time.Sleep(errorRetryIntv)
+			errTick = nil
+		} else if errTick != nil {
+			errTick = time.Tick(d.errorRetryIntv)
 		}
 	}
+
+	// Announce once, immediately
+	sendOneAnnouncement()
+
+loop:
+	for {
+		select {
+		case <-d.stopGlobal:
+			break loop
+
+		case <-errTick:
+			sendOneAnnouncement()
+
+		case <-bcastTick:
+			sendOneAnnouncement()
+		}
+	}
+
+	if debug {
+		l.Debugln("discover: stopping global")
+	}
 }
 
-func (d *Discoverer) recvAnnouncements() {
+func (d *Discoverer) recvAnnouncements(b beacon.Interface) {
 	for {
-		buf, addr := d.beacon.Recv()
+		buf, addr := b.Recv()
 
 		if debug {
-			l.Debugf("discover: read announcement:\n%s", hex.Dump(buf))
+			l.Debugf("discover: read announcement from %s:\n%s", addr, hex.Dump(buf))
 		}
 
 		var pkt Announce
@@ -251,20 +319,9 @@ func (d *Discoverer) recvAnnouncements() {
 			continue
 		}
 
-		if debug {
-			l.Debugf("discover: parsed announcement: %#v", pkt)
-		}
-
 		var newNode bool
 		if bytes.Compare(pkt.This.ID, d.myID[:]) != 0 {
 			newNode = d.registerNode(addr, pkt.This)
-			for _, node := range pkt.Extra {
-				if bytes.Compare(node.ID, d.myID[:]) != 0 {
-					if d.registerNode(nil, node) {
-						newNode = true
-					}
-				}
-			}
 		}
 
 		if newNode {
@@ -276,41 +333,57 @@ func (d *Discoverer) recvAnnouncements() {
 }
 
 func (d *Discoverer) registerNode(addr net.Addr, node Node) bool {
-	var addrs []string
+	var id protocol.NodeID
+	copy(id[:], node.ID)
+
+	d.registryLock.RLock()
+	current := d.filterCached(d.registry[id])
+	d.registryLock.RUnlock()
+
+	orig := current
+
 	for _, a := range node.Addresses {
 		var nodeAddr string
 		if len(a.IP) > 0 {
-			nodeAddr = fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
-			addrs = append(addrs, nodeAddr)
+			nodeAddr = net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
 		} else if addr != nil {
 			ua := addr.(*net.UDPAddr)
 			ua.Port = int(a.Port)
 			nodeAddr = ua.String()
-			addrs = append(addrs, nodeAddr)
 		}
-	}
-	if len(addrs) == 0 {
-		if debug {
-			l.Debugln("discover: no valid address for", node.ID)
+		for i := range current {
+			if current[i].addr == nodeAddr {
+				current[i].seen = time.Now()
+				goto done
+			}
 		}
+		current = append(current, cacheEntry{
+			addr: nodeAddr,
+			seen: time.Now(),
+		})
+	done:
 	}
+
 	if debug {
-		l.Debugf("discover: register: %s -> %#v", node.ID, addrs)
+		l.Debugf("discover: register: %v -> %v", id, current)
 	}
-	var id protocol.NodeID
-	copy(id[:], node.ID)
+
 	d.registryLock.Lock()
-	_, seen := d.registry[id]
-	d.registry[id] = addrs
+	d.registry[id] = current
 	d.registryLock.Unlock()
 
-	if !seen {
+	if len(current) > len(orig) {
+		addrs := make([]string, len(current))
+		for i := range current {
+			addrs[i] = current[i].addr
+		}
 		events.Default.Log(events.NodeDiscovered, map[string]interface{}{
 			"node":  id.String(),
 			"addrs": addrs,
 		})
 	}
-	return !seen
+
+	return len(current) > len(orig)
 }
 
 func (d *Discoverer) externalLookup(node protocol.NodeID) []string {
@@ -374,18 +447,29 @@ func (d *Discoverer) externalLookup(node protocol.NodeID) []string {
 		return nil
 	}
 
-	if debug {
-		l.Debugf("discover: parsed external: %#v", pkt)
-	}
-
 	var addrs []string
 	for _, a := range pkt.This.Addresses {
-		nodeAddr := fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
+		nodeAddr := net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
 		addrs = append(addrs, nodeAddr)
 	}
 	return addrs
 }
 
+func (d *Discoverer) filterCached(c []cacheEntry) []cacheEntry {
+	for i := 0; i < len(c); {
+		if ago := time.Since(c[i].seen); ago > d.cacheLifetime {
+			if debug {
+				l.Debugf("removing cached address %s: seen %v ago", c[i].addr, ago)
+			}
+			c[i] = c[len(c)-1]
+			c = c[:len(c)-1]
+		} else {
+			i++
+		}
+	}
+	return c
+}
+
 func addrToAddr(addr *net.TCPAddr) Address {
 	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
 		return Address{Port: uint16(addr.Port)}

discover/discover_test.go

@@ -0,0 +1,7 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package discover_test
+
+// Empty test file to generate 0% coverage rather than no coverage

events/debug.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/syncthing/logger"
+	"github.com/syncthing/syncthing/logger"
 )
 
 var (

events/events.go

@@ -20,11 +20,12 @@ const (
 	NodeDiscovered
 	NodeConnected
 	NodeDisconnected
+	NodeRejected
 	LocalIndexUpdated
 	RemoteIndexUpdated
 	ItemStarted
-	ItemCompleted
 	StateChanged
+	RepoRejected
 
 	AllEvents = ^EventType(0)
 )
@@ -43,6 +44,8 @@ func (t EventType) String() string {
 		return "NodeConnected"
 	case NodeDisconnected:
 		return "NodeDisconnected"
+	case NodeRejected:
+		return "NodeRejected"
 	case LocalIndexUpdated:
 		return "LocalIndexUpdated"
 	case RemoteIndexUpdated:
@@ -51,6 +54,8 @@ func (t EventType) String() string {
 		return "ItemStarted"
 	case StateChanged:
 		return "StateChanged"
+	case RepoRejected:
+		return "RepoRejected"
 	default:
 		return "Unknown"
 	}

events/events_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/calmh/syncthing/events"
+	"github.com/syncthing/syncthing/events"
 )
 
 var timeout = 100 * time.Millisecond

files/debug.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/calmh/syncthing/logger"
+	"github.com/syncthing/syncthing/logger"
 )
 
 var (

files/filenames_darwin.go

@@ -0,0 +1,11 @@
+package files
+
+import "code.google.com/p/go.text/unicode/norm"
+
+func normalizedFilename(s string) string {
+	return norm.NFC.String(s)
+}
+
+func nativeFilename(s string) string {
+	return norm.NFD.String(s)
+}

files/filenames_unix.go

@@ -0,0 +1,13 @@
+// +build !windows,!darwin
+
+package files
+
+import "code.google.com/p/go.text/unicode/norm"
+
+func normalizedFilename(s string) string {
+	return norm.NFC.String(s)
+}
+
+func nativeFilename(s string) string {
+	return s
+}

files/filenames_windows.go

@@ -0,0 +1,15 @@
+package files
+
+import (
+	"path/filepath"
+
+	"code.google.com/p/go.text/unicode/norm"
+)
+
+func normalizedFilename(s string) string {
+	return norm.NFC.String(filepath.ToSlash(s))
+}
+
+func nativeFilename(s string) string {
+	return filepath.FromSlash(s)
+}

files/leveldb.go

@@ -2,11 +2,12 @@ package files
 
 import (
 	"bytes"
+	"runtime"
 	"sort"
 	"sync"
 
-	"github.com/calmh/syncthing/lamport"
-	"github.com/calmh/syncthing/protocol"
+	"github.com/syncthing/syncthing/lamport"
+	"github.com/syncthing/syncthing/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -116,11 +117,19 @@ func globalKeyName(key []byte) []byte {
 	return key[1+64:]
 }
 
+func globalKeyRepo(key []byte) []byte {
+	repo := key[1 : 1+64]
+	izero := bytes.IndexByte(repo, 0)
+	return repo[:izero]
+}
+
 type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64
 
-type fileIterator func(f protocol.FileInfo) bool
+type fileIterator func(f protocol.FileIntf) bool
 
 func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
+	defer runtime.GC()
+
 	sort.Sort(fileList(fs)) // sort list on name, same as on disk
 
 	start := nodeKey(repo, node, nil)                            // before all repo/node files
@@ -178,7 +187,7 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo
 
 		case moreFs && moreDb && cmp == 0:
 			// File exists on both sides - compare versions.
-			var ef protocol.FileInfo
+			var ef protocol.FileInfoTruncated
 			ef.UnmarshalXDR(dbi.Value())
 			if fs[fsi].Version > ef.Version {
 				if lv := ldbInsert(batch, repo, node, newName, fs[fsi]); lv > maxLocalVer {
@@ -223,20 +232,23 @@ func ldbReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint6
 
 func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
 	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64 {
-		var f protocol.FileInfo
-		err := f.UnmarshalXDR(dbi.Value())
+		var tf protocol.FileInfoTruncated
+		err := tf.UnmarshalXDR(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
-		if !protocol.IsDeleted(f.Flags) {
+		if !tf.IsDeleted() {
 			if debug {
 				l.Debugf("mark deleted; repo=%q node=%v name=%q", repo, protocol.NodeIDFromBytes(node), name)
 			}
-			ts := clock(f.LocalVersion)
-			f.Blocks = nil
-			f.Version = lamport.Default.Tick(f.Version)
-			f.Flags |= protocol.FlagDeleted
-			f.LocalVersion = ts
+			ts := clock(tf.LocalVersion)
+			f := protocol.FileInfo{
+				Name:         tf.Name,
+				Version:      lamport.Default.Tick(tf.Version),
+				LocalVersion: ts,
+				Flags:        tf.Flags | protocol.FlagDeleted,
+				Modified:     tf.Modified,
+			}
 			batch.Put(dbi.Key(), f.MarshalXDR())
 			ldbUpdateGlobal(db, batch, repo, node, nodeKeyName(dbi.Key()), f.Version)
 			return ts
@@ -246,6 +258,8 @@ func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileI
 }
 
 func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
+	defer runtime.GC()
+
 	batch := new(leveldb.Batch)
 	snap, err := db.GetSnapshot()
 	if err != nil {
@@ -266,7 +280,7 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64
 			continue
 		}
 
-		var ef protocol.FileInfo
+		var ef protocol.FileInfoTruncated
 		err = ef.UnmarshalXDR(bs)
 		if err != nil {
 			panic(err)
@@ -390,7 +404,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, repo, node, file []byte) {
 	}
 }
 
-func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
+func ldbWithHave(db *leveldb.DB, repo, node []byte, truncate bool, fn fileIterator) {
 	start := nodeKey(repo, node, nil)                            // before all repo/node files
 	limit := nodeKey(repo, node, []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
 	snap, err := db.GetSnapshot()
@@ -402,8 +416,7 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
 	defer dbi.Release()
 
 	for dbi.Next() {
-		var f protocol.FileInfo
-		err := f.UnmarshalXDR(dbi.Value())
+		f, err := unmarshalTrunc(dbi.Value(), truncate)
 		if err != nil {
 			panic(err)
 		}
@@ -413,7 +426,9 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
 	}
 }
 
-func ldbWithAllRepo(db *leveldb.DB, repo []byte, fn func(node []byte, f protocol.FileInfo) bool) {
+func ldbWithAllRepoTruncated(db *leveldb.DB, repo []byte, fn func(node []byte, f protocol.FileInfoTruncated) bool) {
+	defer runtime.GC()
+
 	start := nodeKey(repo, nil, nil)                                                // before all repo/node files
 	limit := nodeKey(repo, protocol.LocalNodeID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
 	snap, err := db.GetSnapshot()
@@ -426,7 +441,7 @@ func ldbWithAllRepo(db *leveldb.DB, repo []byte, fn func(node []byte, f protocol
 
 	for dbi.Next() {
 		node := nodeKeyNode(dbi.Key())
-		var f protocol.FileInfo
+		var f protocol.FileInfoTruncated
 		err := f.UnmarshalXDR(dbi.Value())
 		if err != nil {
 			panic(err)
@@ -437,40 +452,6 @@ func ldbWithAllRepo(db *leveldb.DB, repo []byte, fn func(node []byte, f protocol
 	}
 }
 
-/*
-func ldbCheckGlobalConsistency(db *leveldb.DB, repo []byte) {
-	l.Debugf("Checking global consistency for %q", repo)
-	start := nodeKey(repo, nil, nil)                                                // before all repo/node files
-	limit := nodeKey(repo, protocol.LocalNodeID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
-	snap, err := db.GetSnapshot()
-	if err != nil {
-		panic(err)
-	}
-	defer snap.Release()
-	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
-	defer dbi.Release()
-
-	batch := new(leveldb.Batch)
-	i := 0
-	for dbi.Next() {
-		repo := nodeKeyRepo(dbi.Key())
-		node := nodeKeyNode(dbi.Key())
-		var f protocol.FileInfo
-		err := f.UnmarshalXDR(dbi.Value())
-		if err != nil {
-			panic(err)
-		}
-		if ldbUpdateGlobal(snap, batch, repo, node, []byte(f.Name), f.Version) {
-			var nodeID protocol.NodeID
-			copy(nodeID[:], node)
-			l.Debugf("fixed global for %q %s %q", repo, nodeID, f.Name)
-		}
-		i++
-	}
-	l.Debugln("Done", i)
-}
-*/
-
 func ldbGet(db *leveldb.DB, repo, node, file []byte) protocol.FileInfo {
 	nk := nodeKey(repo, node, file)
 	bs, err := db.Get(nk, nil)
@@ -529,7 +510,9 @@ func ldbGetGlobal(db *leveldb.DB, repo, file []byte) protocol.FileInfo {
 	return f
 }
 
-func ldbWithGlobal(db *leveldb.DB, repo []byte, fn fileIterator) {
+func ldbWithGlobal(db *leveldb.DB, repo []byte, truncate bool, fn fileIterator) {
+	defer runtime.GC()
+
 	start := globalKey(repo, nil)
 	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
 	snap, err := db.GetSnapshot()
@@ -556,8 +539,7 @@ func ldbWithGlobal(db *leveldb.DB, repo []byte, fn fileIterator) {
 			panic(err)
 		}
 
-		var f protocol.FileInfo
-		err = f.UnmarshalXDR(bs)
+		f, err := unmarshalTrunc(bs, truncate)
 		if err != nil {
 			panic(err)
 		}
@@ -596,7 +578,9 @@ func ldbAvailability(db *leveldb.DB, repo, file []byte) []protocol.NodeID {
 	return nodes
 }
 
-func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
+func ldbWithNeed(db *leveldb.DB, repo, node []byte, truncate bool, fn fileIterator) {
+	defer runtime.GC()
+
 	start := globalKey(repo, nil)
 	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
 	snap, err := db.GetSnapshot()
@@ -638,13 +622,12 @@ func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
 				panic(err)
 			}
 
-			var gf protocol.FileInfo
-			err = gf.UnmarshalXDR(bs)
+			gf, err := unmarshalTrunc(bs, truncate)
 			if err != nil {
 				panic(err)
 			}
 
-			if protocol.IsDeleted(gf.Flags) && !have {
+			if gf.IsDeleted() && !have {
 				// We don't need deleted files that we don't have
 				continue
 			}
@@ -659,3 +642,79 @@ func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
 		}
 	}
 }
+
+func ldbListRepos(db *leveldb.DB) []string {
+	defer runtime.GC()
+
+	start := []byte{keyTypeGlobal}
+	limit := []byte{keyTypeGlobal + 1}
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	repoExists := make(map[string]bool)
+	for dbi.Next() {
+		repo := string(globalKeyRepo(dbi.Key()))
+		if !repoExists[repo] {
+			repoExists[repo] = true
+		}
+	}
+
+	repos := make([]string, 0, len(repoExists))
+	for k := range repoExists {
+		repos = append(repos, k)
+	}
+
+	sort.Strings(repos)
+	return repos
+}
+
+func ldbDropRepo(db *leveldb.DB, repo []byte) {
+	defer runtime.GC()
+
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+
+	// Remove all items related to the given repo from the node->file bucket
+	start := []byte{keyTypeNode}
+	limit := []byte{keyTypeNode + 1}
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	for dbi.Next() {
+		itemRepo := nodeKeyRepo(dbi.Key())
+		if bytes.Compare(repo, itemRepo) == 0 {
+			db.Delete(dbi.Key(), nil)
+		}
+	}
+	dbi.Release()
+
+	// Remove all items related to the given repo from the global bucket
+	start = []byte{keyTypeGlobal}
+	limit = []byte{keyTypeGlobal + 1}
+	dbi = snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	for dbi.Next() {
+		itemRepo := globalKeyRepo(dbi.Key())
+		if bytes.Compare(repo, itemRepo) == 0 {
+			db.Delete(dbi.Key(), nil)
+		}
+	}
+	dbi.Release()
+}
+
+func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
+	if truncate {
+		var tf protocol.FileInfoTruncated
+		err := tf.UnmarshalXDR(bs)
+		return tf, err
+	} else {
+		var tf protocol.FileInfo
+		err := tf.UnmarshalXDR(bs)
+		return tf, err
+	}
+}

files/set.go

@@ -2,14 +2,19 @@
 // All rights reserved. Use of this source code is governed by an MIT-style
 // license that can be found in the LICENSE file.
 
-// Package files provides a set type to track local/remote files with newness checks.
+// Package files provides a set type to track local/remote files with newness
+// checks. We must do a certain amount of normalization in here. We will get
+// fed paths with either native or wire-format separators and encodings
+// depending on who calls us. We transform paths to wire-format (NFC and
+// slashes) on the way to the database, and transform to native format
+// (varying separator and encoding) on the way back out.
 package files
 
 import (
 	"sync"
 
-	"github.com/calmh/syncthing/lamport"
-	"github.com/calmh/syncthing/protocol"
+	"github.com/syncthing/syncthing/lamport"
+	"github.com/syncthing/syncthing/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -36,7 +41,7 @@ func NewSet(repo string, db *leveldb.DB) *Set {
 	}
 
 	var nodeID protocol.NodeID
-	ldbWithAllRepo(db, []byte(repo), func(node []byte, f protocol.FileInfo) bool {
+	ldbWithAllRepoTruncated(db, []byte(repo), func(node []byte, f protocol.FileInfoTruncated) bool {
 		copy(nodeID[:], node)
 		if f.LocalVersion > s.localVersion[nodeID] {
 			s.localVersion[nodeID] = f.LocalVersion
@@ -56,6 +61,7 @@ func (s *Set) Replace(node protocol.NodeID, fs []protocol.FileInfo) {
 	if debug {
 		l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs))
 	}
+	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	s.localVersion[node] = ldbReplace(s.db, []byte(s.repo), node[:], fs)
@@ -65,6 +71,7 @@ func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []protocol.FileInfo) {
 	if debug {
 		l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs))
 	}
+	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	if lv := ldbReplaceWithDelete(s.db, []byte(s.repo), node[:], fs); lv > s.localVersion[node] {
@@ -76,6 +83,7 @@ func (s *Set) Update(node protocol.NodeID, fs []protocol.FileInfo) {
 	if debug {
 		l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs))
 	}
+	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	if lv := ldbUpdate(s.db, []byte(s.repo), node[:], fs); lv > s.localVersion[node] {
@@ -87,33 +95,58 @@ func (s *Set) WithNeed(node protocol.NodeID, fn fileIterator) {
 	if debug {
 		l.Debugf("%s WithNeed(%v)", s.repo, node)
 	}
-	ldbWithNeed(s.db, []byte(s.repo), node[:], fn)
+	ldbWithNeed(s.db, []byte(s.repo), node[:], false, nativeFileIterator(fn))
+}
+
+func (s *Set) WithNeedTruncated(node protocol.NodeID, fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithNeedTruncated(%v)", s.repo, node)
+	}
+	ldbWithNeed(s.db, []byte(s.repo), node[:], true, nativeFileIterator(fn))
 }
 
 func (s *Set) WithHave(node protocol.NodeID, fn fileIterator) {
 	if debug {
 		l.Debugf("%s WithHave(%v)", s.repo, node)
 	}
-	ldbWithHave(s.db, []byte(s.repo), node[:], fn)
+	ldbWithHave(s.db, []byte(s.repo), node[:], false, nativeFileIterator(fn))
+}
+
+func (s *Set) WithHaveTruncated(node protocol.NodeID, fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithHaveTruncated(%v)", s.repo, node)
+	}
+	ldbWithHave(s.db, []byte(s.repo), node[:], true, nativeFileIterator(fn))
 }
 
 func (s *Set) WithGlobal(fn fileIterator) {
 	if debug {
 		l.Debugf("%s WithGlobal()", s.repo)
 	}
-	ldbWithGlobal(s.db, []byte(s.repo), fn)
+	ldbWithGlobal(s.db, []byte(s.repo), false, nativeFileIterator(fn))
+}
+
+func (s *Set) WithGlobalTruncated(fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithGlobalTruncated()", s.repo)
+	}
+	ldbWithGlobal(s.db, []byte(s.repo), true, nativeFileIterator(fn))
 }
 
 func (s *Set) Get(node protocol.NodeID, file string) protocol.FileInfo {
-	return ldbGet(s.db, []byte(s.repo), node[:], []byte(file))
+	f := ldbGet(s.db, []byte(s.repo), node[:], []byte(normalizedFilename(file)))
+	f.Name = nativeFilename(f.Name)
+	return f
 }
 
 func (s *Set) GetGlobal(file string) protocol.FileInfo {
-	return ldbGetGlobal(s.db, []byte(s.repo), []byte(file))
+	f := ldbGetGlobal(s.db, []byte(s.repo), []byte(normalizedFilename(file)))
+	f.Name = nativeFilename(f.Name)
+	return f
 }
 
 func (s *Set) Availability(file string) []protocol.NodeID {
-	return ldbAvailability(s.db, []byte(s.repo), []byte(file))
+	return ldbAvailability(s.db, []byte(s.repo), []byte(normalizedFilename(file)))
 }
 
 func (s *Set) LocalVersion(node protocol.NodeID) uint64 {
@@ -121,3 +154,35 @@ func (s *Set) LocalVersion(node protocol.NodeID) uint64 {
 	defer s.mutex.Unlock()
 	return s.localVersion[node]
 }
+
+// ListRepos returns the repository IDs seen in the database.
+func ListRepos(db *leveldb.DB) []string {
+	return ldbListRepos(db)
+}
+
+// DropRepo clears out all information related to the given repo from the
+// database.
+func DropRepo(db *leveldb.DB, repo string) {
+	ldbDropRepo(db, []byte(repo))
+}
+
+func normalizeFilenames(fs []protocol.FileInfo) {
+	for i := range fs {
+		fs[i].Name = normalizedFilename(fs[i].Name)
+	}
+}
+
+func nativeFileIterator(fn fileIterator) fileIterator {
+	return func(fi protocol.FileIntf) bool {
+		switch f := fi.(type) {
+		case protocol.FileInfo:
+			f.Name = nativeFilename(f.Name)
+			return fn(f)
+		case protocol.FileInfoTruncated:
+			f.Name = nativeFilename(f.Name)
+			return fn(f)
+		default:
+			panic("unknown interface type")
+		}
+	}
+}

files/set_test.go

@@ -5,13 +5,15 @@
 package files_test
 
 import (
+	"bytes"
 	"fmt"
+	"reflect"
 	"sort"
 	"testing"
 
-	"github.com/calmh/syncthing/files"
-	"github.com/calmh/syncthing/lamport"
-	"github.com/calmh/syncthing/protocol"
+	"github.com/syncthing/syncthing/files"
+	"github.com/syncthing/syncthing/lamport"
+	"github.com/syncthing/syncthing/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
@@ -37,7 +39,8 @@ func genBlocks(n int) []protocol.BlockInfo {
 
 func globalList(s *files.Set) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithGlobal(func(f protocol.FileInfo) bool {
+	s.WithGlobal(func(fi protocol.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
 	})
@@ -46,7 +49,8 @@ func globalList(s *files.Set) []protocol.FileInfo {
 
 func haveList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithHave(n, func(f protocol.FileInfo) bool {
+	s.WithHave(n, func(fi protocol.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
 	})
@@ -55,7 +59,8 @@ func haveList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
 
 func needList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithNeed(n, func(f protocol.FileInfo) bool {
+	s.WithNeed(n, func(fi protocol.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
 	})
@@ -592,3 +597,144 @@ func TestLocalVersion(t *testing.T) {
 		t.Fatal("Local version number should be unchanged")
 	}
 }
+
+func TestListDropRepo(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s0 := files.NewSet("test0", db)
+	local1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+	}
+	s0.Replace(protocol.LocalNodeID, local1)
+
+	s1 := files.NewSet("test1", db)
+	local2 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "d", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1002},
+		protocol.FileInfo{Name: "f", Version: 1002},
+	}
+	s1.Replace(remoteNode, local2)
+
+	// Check that we have both repos and their data is in the global list
+
+	expectedRepoList := []string{"test0", "test1"}
+	if actualRepoList := files.ListRepos(db); !reflect.DeepEqual(actualRepoList, expectedRepoList) {
+		t.Fatalf("RepoList mismatch\nE: %v\nA: %v", expectedRepoList, actualRepoList)
+	}
+	if l := len(globalList(s0)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s0", l)
+	}
+	if l := len(globalList(s1)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s1", l)
+	}
+
+	// Drop one of them and check that it's gone.
+
+	files.DropRepo(db, "test1")
+
+	expectedRepoList = []string{"test0"}
+	if actualRepoList := files.ListRepos(db); !reflect.DeepEqual(actualRepoList, expectedRepoList) {
+		t.Fatalf("RepoList mismatch\nE: %v\nA: %v", expectedRepoList, actualRepoList)
+	}
+	if l := len(globalList(s0)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s0", l)
+	}
+	if l := len(globalList(s1)); l != 0 {
+		t.Errorf("Incorrect global length %d != 0 for s1", l)
+	}
+}
+
+func TestLongPath(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test", db)
+
+	var b bytes.Buffer
+	for i := 0; i < 100; i++ {
+		b.WriteString("012345678901234567890123456789012345678901234567890")
+	}
+	name := b.String() // 5000 characters
+
+	local := []protocol.FileInfo{
+		protocol.FileInfo{Name: string(name), Version: 1000},
+	}
+
+	s.ReplaceWithDelete(protocol.LocalNodeID, local)
+
+	gf := globalList(s)
+	if l := len(gf); l != 1 {
+		t.Fatalf("Incorrect len %d != 1 for global list", l)
+	}
+	if gf[0].Name != local[0].Name {
+		t.Error("Incorrect long filename;\n%q !=\n%q", gf[0].Name, local[0].Name)
+	}
+}
+
+/*
+var gf protocol.FileInfo
+
+func TestStressGlobalVersion(t *testing.T) {
+	dur := 15 * time.Second
+	if testing.Short() {
+		dur = 1 * time.Second
+	}
+
+	set1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+	}
+	set2 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1000},
+	}
+
+	db, err := leveldb.OpenFile("testdata/global.db", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+
+	done := make(chan struct{})
+	go stressWriter(m, remoteNode, set1, nil, done)
+	go stressWriter(m, protocol.LocalNodeID, set2, nil, done)
+
+	t0 := time.Now()
+	for time.Since(t0) < dur {
+		m.WithGlobal(func(f protocol.FileInfo) bool {
+			gf = f
+			return true
+		})
+	}
+
+	close(done)
+}
+
+func stressWriter(s *files.Set, id protocol.NodeID, set1, set2 []protocol.FileInfo, done chan struct{}) {
+	one := true
+	i := 0
+	for {
+		select {
+		case <-done:
+			return
+
+		default:
+			if one {
+				s.Replace(id, set1)
+			} else {
+				s.Replace(id, set2)
+			}
+			one = !one
+		}
+		i++
+	}
+}
+*/

fnmatch/fnmatch.go

@@ -0,0 +1,62 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package fnmatch
+
+import (
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+const (
+	FNM_NOESCAPE = (1 << iota)
+	FNM_PATHNAME
+	FNM_CASEFOLD
+)
+
+func Convert(pattern string, flags int) (*regexp.Regexp, error) {
+	any := "."
+
+	if runtime.GOOS == "windows" {
+		flags |= FNM_NOESCAPE
+		pattern = filepath.FromSlash(pattern)
+		if flags&FNM_PATHNAME != 0 {
+			any = "[^\\\\]"
+		}
+	} else if flags&FNM_PATHNAME != 0 {
+		any = "[^/]"
+	}
+	if flags&FNM_NOESCAPE != 0 {
+		pattern = strings.Replace(pattern, "\\", "\\\\", -1)
+	} else {
+		pattern = strings.Replace(pattern, "\\*", "[:escapedstar:]", -1)
+		pattern = strings.Replace(pattern, "\\?", "[:escapedques:]", -1)
+		pattern = strings.Replace(pattern, "\\.", "[:escapeddot:]", -1)
+	}
+	pattern = strings.Replace(pattern, ".", "\\.", -1)
+	pattern = strings.Replace(pattern, "**", "[:doublestar:]", -1)
+	pattern = strings.Replace(pattern, "*", any+"*", -1)
+	pattern = strings.Replace(pattern, "[:doublestar:]", ".*", -1)
+	pattern = strings.Replace(pattern, "?", any, -1)
+	pattern = strings.Replace(pattern, "[:escapedstar:]", "\\*", -1)
+	pattern = strings.Replace(pattern, "[:escapedques:]", "\\?", -1)
+	pattern = strings.Replace(pattern, "[:escapeddot:]", "\\.", -1)
+	pattern = "^" + pattern + "$"
+	if flags&FNM_CASEFOLD != 0 {
+		pattern = "(?i)" + pattern
+	}
+	return regexp.Compile(pattern)
+}
+
+// Matches the pattern against the string, with the given flags,
+// and returns true if the match is successful.
+func Match(pattern, s string, flags int) (bool, error) {
+	exp, err := Convert(pattern, flags)
+	if err != nil {
+		return false, err
+	}
+	return exp.MatchString(s), nil
+}

fnmatch/fnmatch_test.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package fnmatch
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+type testcase struct {
+	pat   string
+	name  string
+	flags int
+	match bool
+}
+
+var testcases = []testcase{
+	{"", "", 0, true},
+	{"*", "", 0, true},
+	{"*", "foo", 0, true},
+	{"*", "bar", 0, true},
+	{"*", "*", 0, true},
+	{"**", "f", 0, true},
+	{"**", "foo.txt", 0, true},
+	{"*.*", "foo.txt", 0, true},
+	{"foo*.txt", "foobar.txt", 0, true},
+	{"foo.txt", "foo.txt", 0, true},
+
+	{"foo.txt", "bar/foo.txt", 0, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"f?o.txt", "foo.txt", 0, true},
+	{"f?o.txt", "fooo.txt", 0, false},
+	{"f[ab]o.txt", "foo.txt", 0, false},
+	{"f[ab]o.txt", "fao.txt", 0, true},
+	{"f[ab]o.txt", "fbo.txt", 0, true},
+	{"f[ab]o.txt", "fco.txt", 0, false},
+	{"f[ab]o.txt", "fabo.txt", 0, false},
+	{"f[ab]o.txt", "f[ab]o.txt", 0, false},
+	{"f\\[ab\\]o.txt", "f[ab]o.txt", FNM_NOESCAPE, false},
+
+	{"*foo.txt", "bar/foo.txt", 0, true},
+	{"*foo.txt", "bar/foo.txt", FNM_PATHNAME, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"*/foo.txt", "bar/foo.txt", FNM_PATHNAME, true},
+	{"*/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"*/foo.txt", "bar/baz/foo.txt", FNM_PATHNAME, false},
+	{"**/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"**/foo.txt", "bar/baz/foo.txt", FNM_PATHNAME, true},
+
+	{"foo.txt", "foo.TXT", 0, false},
+	{"foo.txt", "foo.TXT", FNM_CASEFOLD, true},
+}
+
+func TestMatch(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		testcases = append(testcases, testcase{"f\\[ab\\]o.txt", "f[ab]o.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\*.txt", "foo*.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", FNM_NOESCAPE, false})
+		testcases = append(testcases, testcase{"f\\\\\\[ab\\\\\\]o.txt", "f\\[ab\\]o.txt", 0, true})
+	}
+
+	for _, tc := range testcases {
+		if m, err := Match(tc.pat, filepath.FromSlash(tc.name), tc.flags); m != tc.match {
+			if err != nil {
+				t.Error(err)
+			} else {
+				t.Errorf("Match(%q, %q, %d) != %v", tc.pat, tc.name, tc.flags, tc.match)
+			}
+		}
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	if _, err := Match("foo[bar", "...", 0); err == nil {
+		t.Error("Unexpected nil error")
+	}
+}

gui/app.js

@@ -22,15 +22,21 @@ syncthing.config(function ($httpProvider, $translateProvider) {
 
 syncthing.controller('EventCtrl', function ($scope, $http) {
     $scope.lastEvent = null;
-    var online = false;
     var lastID = 0;
 
     var successFn = function (data) {
-        if (!online) {
-            $scope.$emit('UIOnline');
-            online = true;
+        // When Syncthing restarts while the long polling connection is in
+        // progress the browser on some platforms returns a 200 (since the
+        // headers has been flushed with the return code 200), with no data.
+        // This basically means that the connection has been reset, and the call
+        // was not actually sucessful.
+        if (!data) {
+            errorFn(data);
+            return;
         }
 
+        $scope.$emit('UIOnline');
+
         if (lastID > 0) {
             data.forEach(function (event) {
                 console.log("event", event.id, event.type, event.data);
@@ -49,10 +55,8 @@ syncthing.controller('EventCtrl', function ($scope, $http) {
     };
 
     var errorFn = function (data) {
-        if (online) {
-            $scope.$emit('UIOffline');
-            online = false;
-        }
+        $scope.$emit('UIOffline');
+
         setTimeout(function () {
             $http.get(urlbase + '/events?limit=1')
             .success(successFn)
@@ -68,6 +72,8 @@ syncthing.controller('EventCtrl', function ($scope, $http) {
 syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $location) {
     var prevDate = 0;
     var getOK = true;
+    var navigatingAway = false;
+    var online = false;
     var restarting = false;
 
     $scope.completion = {};
@@ -84,18 +90,44 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
     $scope.repos = {};
     $scope.seenError = '';
     $scope.upgradeInfo = {};
+    $scope.stats = {};
 
     $http.get(urlbase+"/lang").success(function (langs) {
-        var lang;
+        // Find the first language in the list provided by the user's browser
+        // that is a prefix of a language we have available. That is, "en"
+        // sent by the browser will match "en" or "en-US", while "zh-TW" will
+        // match only "zh-TW" and not "zh-CN".
+
+        var lang, matching;
         for (var i = 0; i < langs.length; i++) {
             lang = langs[i];
-            if (validLangs.indexOf(lang) >= 0) {
-                $translate.use(lang);
-                break;
+            if (lang.length < 2) {
+                continue;
+            }
+            matching = validLangs.filter(function (possibleLang) {
+                // The langs returned by the /rest/langs call will be in lower
+                // case. We compare to the lowercase version of the language
+                // code we have as well.
+                possibleLang = possibleLang.toLowerCase()
+                if (possibleLang.length > lang.length) {
+                    return possibleLang.indexOf(lang) == 0;
+                } else {
+                    return lang.indexOf(possibleLang) == 0;
+                }
+            });
+            if (matching.length >= 1) {
+                $translate.use(matching[0]);
+                return;
             }
         }
+        // Fallback if nothing matched
+        $translate.use("en");
     })
 
+    $(window).bind('beforeunload', function() {
+        navigatingAway = true;
+    });
+
     $scope.$on("$locationChangeSuccess", function () {
         var lang = $location.search().lang;
         if (lang) {
@@ -117,8 +149,13 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
     }
 
     $scope.$on('UIOnline', function (event, arg) {
+        if (online && !restarting) {
+            return;
+        }
+
         console.log('UIOnline');
         $scope.init();
+        online = true;
         restarting = false;
         $('#networkError').modal('hide');
         $('#restarting').modal('hide');
@@ -126,9 +163,14 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
     });
 
     $scope.$on('UIOffline', function (event, arg) {
+        if (navigatingAway || !online) {
+            return;
+        }
+
         console.log('UIOffline');
+        online = false;
         if (!restarting) {
-            $('#networkError').modal({backdrop: 'static', keyboard: false});
+            $('#networkError').modal();
         }
     });
 
@@ -157,6 +199,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
 
     $scope.$on('NodeDisconnected', function (event, arg) {
         delete $scope.connections[arg.data.id];
+        refreshNodeStats();
     });
 
     $scope.$on('NodeConnected', function (event, arg) {
@@ -168,6 +211,9 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
                 OutBytesTotal: 0,
                 Address: arg.data.addr,
             };
+            $scope.completion[arg.data.id] = {
+                _total: 100,
+            };
         }
     });
 
@@ -185,7 +231,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
                 document.cookie = "firstVisit=" + Date.now() + ";max-age=" + 30*24*3600;
             } else {
                 if (+firstVisit < Date.now() - 4*3600*1000){
-                    $('#ur').modal({backdrop: 'static', keyboard: false});
+                    $('#ur').modal();
                 }
             }
         }
@@ -284,6 +330,11 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
             $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
 
             $scope.nodes = $scope.config.Nodes;
+            $scope.nodes.forEach(function (nodeCfg) {
+                $scope.completion[nodeCfg.NodeID] = {
+                    _total: 100,
+                };
+            });
             $scope.nodes.sort(nodeCompare);
 
             $scope.repos = repoMap($scope.config.Repositories);
@@ -306,10 +357,18 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
         });
     }
 
+    var refreshNodeStats = debounce(function () {
+        $http.get(urlbase+"/stats/node").success(function (data) {
+            $scope.stats = data;
+            console.log("refreshNodeStats", data);
+        });
+    }, 500);
+
     $scope.init = function() {
         refreshSystem();
         refreshConfig();
         refreshConnectionStats();
+        refreshNodeStats();
 
         $http.get(urlbase + '/version').success(function (data) {
             $scope.version = data;
@@ -463,7 +522,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
         $scope.tmpOptions = angular.copy($scope.config.Options);
         $scope.tmpOptions.UREnabled = ($scope.tmpOptions.URAccepted > 0);
         $scope.tmpGUI = angular.copy($scope.config.GUI);
-        $('#settings').modal({backdrop: 'static', keyboard: true});
+        $('#settings').modal();
     };
 
     $scope.saveConfig = function() {
@@ -506,9 +565,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
 
     $scope.restart = function () {
         restarting = true;
-        $scope.restartingTitle = "Restarting"
-        $scope.restartingBody = "Syncthing is restarting."
-        $('#restarting').modal({backdrop: 'static', keyboard: false});
+        $('#restarting').modal();
         $http.post(urlbase + '/restart');
         $scope.configInSync = true;
 
@@ -529,21 +586,20 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
     };
 
     $scope.upgrade = function () {
-        $scope.restartingTitle = "Upgrading"
-        $scope.restartingBody = "Syncthing is upgrading."
-        $('#restarting').modal({backdrop: 'static', keyboard: false});
+        restarting = true;
+        $('#upgrading').modal();
         $http.post(urlbase + '/upgrade').success(function () {
-            restarting = true;
-            $scope.restartingBody = "Syncthing is restarting into the new version."
+            $('#restarting').modal();
+            $('#upgrading').modal('hide');
         }).error(function () {
-            $('#restarting').modal('hide');
+            $('#upgrading').modal('hide');
         });
     };
 
     $scope.shutdown = function () {
         restarting = true;
         $http.post(urlbase + '/shutdown').success(function () {
-            $('#shutdown').modal({backdrop: 'static', keyboard: false});
+            $('#shutdown').modal();
         });
         $scope.configInSync = true;
     };
@@ -554,7 +610,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
         $scope.editingSelf = (nodeCfg.NodeID == $scope.myID);
         $scope.currentNode.AddressesStr = nodeCfg.Addresses.join(', ');
         $scope.nodeEditor.$setPristine();
-        $('#editNode').modal({backdrop: 'static', keyboard: true});
+        $('#editNode').modal();
     };
 
     $scope.idNode = function () {
@@ -566,7 +622,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
         $scope.editingExisting = false;
         $scope.editingSelf = false;
         $scope.nodeEditor.$setPristine();
-        $('#editNode').modal({backdrop: 'static', keyboard: true});
+        $('#editNode').modal();
     };
 
     $scope.deleteNode = function () {
@@ -669,19 +725,44 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
         });
         if ($scope.currentRepo.Versioning && $scope.currentRepo.Versioning.Type === "simple") {
             $scope.currentRepo.simpleFileVersioning = true;
+            $scope.currentRepo.FileVersioningSelector = "simple";
             $scope.currentRepo.simpleKeep = +$scope.currentRepo.Versioning.Params.keep;
+        } else if ($scope.currentRepo.Versioning && $scope.currentRepo.Versioning.Type === "staggered") {
+            $scope.currentRepo.staggeredFileVersioning = true;
+            $scope.currentRepo.FileVersioningSelector = "staggered";
+            $scope.currentRepo.staggeredMaxAge = Math.floor(+$scope.currentRepo.Versioning.Params.maxAge / 86400);
+            $scope.currentRepo.staggeredCleanInterval = +$scope.currentRepo.Versioning.Params.cleanInterval;
+            $scope.currentRepo.staggeredVersionsPath = $scope.currentRepo.Versioning.Params.versionsPath;
+        } else {
+            $scope.currentRepo.FileVersioningSelector = "none";
         }
         $scope.currentRepo.simpleKeep = $scope.currentRepo.simpleKeep || 5;
+        $scope.currentRepo.staggeredCleanInterval = $scope.currentRepo.staggeredCleanInterval || 3600;
+        $scope.currentRepo.staggeredVersionsPath = $scope.currentRepo.staggeredVersionsPath || "";
+
+        // staggeredMaxAge can validly be zero, which we should not replace
+        // with the default value of 365. So only set the default if it's
+        // actually undefined.
+        if (typeof $scope.currentRepo.staggeredMaxAge === 'undefined') {
+            $scope.currentRepo.staggeredMaxAge = 365;
+        }
+
         $scope.editingExisting = true;
         $scope.repoEditor.$setPristine();
-        $('#editRepo').modal({backdrop: 'static', keyboard: true});
+        $('#editRepo').modal();
     };
 
     $scope.addRepo = function () {
         $scope.currentRepo = {selectedNodes: {}};
+        $scope.currentRepo.RescanIntervalS = 60;
+        $scope.currentRepo.FileVersioningSelector = "none";
+        $scope.currentRepo.simpleKeep = 5;
+        $scope.currentRepo.staggeredMaxAge = 365;
+        $scope.currentRepo.staggeredCleanInterval = 3600;
+        $scope.currentRepo.staggeredVersionsPath = "";
         $scope.editingExisting = false;
         $scope.repoEditor.$setPristine();
-        $('#editRepo').modal({backdrop: 'static', keyboard: true});
+        $('#editRepo').modal();
     };
 
     $scope.saveRepo = function () {
@@ -698,7 +779,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
         }
         delete repoCfg.selectedNodes;
 
-        if (repoCfg.simpleFileVersioning) {
+        if (repoCfg.FileVersioningSelector === "simple") {
             repoCfg.Versioning = {
                 'Type': 'simple',
                 'Params': {
@@ -707,6 +788,20 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
             };
             delete repoCfg.simpleFileVersioning;
             delete repoCfg.simpleKeep;
+        } else if (repoCfg.FileVersioningSelector === "staggered") {
+            repoCfg.Versioning = {
+                'Type': 'staggered',
+                'Params': {
+                    'maxAge': '' + (repoCfg.staggeredMaxAge * 86400),
+                    'cleanInterval': '' + repoCfg.staggeredCleanInterval,
+                    'versionsPath': '' + repoCfg.staggeredVersionsPath,
+                }
+            };
+            delete repoCfg.staggeredFileVersioning;
+            delete repoCfg.staggeredMaxAge;
+            delete repoCfg.staggeredCleanInterval;
+            delete repoCfg.staggeredVersionsPath;
+
         } else {
             delete repoCfg.Versioning;
         }
@@ -742,8 +837,6 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
         cfg.APIKey = randomString(30, 32);
     };
 
-
-
     $scope.acceptUR = function () {
         $scope.config.Options.URAccepted = 1000; // Larger than the largest existing report version
         $scope.saveConfig();
@@ -758,7 +851,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
 
     $scope.showNeed = function (repo) {
         $scope.neededLoaded = false;
-        $('#needed').modal({backdrop: 'static', keyboard: true});
+        $('#needed').modal();
         $http.get(urlbase + "/need?repo=" + encodeURIComponent(repo)).success(function (data) {
             $scope.needed = data;
             $scope.neededLoaded = true;
@@ -781,15 +874,21 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
     };
 
     $scope.override = function (repo) {
-        $http.post(urlbase + "/model/override?repo=" + encodeURIComponent(repo)).success(function () {
-            $scope.refresh();
-        });
+        $http.post(urlbase + "/model/override?repo=" + encodeURIComponent(repo));
     };
 
     $scope.about = function () {
         $('#about').modal('show');
     };
 
+    $scope.showReportPreview = function () {
+        $scope.reportPreview = true;
+    };
+
+    $scope.rescanRepo = function (repo) {
+        $http.post(urlbase + "/scan?repo=" + encodeURIComponent(repo));
+    };
+
     $scope.init();
     setInterval($scope.refresh, 10000);
 });
@@ -1035,7 +1134,6 @@ syncthing.directive('validNodeid', function($http) {
                         if (resp.error) {
                             ctrl.$setValidity('validNodeid', false);
                         } else {
-                            scope.currentNode.NodeID = resp.id;
                             ctrl.$setValidity('validNodeid', true);
                         }
                     });

gui/index.html

@@ -15,12 +15,17 @@
 
   <title>Syncthing | {{thisNodeName()}}</title>
   <link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
+  <link href="raleway.css" rel="stylesheet">
   <style type="text/css">
     body {
       padding-bottom: 70px;
       font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
     }
 
+    h1, h2, h3, h4, h5 {
+      font-family: "Raleway", "Helvetica Neue", Helvetica, Arial, sans-serif;
+    }
+
     ul+h5 {
       margin-top: 1.5em;
     }
@@ -77,10 +82,17 @@
       white-space: nowrap;
     }
 
+    table.table-condensed {
+      table-layout: fixed;
+    }
+    table.table-condensed td {
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+    }
     @media (max-width:767px) {
-      .table-responsive>.table>tbody>tr>td {
-        /* revert a bootstrap setting e.g.:
-         * for mobile phones to allow linebreaks in long repro folder/shared with
+      table.table-condensed td {
+        /* for mobile phones to allow linebreaks in long repro folder/shared with
          * columns. */
         white-space: normal;
       }
@@ -95,7 +107,8 @@
 
   <nav class="navbar navbar-top navbar-default" role="navigation">
     <div class="container">
-      <span class="navbar-brand"><img class="logo" src="st-logo-128.png" width="32" height="32" /> Syncthing<small class="hidden-xs"> <span class="text-muted">|</span> {{thisNodeName()}}</small></span>
+      <span class="navbar-brand"><img class="logo" src="logo-text-64.png" height="32" width="117"/></span>
+      <p class="navbar-text hidden-xs">{{thisNodeName()}}</p>
       <ul class="nav navbar-nav navbar-right">
         <li ng-if="upgradeInfo.newer">
           <button type="button" class="btn navbar-btn btn-default" href="" ng-click="upgrade()">
@@ -153,7 +166,7 @@
             <div class="panel-heading">
               <h3 class="panel-title">
                 <a data-toggle="collapse" data-parent="#repositories" href="#repo-{{$index}}">
-                  <span class="glyphicon glyphicon-hdd"></span> {{repo.Directory | shortPath}}
+                  <span class="glyphicon glyphicon-hdd"></span> {{repo.ID}}
                   <span class="pull-right hidden-xs" ng-switch="repoStatus(repo.ID)">
                     <span translate ng-switch-when="unknown">Unknown</span>
                     <span translate ng-switch-when="stopped">Stopped</span>
@@ -172,7 +185,6 @@
             </div>
             <div id="repo-{{$index}}" class="panel-collapse collapse">
               <div class="panel-body">
-                <div class="table-responsive">
                   <table class="table table-condensed table-striped">
                     <tbody>
                       <tr>
@@ -189,16 +201,16 @@
                       </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-globe"></span>&emsp;<span translate>Global Repository</span></th>
-                        <td class="text-right">{{model[repo.ID].globalFiles | alwaysNumber}} <span translate>items</span>, {{model[repo.ID].globalBytes | binary}}B</td>
+                        <td class="text-right">{{model[repo.ID].globalFiles | alwaysNumber}} <span translate>items</span>, ~{{model[repo.ID].globalBytes | binary}}B</td>
                       </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-home"></span>&emsp;<span translate>Local Repository</span></th>
-                        <td class="text-right">{{model[repo.ID].localFiles | alwaysNumber}} <span translate>items</span>, {{model[repo.ID].localBytes | binary}}B</td>
+                        <td class="text-right">{{model[repo.ID].localFiles | alwaysNumber}} <span translate>items</span>, ~{{model[repo.ID].localBytes | binary}}B</td>
                       </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-cloud-download"></span>&emsp;<span translate>Out Of Sync</span></th>
                         <td class="text-right">
-                        <a ng-if="model[repo.ID].needFiles > 0" ng-click="showNeed(repo.ID)" href="">{{model[repo.ID].needFiles | alwaysNumber}} <span translate>items</span>, {{model[repo.ID].needBytes | binary}}B</a>
+                        <a ng-if="model[repo.ID].needFiles > 0" ng-click="showNeed(repo.ID)" href="">{{model[repo.ID].needFiles | alwaysNumber}} <span translate>items</span>, ~{{model[repo.ID].needBytes | binary}}B</a>
                         <span ng-if="model[repo.ID].needFiles == 0">0 <span translate>items</span>, 0 B</span>
                         </td>
                       </tr>
@@ -216,14 +228,18 @@
                           <span translate ng-if="!repo.IgnorePerms">No</span>
                         </td>
                       </tr>
+                      <tr>
+                        <th><span class="glyphicon glyphicon-refresh"></span>&emsp;<span translate>Rescan Interval</span></th>
+                        <td class="text-right">{{repo.RescanIntervalS}} s</td>
+                      </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-share-alt"></span>&emsp;<span translate>Shared With</span></th>
                         <td class="text-right">{{sharesRepo(repo)}}</td>
                       </tr>
                     </tbody>
                   </table>
-                </div>
                 <span class="pull-right">
+                  <a class="btn btn-sm btn-default" href="" ng-show="repoStatus(repo.ID) == 'idle'" ng-click="rescanRepo(repo.ID)"><span class="glyphicon glyphicon-refresh"></span>&emsp;<span translate>Rescan</span></a>
                   <a class="btn btn-sm btn-primary" href="" ng-click="editRepo(repo)"><span class="glyphicon glyphicon-pencil"></span>&emsp;<span translate>Edit</span></a>
                   <a class="btn btn-sm btn-danger" ng-if="repo.ReadOnly && model[repo.ID].needFiles > 0" ng-click="override(repo.ID)" href=""><span class="glyphicon glyphicon-upload"></span>&emsp;<span translate>Override Changes</span></a>
                 </span>
@@ -245,7 +261,6 @@
             </div>
             <div id="node-this" class="panel-collapse collapse in">
               <div class="panel-body">
-                <div class="table-responsive">
                   <table class="table table-condensed table-striped">
                     <tbody>
                       <tr>
@@ -277,7 +292,6 @@
                       </tr>
                     </tbody>
                   </table>
-                </div>
                 <span class="pull-right"><a class="btn btn-sm btn-primary" href="" ng-click="editNode(nodeCfg)"><span class="glyphicon glyphicon-pencil"></span>&emsp;<span translate>Edit</span></a></span>
               </div>
             </div>
@@ -303,7 +317,6 @@
             </div>
             <div id="node-{{$index}}" class="panel-collapse collapse">
               <div class="panel-body">
-                <div class="table-responsive">
                   <table class="table table-condensed table-striped">
                     <tbody>
                       <tr>
@@ -331,9 +344,13 @@
                         <th><span class="glyphicon glyphicon-tag"></span>&emsp;<span translate>Version</span></th>
                         <td class="text-right">{{nodeVer(nodeCfg)}}</td>
                       </tr>
+                      <tr ng-if="!connections[nodeCfg.NodeID]">
+                        <th><span class="glyphicon glyphicon-eye-open"></span>&emsp;<span translate>Last seen</span></th>
+                        <td translate ng-if="stats[nodeCfg.NodeID].LastSeen.indexOf('1970') > -1" class="text-right">Never</td>
+                        <td ng-if="stats[nodeCfg.NodeID].LastSeen.indexOf('1970') < 0" class="text-right">{{stats[nodeCfg.NodeID].LastSeen | date:"yyyy-MM-dd HH:mm"}}</td>
+                      </tr>
                     </tbody>
                   </table>
-                </div>
                 <span class="pull-right"><a class="btn btn-sm btn-primary" href="" ng-click="editNode(nodeCfg)"><span class="glyphicon glyphicon-pencil"></span>&emsp;<span translate>Edit</span></a></span>
               </div>
             </div>
@@ -367,10 +384,10 @@
     <div class="container">
       <ul class="nav navbar-nav">
         <li><a class="navbar-link" href="http://discourse.syncthing.net/"><span translate>Support / Forum</span></a></li>
-        <li><a class="navbar-link" href="https://github.com/calmh/syncthing/releases"><span translate>Latest Release</span></a></li>
+        <li><a class="navbar-link" href="https://github.com/syncthing/syncthing/releases"><span translate>Latest Release</span></a></li>
         <li><a class="navbar-link" href="http://discourse.syncthing.net/category/documentation"><span translate>Documentation</span></a></li>
-        <li><a class="navbar-link" href="https://github.com/calmh/syncthing/issues"><span translate>Bugs</span></a></li>
-        <li><a class="navbar-link" href="https://github.com/calmh/syncthing"><span translate>Source Code</span></a></li>
+        <li><a class="navbar-link" href="https://github.com/syncthing/syncthing/issues"><span translate>Bugs</span></a></li>
+        <li><a class="navbar-link" href="https://github.com/syncthing/syncthing"><span translate>Source Code</span></a></li>
       </ul>
     </div>
   </nav>
@@ -385,8 +402,14 @@
 
   <!-- Restarting modal -->
 
-  <modal id="restarting" icon="refresh" title="{{restartingTitle}}" status="info">
-    <p>{{restartingBody}} <span translate>Please wait</span>&hellip;</p>
+  <modal id="restarting" icon="refresh" title="{{'Restarting' | translate}}" status="info">
+    <p><span translate>Syncthing is restarting.</span> <span translate>Please wait</span>...</p>
+  </modal>
+
+  <!-- Upgrading modal -->
+
+  <modal id="upgrading" icon="refresh" title="{{'Upgrading' | translate}}" status="info">
+    <p><span translate>Syncthing is upgrading.</span> <span translate>Please wait</span>...</p>
   </modal>
 
   <!-- Shutdown modal -->
@@ -399,12 +422,12 @@
 
   <modal id="idqr" large="yes" status="info" close="yes" icon="qrcode" title="{{'Node Identification' | translate}} &mdash; {{nodeName(thisNode())}}">
     <div class="well well-sm text-monospace text-center">{{myID}}</div>
-    <img ng-if="myID" class="center-block img-thumbnail" src="qr/{{myID}}"/>
+    <img ng-if="myID" class="center-block img-thumbnail" src="qr/?text={{myID}}"/>
   </modal>
 
   <!-- Node editor modal -->
 
-  <div id="editNode" class="modal fade">
+  <div id="editNode" class="modal fade" tabindex="-1">
     <div class="modal-dialog modal-lg">
       <div class="modal-content">
         <div class="modal-header">
@@ -421,13 +444,14 @@
                 <span translate ng-if="nodeEditor.nodeID.$valid || nodeEditor.nodeID.$pristine">The node ID to enter here can be found in the "Edit > Show ID" dialog on the other node. Spaces and dashes are optional (ignored).</span>
                 <span translate ng-show="!editingExisting && (nodeEditor.nodeID.$valid || nodeEditor.nodeID.$pristine)">When adding a new node, keep in mind that this node must be added on the other side too.</span>
                 <span translate ng-if="nodeEditor.nodeID.$error.required && nodeEditor.nodeID.$dirty">The node ID cannot be blank.</span>
-                <span translate ng-if="nodeEditor.nodeID.$error.validNodeid && nodeEditor.nodeID.$dirty">The entered node ID does not look valid. It should be a 52 character string consisting of letters and numbers, with spaces and dashes being optional.</span>
+                <span translate ng-if="nodeEditor.nodeID.$error.validNodeid && nodeEditor.nodeID.$dirty">The entered node ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.</span>
               </p>
             </div>
             <div class="form-group">
               <label translate for="name">Node Name</label>
               <input placeholder="Home Server" id="name" class="form-control" type="text" ng-model="currentNode.Name"></input>
-              <p translate class="help-block">Shown instead of Node ID in the cluster status.</p>
+              <p translate ng-if="currentNode.NodeID == myID" class="help-block">Shown instead of Node ID in the cluster status. Will be advertised to other nodes as an optional default name.</p>
+              <p translate ng-if="currentNode.NodeID != myID" class="help-block">Shown instead of Node ID in the cluster status. Will be updated to the name the node advertises if left empty.</p>
             </div>
             <div class="form-group">
               <label translate for="addresses">Addresses</label>
@@ -454,7 +478,7 @@
 
   <!-- Repo editor modal -->
 
-  <div id="editRepo" class="modal fade">
+  <div id="editRepo" class="modal fade" tabindex="-1">
     <div class="modal-dialog modal-lg">
       <div class="modal-content">
         <div class="modal-header">
@@ -477,12 +501,19 @@
                 </div>
                 <div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
                   <label translate for="repoPath">Repository Path</label>
-                  <input name="repoPath" placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
+                  <input name="repoPath" placeholder="~/Documents" ng-disabled="editingExisting" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
                   <p class="help-block">
                     <span translate ng-if="repoEditor.repoPath.$valid || repoEditor.repoPath.$pristine">Path to the repository on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for</span> <code>{{system.tilde}}</code>.
                     <span translate ng-if="repoEditor.repoPath.$error.required && repoEditor.repoPath.$dirty">The repository path cannot be blank.</span>
                   </p>
                 </div>
+                <div class="form-group" ng-class="{'has-error': repoEditor.rescanIntervalS.$invalid && repoEditor.rescanIntervalS.$dirty}">
+                  <label for="rescanIntervalS"><span translate>Rescan Interval</span> (s)</label>
+                  <input name="rescanIntervalS" placeholder="60" id="rescanIntervalS" class="form-control" type="number" ng-model="currentRepo.RescanIntervalS" required min="5"></input>
+                  <p class="help-block">
+                    <span translate ng-if="!repoEditor.rescanIntervalS.$valid && repoEditor.rescanIntervalS.$dirty">The rescan interval must be at least 5 seconds.</span>
+                  </p>
+                </div>
               </div>
             </div>
             <div class="row">
@@ -515,14 +546,25 @@
               </div>
               <div class="col-md-6">
                 <div class="form-group">
-                  <div class="checkbox">
+                  <label translate>File Versioning</label>
+                  <div class="radio">
                     <label>
-                      <input type="checkbox" ng-model="currentRepo.simpleFileVersioning"> <span translate>File Versioning</span>
+                      <input type="radio" ng-model="currentRepo.FileVersioningSelector" value="none"> <span translate>No File Versioning</span>
+                    </label>
+                  </div>
+                  <div class="radio">
+                    <label>
+                      <input type="radio" ng-model="currentRepo.FileVersioningSelector" value="simple"> <span translate>Simple File Versioning</span>
+                    </label>
+                  </div>
+                  <div class="radio">
+                    <label>
+                      <input type="radio" ng-model="currentRepo.FileVersioningSelector" value="staggered"> <span translate>Staggered File Versioning</span>
                     </label>
                   </div>
-                  <p translate class="help-block">Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.</p>
                 </div>
-                <div class="form-group" ng-if="currentRepo.simpleFileVersioning" ng-class="{'has-error': repoEditor.simpleKeep.$invalid && repoEditor.simpleKeep.$dirty}">
+                <div class="form-group" ng-if="currentRepo.FileVersioningSelector=='simple'" ng-class="{'has-error': repoEditor.simpleKeep.$invalid && repoEditor.simpleKeep.$dirty}">
+                  <p translate class="help-block">Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.</p>
                   <label translate for="simpleKeep">Keep Versions</label>
                   <input name="simpleKeep" id="simpleKeep" class="form-control" type="number" ng-model="currentRepo.simpleKeep" required min="1"></input>
                   <p class="help-block">
@@ -531,7 +573,21 @@
                     <span translate ng-if="repoEditor.simpleKeep.$error.min && repoEditor.simpleKeep.$dirty">You must keep at least one version.</span>
                   </p>
                 </div>
-
+                <div class="form-group" ng-if="currentRepo.FileVersioningSelector=='staggered'" ng-class="{'has-error': repoEditor.staggeredMaxAge.$invalid && repoEditor.staggeredMaxAge.$dirty}">
+                  <p class="help-block"><span translate>Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.</span> <span translate>Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.</span></p>
+                  <p translate class="help-block">The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.</p>
+                  <label translate for="staggeredMaxAge">Maximum Age</label>
+                  <input name="staggeredMaxAge" id="staggeredMaxAge" class="form-control" type="number" ng-model="currentRepo.staggeredMaxAge" required></input>
+                  <p class="help-block">
+                    <span translate ng-if="repoEditor.staggeredMaxAge.$valid || repoEditor.staggeredMaxAge.$pristine">The maximum time to keep a version (in days, set to 0 to keep versions forever).</span>
+                    <span translate ng-if="repoEditor.staggeredMaxAge.$error.required && repoEditor.staggeredMaxAge.$dirty">The maximum age must be a number and cannot be blank.</span>
+                  </p>
+                </div>
+                <div class="form-group" ng-if="currentRepo.FileVersioningSelector == 'staggered'">
+                  <label translate for="staggeredVersionsPath">Versions Path</label>
+                  <input name="staggeredVersionsPath" placeholder="" id="staggeredVersionsPath" class="form-control" type="text" ng-model="currentRepo.staggeredVersionsPath"></input>
+                  <p translate class="help-block">Path where versions should be stored (leave empty for the default .stversions folder in the repository).</p>
+                </div>
               </div>
             </div>
           </form>
@@ -548,7 +604,7 @@
 
   <!-- Settings modal -->
 
-  <div id="settings" class="modal fade">
+  <div id="settings" class="modal fade" tabindex="-1">
     <div class="modal-dialog modal-lg">
       <div class="modal-content">
         <div class="modal-header">
@@ -567,10 +623,6 @@
                   <label translate for="MaxSendKbps">Outgoing Rate Limit (KiB/s)</label>
                   <input id="MaxSendKbps" class="form-control" type="number" ng-model="tmpOptions.MaxSendKbps">
                 </div>
-                <div class="form-group">
-                  <label translate for="RescanIntervalS">Rescan Interval (s)</label>
-                  <input id="RescanIntervalS" class="form-control" type="number" ng-model="tmpOptions.RescanIntervalS">
-                </div>
                 <!--
                 <div class="form-group">
                   <label translate for="ReconnectIntervalS">Reconnect Interval (s)</label>
@@ -600,10 +652,6 @@
                     </label>
                   </div>
                 </div>
-                <div class="form-group">
-                  <label translate for="LocalAnnPort">Local Discovery Port</label>
-                  <input ng-disabled="!tmpOptions.LocalAnnEnabled" id="LocalAnnPort" class="form-control" type="number" ng-model="tmpOptions.LocalAnnPort">
-                </div>
                 <div class="form-group">
                   <div class="checkbox">
                     <label>
@@ -674,7 +722,7 @@
 
   <!-- Usage report modal -->
 
-  <div id="ur" class="modal fade">
+  <div id="ur" class="modal fade" data-backdrop="static" data-keyboard="false" tabindex="-1">
     <div class="modal-dialog modal-lg">
       <div class="modal-content">
         <div class="modal-header alert alert-success">
@@ -682,8 +730,8 @@
         </div>
         <div class="modal-body">
         <p translate>The encrypted usage report is sent daily. It is used to track common platforms, repo sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
-        <p translate translate-value-url="https://data.syncthing.net">The aggregated statistics are publicly available at {{url}}</p>
-        <button translate type="button" class="btn btn-default" ng-show="!reportPreview" ng-click="reportPreview = true">Preview Usage Report</button>
+        <p translate translate-value-url="https://data.syncthing.net">The aggregated statistics are publicly available at {%url%}.</p>
+        <button translate type="button" class="btn btn-default" ng-show="!reportPreview" ng-click="showReportPreview()">Preview Usage Report</button>
         <pre ng-if="reportPreview"><small>{{reportData | json}}</small></pre>
         </div>
         <div class="modal-footer">
@@ -709,7 +757,7 @@
   <!-- About modal -->
 
   <modal id="about" large="yes" close="yes" status="info" title="About">
-    <h1 class="text-center"><img src="st-logo-128.png" style="vertical-align: -16px" width="64" height="64"/> Syncthing<br/><small>{{version}}</small></h1>
+    <h1 class="text-center"><img alt="Syncthing" title="Syncthing" src="logo-text-256.png" style="vertical-align: -16px" height="100" width="366"/><br/><small>{{version}}</small></h1>
     <hr/>
 
     <p translate>Copyright &copy; 2014 Jakob Borg and the following Contributors:</p>
@@ -718,18 +766,22 @@
         <ul>
           <li>Aaron Bieber</li>
           <li>Andrew Dunham</li>
+          <li>Alexander Graf</li>
           <li>Arthur Axel fREW Schmidt</li>
           <li>Audrius Butkevicius</li>
           <li>Ben Sidhom</li>
           <li>Brandon Philips</li>
+          <li>Gilli Sigurdsson</li>
         </ul>
         </div>
       <div class="col-md-6">
         <ul>
           <li>James Patterson</li>
           <li>Jens Diemer</li>
+          <li>Marcin Dziadus</li>
           <li>Philippe Schommers</li>
           <li>Ryan Sullivan</li>
+          <li>Tully Robinson</li>
           <li>Veeti Paananen</li>
         </ul>
         </div>
@@ -738,7 +790,7 @@
 
     <p translate>Syncthing includes the following software or portions thereof:</p>
     <ul>
-      <li><a href="http://golang.org/">The Go Programming Languange</a>, Copyright &copy; 2012 The Go Authors.</li>
+      <li><a href="http://golang.org/">The Go Programming Language</a>, Copyright &copy; 2012 The Go Authors.</li>
       <li><a href="https://bitbucket.org/kardianos/osext">kardianos/osext</a>, Copyright &copy; 2012 Daniel Theophanes.</li>
       <li><a href="https://code.google.com/p/snappy-go/">snappy-go</a>, Copyright &copy; 2011 The Snappy-Go Authors.</li>
       <li><a href="https://github.com/golang/groupcache">groupcache/lru</a>, Copyright &copy; 2013 Google Inc.</li>

gui/lang-bg.json

@@ -0,0 +1,137 @@
+{
+    "API Key": "API Ключ",
+    "About": "За Програмата",
+    "Add Node": "Добави Машина",
+    "Add Repository": "Добави Папка",
+    "Address": "Адрес",
+    "Addresses": "Адреси",
+    "Allow Anonymous Usage Reporting?": "Разреши анонимен доклад за ползване на програмата?",
+    "Announce Server": "Announce Server",
+    "Anonymous Usage Reporting": "Анонимен Доклад",
+    "Bugs": "Бъгове",
+    "CPU Utilization": "Натоварване на Процесора",
+    "Close": "Затвори",
+    "Connection Error": "Грешка при Свързването",
+    "Copyright © 2014 Jakob Borg and the following Contributors:": "Правата запазени © 2014 Jakob Borg и следните Сътрудници:",
+    "Delete": "Изтрий",
+    "Disconnected": "Прекрати Връзката",
+    "Documentation": "Документация",
+    "Download Rate": "Скорост на Теглене",
+    "Edit": "Промени",
+    "Edit Node": "Промени Машината",
+    "Edit Repository": "Промени Папката",
+    "Enable UPnP": "Включи UPnP",
+    "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Въведи \"ip:port\" адреси разделени със запетая или \"dynamic\", за да извършиш автоматична връзка на адреси.",
+    "Error": "Грешка",
+    "File Versioning": "Файлови Версии",
+    "File permission bits are ignored when looking for changes. Use on FAT filesystems.": "Битовете за права за достъп са игнорирани, когато се проверява за промени. Използвай с файлови системи тип FAT.",
+    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.": "Когато syncthing замени или изтрие файл той се премества в  .stversions и преименува с дабавени дата и час.",
+    "Files are protected from changes made on other nodes, but changes made on this node will be sent to the rest of the cluster.": "Файловете са защитени от промени направени на други машини, но промени направени на тази машина ще бъдат синхронизирани до другите машини.",
+    "Folder": "Папка",
+    "GUI Authentication Password": "Парола за Потребителския Интерфейс",
+    "GUI Authentication User": "Потребител за Потребителския Интерфейс",
+    "GUI Listen Addresses": "Адрес за Свързване с Потребителския Интерфейс",
+    "Generate": "Генерирай",
+    "Global Discovery": "Глобавно Откриване",
+    "Global Discovery Server": "Сървър за Глобално Откриване",
+    "Global Repository": "Глобална Папка",
+    "Idle": "Без Работа",
+    "Ignore Permissions": "Игнорирай Права за Достъп",
+    "Keep Versions": "Пази Версии",
+    "Last seen": "Last seen",
+    "Latest Release": "Най-новата Версия",
+    "Local Discovery": "Локално Откриване",
+    "Local Discovery Port": "Порт за Локално Откриване",
+    "Local Repository": "Локална Папка",
+    "Master Repo": "Главна Папка",
+    "Max File Change Rate (KiB/s)": "Макс. Скорост на Промяна (KiB/s)",
+    "Max Outstanding Requests": "Макс. Неизпълени Заявки",
+    "Maximum Age": "Maximum Age",
+    "Never": "Never",
+    "No": "Не",
+    "No File Versioning": "No File Versioning",
+    "Node ID": "Код на Машината",
+    "Node Identification": "Идентификация на Машината",
+    "Node Name": "Име на Машината",
+    "Notice": "Известие",
+    "OK": "ОК",
+    "Offline": "Не е на линия",
+    "Online": "На линия",
+    "Out Of Sync": "Не Синхронизиран",
+    "Outgoing Rate Limit (KiB/s)": "Лимит на Изходящата Скорост (KiB/s)",
+    "Override Changes": "Замени Промените",
+    "Path to the repository on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Пътят до папката на този компютър. Ще бъде създадена ако не съществува. Символът тилда (~) може да бъде използван като заместител на",
+    "Path where versions should be stored (leave empty for the default .stversions folder in the repository).": "Path where versions should be stored (leave empty for the default .stversions folder in the repository).",
+    "Please wait": "Моля изчакай",
+    "Preview Usage Report": "Разгледай Доклада за Използване",
+    "RAM Utilization": "RAM Натоварване",
+    "Reconnect Interval (s)": "Интервал(и) на Свързване",
+    "Repository ID": "Идентификатор на Папката",
+    "Repository Master": "Главна Папка",
+    "Repository Path": "Път до Папката",
+    "Rescan": "Повторно Сканиране",
+    "Rescan Interval": "Rescan Interval",
+    "Rescan Interval (s)": "Интеравал(и) на Сканиране",
+    "Restart": "Рестартирай",
+    "Restart Needed": "Изискава се Рестартиране",
+    "Restarting": "Рестартиране",
+    "Save": "Запази",
+    "Scanning": "Сканиране",
+    "Select the nodes to share this repository with.": "Избери компютрите, с които да споделиш тази папка.",
+    "Settings": "Настройки",
+    "Share With Nodes": "Сподели с Компютри",
+    "Shared With": "Сподел С",
+    "Short identifier for the repository. Must be the same on all cluster nodes.": "Кратък идентификатор на папката. Трябва да бъде същият на всички компютри.",
+    "Show ID": "Покажи Идентификатора",
+    "Shown instead of Node ID in the cluster status.": "Покажи вмест ID-то на Компютъра в статус на клъстъра.",
+    "Shown instead of Node ID in the cluster status. Will be advertised to other nodes as an optional default name.": "Покажи вмест ID-то на Компютъра в статус на клъстъра. Ще бъде предлагано на други комютри като име по подразбиране.",
+    "Shown instead of Node ID in the cluster status. Will be updated to the name the node advertises if left empty.": "Покажи вмест ID-то на Компютъра в статус на клъстъра. Ще бъде обновено с името по подразбиране изпратено от другия компютър.",
+    "Shutdown": "Спри Програмата",
+    "Simple File Versioning": "Simple File Versioning",
+    "Source Code": "Сорс Код",
+    "Staggered File Versioning": "Staggered File Versioning",
+    "Start Browser": "Стартирай Браузъра",
+    "Stopped": "Спряна",
+    "Support / Forum": "Помощ / Форум",
+    "Sync Protocol Listen Addresses": "Адрес за слушане на синхронизиращия протокол",
+    "Synchronization": "Синхронизация",
+    "Syncing": "Синхронизиране",
+    "Syncthing has been shut down.": "Syncthing е спрян.",
+    "Syncthing includes the following software or portions thereof:": "Syncthing включва следният софтуер пълно или частично:",
+    "Syncthing is restarting.": "Syncthing се рестартирва",
+    "Syncthing is upgrading.": "Syncthing се обновява.",
+    "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing изглежда не е включен, или има проблем с интерент връзката. Повторен опит...",
+    "The aggregated statistics are publicly available at {%url%}.": "Сумарната статистика е публично достъпна на {{url}}.",
+    "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "Конфигурацията е запазена, но не е активирана. Syncthing трябва да рестартира, за да се активира новата конфигурация.",
+    "The encrypted usage report is sent daily. It is used to track common platforms, repo sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Криптираният доклад се изпраща дневно. Използва се, за да следи общи платформи, размери на папки и версии на приложението. Ако събираните данни се променят, ще бъдете информиран с подобен на този диалог.",
+    "The entered node ID does not look valid. It should be a 52 character string consisting of letters and numbers, with spaces and dashes being optional.": "Въведни код на машината не е валиден. Трябва да бъде 52 символа и да се състои от букви, цифри като интервалите и тиретата са пожелание.",
+    "The entered node ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Въведни код на машината не е валиден. Трябва да бъде 52 или 56 символа и да се състои от букви, цифри като интервалите и тиретата са пожелание.",
+    "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.",
+    "The maximum age must be a number and cannot be blank.": "The maximum age must be a number and cannot be blank.",
+    "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "The maximum time to keep a version (in days, set to 0 to keep versions forever).",
+    "The node ID cannot be blank.": "Кодът на машината не може да бъде празен.",
+    "The node ID to enter here can be found in the \"Edit > Show ID\" dialog on the other node. Spaces and dashes are optional (ignored).": "Кодът на машината, който си въвел може да бъде намерен в \"Промени > Покажи Идентификатора\". Интервалите и тиретата са пожелание(биват прескачани).",
+    "The number of old versions to keep, per file.": "Броят стари версии, които да бъдат пазени за всеки файл.",
+    "The number of versions must be a number and cannot be blank.": "Броят версии трябва да бъде число и не може да бъде празно.",
+    "The repository ID cannot be blank.": "Полето идентификатор на папка не може д абъде празно.",
+    "The repository ID must be a short identifier (64 characters or less) consisting of letters, numbers and the the dot (.), dash (-) and underscode (_) characters only.": "Идентификаторът на папка трябва да бъде къс(64 символа или по-малко) състоящ се само от букви, цифри, точка(.), тире(-) и подчерта (_).",
+    "The repository ID must be unique.": "Идентификаторът на папката тряба да бъде уникален.",
+    "The repository path cannot be blank.": "Пътят до папката не може да бъде празен.",
+    "The rescan interval must be at least 5 seconds.": "The rescan interval must be at least 5 seconds.",
+    "Unknown": "Неясен",
+    "Up to Date": "Актуален",
+    "Upgrade To {%version%}": "Обновен До {{version}}",
+    "Upgrading": "Обновяване",
+    "Upload Rate": "Скорост на Качване",
+    "Usage": "Употреба",
+    "Use Compression": "Използвай Компресиране",
+    "Use HTTPS for GUI": "Използвай HTTPS за Потребителския Интерфейс",
+    "Version": "Версия",
+    "Versions Path": "Versions Path",
+    "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.",
+    "When adding a new node, keep in mind that this node must be added on the other side too.": "Когато добавяш нова машина помни, че твоята машина също трябва да бъде добавена от другата страна.",
+    "When adding a new repository, keep in mind that the Repository ID is used to tie repositories together between nodes. They are case sensitive and must match exactly between all nodes.": "Когато добавяш нов идентификатор на папка помни, че той се използва за свързване на папките на различни машини. Главни/малки букви са от значение и трябва да са еднакви на всички машини.",
+    "Yes": "Да",
+    "You must keep at least one version.": "Трябва да пазиш поне една версия.",
+    "items": "артикула"
+}

gui/lang-da.json

@@ -0,0 +1,137 @@
+{
+    "API Key": "API-nøgle",
+    "About": "Om",
+    "Add Node": "Tilføj node",
+    "Add Repository": "Tilføj lager",
+    "Address": "Adresse",
+    "Addresses": "Adresser",
+    "Allow Anonymous Usage Reporting?": "Tillad anonym brugerstatistik?",
+    "Announce Server": "Opslagsserver",
+    "Anonymous Usage Reporting": "Anonym brugerstatistik",
+    "Bugs": "Fejl",
+    "CPU Utilization": "CPU-forbrug",
+    "Close": "Luk",
+    "Connection Error": "Tilslutnings fejl",
+    "Copyright © 2014 Jakob Borg and the following Contributors:": "Copyright © 2014 Jakob Borg og følgende bidragsydere:",
+    "Delete": "Slet",
+    "Disconnected": "Ikke tilsluttet",
+    "Documentation": "Dokumentation",
+    "Download Rate": "Downloadhastighed",
+    "Edit": "Rediger",
+    "Edit Node": "Rediger node",
+    "Edit Repository": "Rediger lager",
+    "Enable UPnP": "Anvend UPnP",
+    "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Angiv kommaseparerat \"ip:port\"-adresser eller ordet \"dynamic\" for at benytte automatisk opslag.",
+    "Error": "Fejl",
+    "File Versioning": "Filversionering",
+    "File permission bits are ignored when looking for changes. Use on FAT filesystems.": "Filrettigheder tages der ikke hensyn til ved synkronisering. Anvend på FAT-filsystemer.",
+    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.": "Filer flyttes til et datostemplet versionsnavn i et .stversions-bibliotek, når de bliver opdateret eller slettet af syncthing.",
+    "Files are protected from changes made on other nodes, but changes made on this node will be sent to the rest of the cluster.": "Filer beskyttes mod ændringer fra andre noder, men ændringer på denne node bliver sendt til hele clusteret.",
+    "Folder": "Bibliotek",
+    "GUI Authentication Password": "GUI-kodeord",
+    "GUI Authentication User": "GUI-brugernavn",
+    "GUI Listen Addresses": "GUI-lytteadresse",
+    "Generate": "Opret",
+    "Global Discovery": "Globalt opslag",
+    "Global Discovery Server": "Global opslagsserver",
+    "Global Repository": "Global lagring",
+    "Idle": "Inaktiv",
+    "Ignore Permissions": "Ignorér filrettigheder",
+    "Keep Versions": "Behold versioner",
+    "Last seen": "Last seen",
+    "Latest Release": "Seneste udgivelse",
+    "Local Discovery": "Lokal opslag",
+    "Local Discovery Port": "Lokal opslagsport",
+    "Local Repository": "Lokal lagring",
+    "Master Repo": "Hovedlagring",
+    "Max File Change Rate (KiB/s)": "Højeste filændringshastighed (KiB/s)",
+    "Max Outstanding Requests": "Parallelitet",
+    "Maximum Age": "Maximum Age",
+    "Never": "Never",
+    "No": "Nej",
+    "No File Versioning": "No File Versioning",
+    "Node ID": "Node ID",
+    "Node Identification": "Node identifikation",
+    "Node Name": "Nodenavn",
+    "Notice": "OBS",
+    "OK": "OK",
+    "Offline": "Offline",
+    "Online": "Online",
+    "Out Of Sync": "Ude af sync",
+    "Outgoing Rate Limit (KiB/s)": "Udgående hastighedsbegrænsning (KiB/s)",
+    "Override Changes": "Overskriv ændringer",
+    "Path to the repository on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Sti til lagring på din lokale computer. Hvis biblioteket ikke findes vil det blive oprettet. Tegnet tilde (~) kan bruges som genvej til",
+    "Path where versions should be stored (leave empty for the default .stversions folder in the repository).": "Path where versions should be stored (leave empty for the default .stversions folder in the repository).",
+    "Please wait": "Vent venligst",
+    "Preview Usage Report": "Forhåndsvisning af forbrugsrapport",
+    "RAM Utilization": "RAM-forbrug",
+    "Reconnect Interval (s)": "Gentilslutningsinterval (s)",
+    "Repository ID": "Lagrings-ID",
+    "Repository Master": "Hovedlagring",
+    "Repository Path": "Sti til lagring",
+    "Rescan": "Rescan",
+    "Rescan Interval": "Rescan Interval",
+    "Rescan Interval (s)": "Genscanningsinterval (s)",
+    "Restart": "Genstart",
+    "Restart Needed": "Programmet kræver genstart",
+    "Restarting": "Genstarter",
+    "Save": "Gem",
+    "Scanning": "Opdaterer",
+    "Select the nodes to share this repository with.": "Vælg hvilke noder denne lagring skal deles med.",
+    "Settings": "Indstillinger",
+    "Share With Nodes": "Del med noderne",
+    "Shared With": "Delt med",
+    "Short identifier for the repository. Must be the same on all cluster nodes.": "Kort identifikation for denne lagring. Skal være ens på alle noder i clusteret.",
+    "Show ID": "Vis ID",
+    "Shown instead of Node ID in the cluster status.": "Vises i stedet for node-ID under clusterstatus.",
+    "Shown instead of Node ID in the cluster status. Will be advertised to other nodes as an optional default name.": "Shown instead of Node ID in the cluster status. Will be advertised to other nodes as an optional default name.",
+    "Shown instead of Node ID in the cluster status. Will be updated to the name the node advertises if left empty.": "Shown instead of Node ID in the cluster status. Will be updated to the name the node advertises if left empty.",
+    "Shutdown": "Luk ned",
+    "Simple File Versioning": "Simple File Versioning",
+    "Source Code": "Kildekode",
+    "Staggered File Versioning": "Staggered File Versioning",
+    "Start Browser": "Start browser",
+    "Stopped": "Stoppet",
+    "Support / Forum": "Support / Forum",
+    "Sync Protocol Listen Addresses": "Lytteadresser for indgående forbindelser",
+    "Synchronization": "Synkronisering",
+    "Syncing": "Synkroniserer",
+    "Syncthing has been shut down.": "Syncthing er blevet lukket ned.",
+    "Syncthing includes the following software or portions thereof:": "Syncthing indeholder følgende software eller dele heraf:",
+    "Syncthing is restarting.": "Syncthing genstarter",
+    "Syncthing is upgrading.": "Syncthing opgradere",
+    "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing ser ud til at være stoppet eller oplever problemer med din internetforbindels. Prøver igen...",
+    "The aggregated statistics are publicly available at {%url%}.": "Samlet statistik er offentligt tilgængelig på {{url}}.",
+    "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "Konfigurationen er gemt, men ikke aktiveret. Syncthing skal genstarte for at aktivere den nye konfiguration.",
+    "The encrypted usage report is sent daily. It is used to track common platforms, repo sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Den krypterede forbrugsrapport sendes dagligt. Den benyttes til at spore anvendte platforme, lagringsstørrelser og versioner.  Hvis det typen af opsamlet data ændres på et senere tidspunkt, vil du blive spurgt om tilladelse igen.",
+    "The entered node ID does not look valid. It should be a 52 character string consisting of letters and numbers, with spaces and dashes being optional.": "Det indtastede node ID ser ikke gyldigt ud. Det skal være en 52 tegn streng, bestående af tal og bogstaver, eventuelt med mellemrum og bindestreger.",
+    "The entered node ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Det indtastede node ID ser ikke gyldigt ud. Det skal være en 52 eller 56 tegn streng, bestående af tal og bogstaver, eventuelt med mellemrum og bindestreger.",
+    "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.",
+    "The maximum age must be a number and cannot be blank.": "The maximum age must be a number and cannot be blank.",
+    "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "The maximum time to keep a version (in days, set to 0 to keep versions forever).",
+    "The node ID cannot be blank.": "Node-ID'et kan ikke være blankt.",
+    "The node ID to enter here can be found in the \"Edit > Show ID\" dialog on the other node. Spaces and dashes are optional (ignored).": "Node-ID'et som skal bruges her, kan du finde i \"Rediger > Vis ID\"-dialogen på den anden node. Mellemrum og bindestreg er valgfri (ignoreres).",
+    "The number of old versions to keep, per file.": "Antallet af gamle versioner som gemmes, per fil.",
+    "The number of versions must be a number and cannot be blank.": "Antallet af versioner skal være et tal, og kan ikke være blankt.",
+    "The repository ID cannot be blank.": "Lagrings-ID kan ikke være blankt.",
+    "The repository ID must be a short identifier (64 characters or less) consisting of letters, numbers and the the dot (.), dash (-) and underscode (_) characters only.": "Lagrings-ID'et skal være en kort identificierende streng (64 karaktere eller mindre) bestående af bogstav-, tal-, punktum- (.), bindestreg- (-) og understregskaraktere (_).",
+    "The repository ID must be unique.": "Lagrings-ID'et skal være unikt.",
+    "The repository path cannot be blank.": "Lagringsstien kan ikke være blank.",
+    "The rescan interval must be at least 5 seconds.": "The rescan interval must be at least 5 seconds.",
+    "Unknown": "Ukendt",
+    "Up to Date": "Fuldt opdateret",
+    "Upgrade To {%version%}": "Opgradér til {{version}}",
+    "Upgrading": "Opgradere",
+    "Upload Rate": "Uploadhastighed",
+    "Usage": "Forbrug",
+    "Use Compression": "Anvend komprimering",
+    "Use HTTPS for GUI": "Anvend HTTPS til GUI adgang",
+    "Version": "Version",
+    "Versions Path": "Versions Path",
+    "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.",
+    "When adding a new node, keep in mind that this node must be added on the other side too.": "Når du tilføjer en ny node skal du huske, at den også skal tilføjes på den anden side.",
+    "When adding a new repository, keep in mind that the Repository ID is used to tie repositories together between nodes. They are case sensitive and must match exactly between all nodes.": "Når du tilføjer en ny node skal du huske, at lagrings-ID'et bliver brugt til at knytte noder sammen. De er følsomme for store og små bogstaver og skal matche på alle noder.",
+    "Yes": "Ja",
+    "You must keep at least one version.": "Du skal beholde mindst én version.",
+    "items": "poster"
+}

gui/lang-de.json

@@ -1,6 +1,6 @@
 {
-    "API Key": "API-Schlüssel",
-    "About": "Über",
+    "API Key": "API-Key",
+    "About": "Über Syncthing",
     "Add Node": "Knoten hinzufügen",
     "Add Repository": "Verzeichnis hinzufügen",
     "Address": "Adresse",
@@ -11,41 +11,47 @@
     "Bugs": "Fehler",
     "CPU Utilization": "Prozessorauslastung",
     "Close": "Schließen",
+    "Connection Error": "Verbindungsfehler",
     "Copyright © 2014 Jakob Borg and the following Contributors:": "Copyright © 2014 Jakob Borg und folgende Unterstützer:",
     "Delete": "Löschen",
     "Disconnected": "Verbindung getrennt",
     "Documentation": "Dokumentation",
     "Download Rate": "Downloadgeschwindigkeit",
-    "Edit": "Bearbeiten",
+    "Edit": "Einstellungen bearbeiten",
     "Edit Node": "Knoten bearbeiten",
     "Edit Repository": "Verzeichnis ändern",
-    "Enable UPnP": "Aktiviere UPnP",
+    "Enable UPnP": "UPnP aktivieren",
     "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Trage durch ein Komma getrennte \"IP:Port\" Adressen oder \"dynamic\" ein um automatische Adresserkennung durchzuführen.",
     "Error": "Fehler",
     "File Versioning": "Dateiversionierung",
     "File permission bits are ignored when looking for changes. Use on FAT filesystems.": "Dateizugriffsrechte beim Suchen nach Veränderungen ignorieren. Bei FAT-Dateisystemen verwenden.",
-    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.": "Dateien werden beim Löschen oder Ersetzen als datierte Versionen in einen .stversions -Ordner verschoben.",
+    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.": "Dateien werden, bevor syncthing sie löscht oder ersetzt, als datierte Versionen in einen Ordner names .stversions verschoben.",
     "Files are protected from changes made on other nodes, but changes made on this node will be sent to the rest of the cluster.": "Dateien sind vor Veränderung durch andere Knoten geschützt, auf diesem Knoten durchgeführte Veränderungen werden aber auf den Rest des Netzwerks übertragen.",
     "Folder": "Ordner",
     "GUI Authentication Password": "Passwort für Zugang zur Benutzeroberfläche",
     "GUI Authentication User": "Nutzername für Zugang zur Benutzeroberfläche.",
     "GUI Listen Addresses": "Adresse(n) für die Benutzeroberfläche",
-    "Generate": "Generiere",
+    "Generate": "Generieren",
     "Global Discovery": "Globale Auffindung",
     "Global Discovery Server": "Globaler Auffindungsserver",
     "Global Repository": "Globales Verzeichnis",
     "Idle": "Untätig",
     "Ignore Permissions": "Berechtigungen ignorieren",
     "Keep Versions": "Versionen erhalten",
+    "Last seen": "Zuletzt online",
     "Latest Release": "Letzte Veröffentlichung",
     "Local Discovery": "Lokale Auffindung",
     "Local Discovery Port": "Lokaler Aufindungsport",
     "Local Repository": "Lokales Verzeichnis",
-    "Master Repo": "Keine Veränderungen zugelassen",
+    "Master Repo": "Originalverzeichnis",
     "Max File Change Rate (KiB/s)": "Maximale Datenänderungsrate (KiB/s)",
     "Max Outstanding Requests": "Max. ausstehende Anfragen",
+    "Maximum Age": "Höchstalter",
+    "Never": "Nie",
     "No": "Nein",
+    "No File Versioning": "Keine Dateiversionierung",
     "Node ID": "Knoten-ID",
+    "Node Identification": "Knoten Identifikation",
     "Node Name": "Knoten-Name",
     "Notice": "Benachrichtigung",
     "OK": "Ok",
@@ -55,18 +61,22 @@
     "Outgoing Rate Limit (KiB/s)": "Ausgehendes Datenratelimit (KiB/s)",
     "Override Changes": "Änderungen überschreiben",
     "Path to the repository on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Pfad zum Verzeichnis auf dem lokalen Rechner. Wird erzeugt, wenn es nicht existiert. Das Tilden-Zeichen (~)  kann als Abkürzung benutzt werden für",
+    "Path where versions should be stored (leave empty for the default .stversions folder in the repository).": "Pfad in dem die Versionen gespeichert werden sollen (ohne Angabe wird der Ordner .stversions im Verzeichnis verwendet).",
     "Please wait": "Bitte warten",
     "Preview Usage Report": "Vorschau des Nutzungsberichts",
     "RAM Utilization": "Verwendeter Arbeitsspeicher",
     "Reconnect Interval (s)": "Wiederverbindungsintervall (s)",
     "Repository ID": "Verzeichnis-ID",
-    "Repository Master": "Keine Veränderungen zulassen",
+    "Repository Master": "Originalverzeichnis",
     "Repository Path": "Pfad zum Verzeichnis",
+    "Rescan": "Überprüfen",
+    "Rescan Interval": "Suchintervall",
     "Rescan Interval (s)": "Suchintervall (s)",
     "Restart": "Neustart",
     "Restart Needed": "Neustart notwendig",
+    "Restarting": "Wird neu gestartet",
     "Save": "Speichern",
-    "Scanning": "Überprüfe",
+    "Scanning": "Sucht",
     "Select the nodes to share this repository with.": "Wähle die Knoten aus, mit denen du dieses Verzeichnis teilen willst.",
     "Settings": "Einstellungen",
     "Share With Nodes": "Teile mit diesen Knoten",
@@ -74,8 +84,12 @@
     "Short identifier for the repository. Must be the same on all cluster nodes.": "Kurze ID für das Verzeichnis. Muss auf allen Verbunds-Knoten gleich sein.",
     "Show ID": "ID anzeigen",
     "Shown instead of Node ID in the cluster status.": "Wird anstatt der Knoten-ID im Verbunds-Status angezeigt.",
+    "Shown instead of Node ID in the cluster status. Will be advertised to other nodes as an optional default name.": "Wird anstatt der Knoten-ID im Verbunds-Status angezeigt. Wird als optionaler Standardname an andere Knoten bekannt gegeben.",
+    "Shown instead of Node ID in the cluster status. Will be updated to the name the node advertises if left empty.": "Wird anstatt der Knoten-ID im Verbunds-Status angezeigt. Wird auf den Namen aktualisiert, den der Knoten angibt.",
     "Shutdown": "Herunterfahren",
-    "Source Code": "Quellcode",
+    "Simple File Versioning": "Einfache Dateiversionierung",
+    "Source Code": "Sourcecode",
+    "Staggered File Versioning": "Stufenweise Dateiversionierung",
     "Start Browser": "Starte Browser",
     "Stopped": "Gestoppt",
     "Support / Forum": "Support / Forum",
@@ -84,27 +98,37 @@
     "Syncing": "Synchronisiere",
     "Syncthing has been shut down.": "Syncthing wurde heruntergefahren.",
     "Syncthing includes the following software or portions thereof:": "Syncthing enthält die folgende Software oder Teile davon:",
+    "Syncthing is restarting.": "Syncthing wird neu gestartet",
+    "Syncthing is upgrading.": "Syncthing wird aktualisiert",
     "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing scheint nicht erreichbar zu sein oder es gibt ein Problem mit Ihrer Internetverbindung. Versuche erneut...",
-    "The aggregated statistics are publicly available at {{url}}": "Die aggregierten Statistiken sind öffentlich verfügbar unter {{url}}",
+    "The aggregated statistics are publicly available at {%url%}.": "Die gesammelten Statistiken sind öffentlich verfügbar unter {{url}}.",
     "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "Die Konfiguration wurde gespeichert, aber nicht aktiviert. Syncthing muss neugestartet werden um die neue Konfiguration zu aktivieren.",
     "The encrypted usage report is sent daily. It is used to track common platforms, repo sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Der verschlüsselte Benutzungsbericht wird täglich gesendet. Er wird benutzt um Statistiken über verwendete Betriebssysteme, Verzeichnis-Größen und Programm-Versionen zu erstellen. Sobald der Bericht in Zukunft weitere Daten erfasst, wird dir dieses Fenster erneut angezeigt.",
     "The entered node ID does not look valid. It should be a 52 character string consisting of letters and numbers, with spaces and dashes being optional.": "Die eingegebene Knoten-ID scheint nicht gültig zu sein. Sie sollte eine 52 Stellen lange Zeichenkette aus Buchstaben und Zahlen sein. Leerzeichen und Striche sind optional (werden ignoriert).",
+    "The entered node ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Die eingegebene Knoten-ID scheint nicht gültig zu sein. Es sollte eine 52 oder 56 stellige Zeichenkette aus Buchstaben und Nummern sein. Leerzeichen und Bindestriche sind optional.",
+    "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "Es wird in folgenden Abständen versioniert: in der ersten Stunde wird alle 30 Sekunden eine Version behalten, am ersten Tag eine jede Stunde, in den ersten 30 Tagen eine jeden Tag, danach wird bis zum Höchstalter eine Version pro Woche beibehalten.",
+    "The maximum age must be a number and cannot be blank.": "Das Höchstalter muss angegeben werden und eine Zahl sein.",
+    "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "Die längste Zeit, die alte Versionen vorgehalten werden (in Tagen, 0 bedeutet, alte Versionen für immer zu behalten).",
     "The node ID cannot be blank.": "Die Knoten-ID darf nicht leer sein.",
     "The node ID to enter here can be found in the \"Edit > Show ID\" dialog on the other node. Spaces and dashes are optional (ignored).": "Die hier einzutragende Knoten-ID kann im \"Bearbeiten > Zeige ID\"-Dialog auf dem anderen Knoten gefunden werden. Leerzeichen und Striche sind optional (werden ignoriert).",
     "The number of old versions to keep, per file.": "Anzahl der alten Versionen, die von jeder Datei gespeichert werden sollen.",
-    "The number of versions must be a number and cannot be blank.": "Die Anzahl von Versionen muss eine Zahl sein und darf nicht leer sein.",
+    "The number of versions must be a number and cannot be blank.": "Die Anzahl von Versionen muss eine Zahl und darf nicht leer sein.",
     "The repository ID cannot be blank.": "Die Verzeichnis-ID darf nicht leer sein.",
     "The repository ID must be a short identifier (64 characters or less) consisting of letters, numbers and the the dot (.), dash (-) and underscode (_) characters only.": "Die Verzeichnis-ID muss eine kurze Kennung (64 Zeichen oder weniger) sein. Sie kann aus Buchstaben, Zahlen und den Punkt- (.), Strich- (-), und Unterstrich- (_) Zeichen bestehen.",
     "The repository ID must be unique.": "Die Verzeichnis-ID muss eindeutig sein.",
     "The repository path cannot be blank.": "Der Verzeichnis-Pfad kann nicht leer sein",
+    "The rescan interval must be at least 5 seconds.": "Das Suchintervall muss mindestens 5 Sekunden betragen.",
     "Unknown": "Unbekannt",
     "Up to Date": "Aktuell",
     "Upgrade To {%version%}": "Upgrade auf {{version}}",
+    "Upgrading": "Wird aktualisiert",
     "Upload Rate": "Uploadgeschwindigkeit",
     "Usage": "Benutzung",
     "Use Compression": "Benutze Komprimierung",
     "Use HTTPS for GUI": "Benutze HTTPS für Benutzeroberfläche",
     "Version": "Version",
+    "Versions Path": "Versionierungspfad",
+    "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Alte Versionen werden automatisch gelöscht wenn sie älter als das angegebene Höchstalter sind oder die Höchstzahl der Dateien pro Zeitabschnitt überschritten wird.",
     "When adding a new node, keep in mind that this node must be added on the other side too.": "Beachte beim Hinzufügen eines neuen Knotens, dass dieser Knoten auch auf der Gegenseite hinzugefügt werden muss.",
     "When adding a new repository, keep in mind that the Repository ID is used to tie repositories together between nodes. They are case sensitive and must match exactly between all nodes.": "Beim Hinzufügen eines neuen Verzeichnisses, beachte dass die Verzeichnis-ID dazu verwendet wird, Verzeichnisse zwischen Knoten zu verbinden. Die ID muss also auf allen Knoten gleich sein, Groß- und Kleinschreibung muss dabei beachtet werden.",
     "Yes": "Ja",