Always use correct format Node IDs in GUI

.gitignore

@@ -8,3 +8,6 @@ discosrv
 .jshintrc
 coverage.out
 files/pidx
+discosrv.exe
+bin
+perfstats*.csv

build.sh

@@ -105,7 +105,8 @@ xdr() {
 case "$1" in
 	"")
 		shift
-		build $*
+		export GOBIN=$(pwd)/bin
+		godep go install $* -ldflags "$ldflags" ./cmd/...
 		;;
 
 	race)

cmd/syncthing/perfstats.go

@@ -0,0 +1,45 @@
+// +build perfstats
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"syscall"
+	"time"
+)
+
+func init() {
+	go savePerfStats(fmt.Sprintf("perfstats-%d.csv", syscall.Getpid()))
+}
+
+func savePerfStats(file string) {
+	fd, err := os.Create(file)
+	if err != nil {
+		panic(err)
+	}
+
+	var prevUsage int64
+	var prevTime int64
+	var rusage syscall.Rusage
+	var memstats runtime.MemStats
+
+	t0 := time.Now()
+	for t := range time.NewTicker(250 * time.Millisecond).C {
+		syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
+		curTime := time.Now().UnixNano()
+		timeDiff := curTime - prevTime
+		curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()
+		usageDiff := curUsage - prevUsage
+		cpuUsagePercent := 100 * float64(usageDiff) / float64(timeDiff)
+		prevTime = curTime
+		prevUsage = curUsage
+
+		runtime.ReadMemStats(&memstats)
+
+		startms := int(t.Sub(t0).Seconds() * 1000)
+
+		fmt.Fprintf(fd, "%d\t%f\t%d\t%d\n", startms, cpuUsagePercent, memstats.Alloc, memstats.Sys)
+	}
+}

cmd/syncthing/upgrade_common.go

@@ -7,9 +7,9 @@ import (
 )
 
 type githubRelease struct {
-	Tag      string        `json:"tag_name"`
-	Prelease bool          `json:"prerelease"`
-	Assets   []githubAsset `json:"assets"`
+	Tag        string        `json:"tag_name"`
+	Prerelease bool          `json:"prerelease"`
+	Assets     []githubAsset `json:"assets"`
 }
 
 type githubAsset struct {

cmd/syncthing/upgrade_supported.go

@@ -84,7 +84,7 @@ func upgrade() error {
 }
 
 func currentRelease() (githubRelease, error) {
-	resp, err := http.Get("https://api.github.com/repos/calmh/syncthing/releases?per_page=1")
+	resp, err := http.Get("https://api.github.com/repos/calmh/syncthing/releases?per_page=10")
 	if err != nil {
 		return githubRelease{}, err
 	}
@@ -93,10 +93,13 @@ func currentRelease() (githubRelease, error) {
 	json.NewDecoder(resp.Body).Decode(&rels)
 	resp.Body.Close()
 
-	if len(rels) != 1 {
-		return githubRelease{}, fmt.Errorf("Unexpected number of releases: %d", len(rels))
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			return rel, nil
+		}
 	}
-	return rels[0], nil
+
+	return githubRelease{}, errors.New("no suitable release found")
 }
 
 func readTarGZ(url string, dir string) (string, error) {

config/config_test.go

@@ -189,7 +189,7 @@ func TestOverriddenValues(t *testing.T) {
 	}
 }
 
-func TestNodeAddresses(t *testing.T) {
+func TestNodeAddressesDynamic(t *testing.T) {
 	data := []byte(`
 <configuration version="2">
     <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
@@ -233,3 +233,52 @@ func TestNodeAddresses(t *testing.T) {
 		t.Errorf("Nodes differ;\n  E: %#v\n  A: %#v", expected, cfg.Nodes)
 	}
 }
+
+func TestNodeAddressesStatic(t *testing.T) {
+	data := []byte(`
+<configuration version="2">
+    <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
+        <address>192.0.2.1</address>
+        <address>192.0.2.2</address>
+    </node>
+    <node id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA">
+        <address>192.0.2.3:6070</address>
+        <address>[2001:db8::42]:4242</address>
+    </node>
+    <node id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q">
+        <address>[2001:db8::44]:4444</address>
+        <address>192.0.2.4:6090</address>
+    </node>
+</configuration>
+`)
+
+	name, _ := os.Hostname()
+	expected := []NodeConfiguration{
+		{
+			NodeID:    node1,
+			Addresses: []string{"192.0.2.1", "192.0.2.2"},
+		},
+		{
+			NodeID:    node2,
+			Addresses: []string{"192.0.2.3:6070", "[2001:db8::42]:4242"},
+		},
+		{
+			NodeID:    node3,
+			Addresses: []string{"[2001:db8::44]:4444", "192.0.2.4:6090"},
+		},
+		{
+			NodeID:    node4,
+			Name:      name, // Set when auto created
+			Addresses: []string{"dynamic"},
+		},
+	}
+
+	cfg, err := Load(bytes.NewReader(data), node4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Nodes, expected) {
+		t.Errorf("Nodes differ;\n  E: %#v\n  A: %#v", expected, cfg.Nodes)
+	}
+}

gui/app.js

@@ -413,7 +413,6 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
 
         $('#editNode').modal('hide');
         nodeCfg = $scope.currentNode;
-        nodeCfg.NodeID = nodeCfg.NodeID.replace(/ /g, '').replace(/-/g, '').toLowerCase().trim();
         nodeCfg.Addresses = nodeCfg.AddressesStr.split(',').map(function (x) { return x.trim(); });
 
         done = false;
@@ -866,6 +865,7 @@ syncthing.directive('validNodeid', function($http) {
                         if (resp.error) {
                             ctrl.$setValidity('validNodeid', false);
                         } else {
+                            scope.currentNode.NodeID = resp.id;
                             ctrl.$setValidity('validNodeid', true);
                         }
                     });

model/blockqueue.go

@@ -4,11 +4,7 @@
 
 package model
 
-import (
-	"sync"
-
-	"github.com/calmh/syncthing/protocol"
-)
+import "github.com/calmh/syncthing/protocol"
 
 type bqAdd struct {
 	file protocol.FileInfo
@@ -25,27 +21,10 @@ type bqBlock struct {
 }
 
 type blockQueue struct {
-	inbox  chan bqAdd
-	outbox chan bqBlock
-
 	queued []bqBlock
-
-	mut sync.Mutex
 }
 
-func newBlockQueue() *blockQueue {
-	q := &blockQueue{
-		inbox:  make(chan bqAdd),
-		outbox: make(chan bqBlock),
-	}
-	go q.run()
-	return q
-}
-
-func (q *blockQueue) addBlock(a bqAdd) {
-	q.mut.Lock()
-	defer q.mut.Unlock()
-
+func (q *blockQueue) put(a bqAdd) {
 	// If we already have it queued, return
 	for _, b := range q.queued {
 		if b.file.Name == a.file.Name {
@@ -84,36 +63,11 @@ func (q *blockQueue) addBlock(a bqAdd) {
 	}
 }
 
-func (q *blockQueue) run() {
-	for {
-		if len(q.queued) == 0 {
-			q.addBlock(<-q.inbox)
-		} else {
-			q.mut.Lock()
-			next := q.queued[0]
-			q.mut.Unlock()
-			select {
-			case a := <-q.inbox:
-				q.addBlock(a)
-			case q.outbox <- next:
-				q.mut.Lock()
-				q.queued = q.queued[1:]
-				q.mut.Unlock()
-			}
-		}
+func (q *blockQueue) get() (bqBlock, bool) {
+	if len(q.queued) == 0 {
+		return bqBlock{}, false
 	}
-}
-
-func (q *blockQueue) put(a bqAdd) {
-	q.inbox <- a
-}
-
-func (q *blockQueue) get() bqBlock {
-	return <-q.outbox
-}
-
-func (q *blockQueue) empty() bool {
-	q.mut.Lock()
-	defer q.mut.Unlock()
-	return len(q.queued) == 0
+	b := q.queued[0]
+	q.queued = q.queued[1:]
+	return b, true
 }

model/model.go

@@ -584,8 +584,10 @@ func sendIndexes(conn protocol.Connection, repo string, fs *files.Set) {
 	}()
 
 	for err == nil {
-		if !initial && fs.LocalVersion(protocol.LocalNodeID) <= minLocalVer {
-			time.Sleep(1 * time.Second)
+		if !initial {
+			time.Sleep(5 * time.Second)
+		}
+		if fs.LocalVersion(protocol.LocalNodeID) <= minLocalVer {
 			continue
 		}
 

model/puller.go

@@ -63,7 +63,8 @@ var errNoNode = errors.New("no available source node")
 type puller struct {
 	cfg               *config.Configuration
 	repoCfg           config.RepositoryConfiguration
-	bq                *blockQueue
+	bq                blockQueue
+	slots             int
 	model             *Model
 	oustandingPerNode activityMap
 	openFiles         map[string]openFile
@@ -75,9 +76,9 @@ type puller struct {
 
 func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int, cfg *config.Configuration) *puller {
 	p := &puller{
-		repoCfg:           repoCfg,
 		cfg:               cfg,
-		bq:                newBlockQueue(),
+		repoCfg:           repoCfg,
+		slots:             slots,
 		model:             model,
 		oustandingPerNode: make(activityMap),
 		openFiles:         make(map[string]openFile),
@@ -96,9 +97,6 @@ func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int,
 
 	if slots > 0 {
 		// Read/write
-		for i := 0; i < slots; i++ {
-			p.requestSlots <- true
-		}
 		if debug {
 			l.Debugf("starting puller; repo %q dir %q slots %d", repoCfg.ID, repoCfg.Directory, slots)
 		}
@@ -114,57 +112,70 @@ func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int,
 }
 
 func (p *puller) run() {
-	go func() {
-		// fill blocks queue when there are free slots
-		for {
-			<-p.requestSlots
-			b := p.bq.get()
-			if debug {
-				l.Debugf("filler: queueing %q / %q offset %d copy %d", p.repoCfg.ID, b.file.Name, b.block.Offset, len(b.copy))
-			}
-			p.blocks <- b
-		}
-	}()
-
-	timeout := time.Tick(5 * time.Second)
 	changed := true
 	scanintv := time.Duration(p.cfg.Options.RescanIntervalS) * time.Second
 	lastscan := time.Now()
 	var prevVer uint64
+	var queued int
+
+	// Load up the request slots
+	for i := 0; i < cap(p.requestSlots); i++ {
+		p.requestSlots <- true
+	}
 
 	for {
 		// Run the pulling loop as long as there are blocks to fetch
-	pull:
-		for {
-			select {
-			case res := <-p.requestResults:
-				p.model.setState(p.repoCfg.ID, RepoSyncing)
-				changed = true
-				p.requestSlots <- true
-				p.handleRequestResult(res)
 
-			case b := <-p.blocks:
-				p.model.setState(p.repoCfg.ID, RepoSyncing)
-				changed = true
-				if p.handleBlock(b) {
-					// Block was fully handled, free up the slot
+		prevVer, queued = p.queueNeededBlocks(prevVer)
+		if queued > 0 {
+
+		pull:
+			for {
+				select {
+				case res := <-p.requestResults:
+					p.model.setState(p.repoCfg.ID, RepoSyncing)
+					changed = true
 					p.requestSlots <- true
-				}
+					p.handleRequestResult(res)
 
-			case <-timeout:
-				if len(p.openFiles) == 0 && p.bq.empty() {
-					// Nothing more to do for the moment
-					break pull
-				}
-				if debug {
-					l.Debugf("%q: idle but have %d open files", p.repoCfg.ID, len(p.openFiles))
-					i := 5
-					for _, f := range p.openFiles {
-						l.Debugf("  %v", f)
-						i--
-						if i == 0 {
-							break
+				case <-p.requestSlots:
+					b, ok := p.bq.get()
+
+					if !ok {
+						if debug {
+							l.Debugf("%q: pulling loop needs more blocks", p.repoCfg.ID)
 						}
+						prevVer, _ = p.queueNeededBlocks(prevVer)
+						b, ok = p.bq.get()
+					}
+
+					if !ok && len(p.openFiles) == 0 {
+						// Nothing queued, nothing outstanding
+						if debug {
+							l.Debugf("%q: pulling loop done", p.repoCfg.ID)
+						}
+						break pull
+					}
+
+					if !ok {
+						// Nothing queued, but there are still open files.
+						// Give the situation a moment to change.
+						if debug {
+							l.Debugf("%q: pulling loop paused", p.repoCfg.ID)
+						}
+						p.requestSlots <- true
+						time.Sleep(100 * time.Millisecond)
+						continue pull
+					}
+
+					if debug {
+						l.Debugf("queueing %q / %q offset %d copy %d", p.repoCfg.ID, b.file.Name, b.block.Offset, len(b.copy))
+					}
+					p.model.setState(p.repoCfg.ID, RepoSyncing)
+					changed = true
+					if p.handleBlock(b) {
+						// Block was fully handled, free up the slot
+						p.requestSlots <- true
 					}
 				}
 			}
@@ -192,19 +203,7 @@ func (p *puller) run() {
 			lastscan = time.Now()
 		}
 
-		if v := p.model.LocalVersion(p.repoCfg.ID); v != prevVer {
-			if debug {
-				l.Debugf("%q: checking for more needed blocks", p.repoCfg.ID)
-			}
-			// Queue more blocks to fetch, if any
-			if p.queueNeededBlocks() == 0 {
-				if debug {
-					l.Debugf("%q: no more needed blocks", p.repoCfg.ID)
-				}
-				// We've fetched all blocks we need
-				prevVer = v
-			}
-		}
+		time.Sleep(5 * time.Second)
 	}
 }
 
@@ -620,9 +619,21 @@ func (p *puller) handleEmptyBlock(b bqBlock) {
 	delete(p.openFiles, f.Name)
 }
 
-func (p *puller) queueNeededBlocks() int {
+func (p *puller) queueNeededBlocks(prevVer uint64) (uint64, int) {
+	curVer := p.model.LocalVersion(p.repoCfg.ID)
+	if curVer == prevVer {
+		return curVer, 0
+	}
+
+	if debug {
+		l.Debugf("%q: checking for more needed blocks", p.repoCfg.ID)
+	}
+
 	queued := 0
 	for _, f := range p.model.NeedFilesRepo(p.repoCfg.ID) {
+		if _, ok := p.openFiles[f.Name]; ok {
+			continue
+		}
 		lf := p.model.CurrentRepoFile(p.repoCfg.ID, f.Name)
 		have, need := scanner.BlockDiff(lf.Blocks, f.Blocks)
 		if debug {
@@ -638,7 +649,12 @@ func (p *puller) queueNeededBlocks() int {
 	if debug && queued > 0 {
 		l.Debugf("%q: queued %d items", p.repoCfg.ID, queued)
 	}
-	return queued
+
+	if queued > 0 {
+		return prevVer, queued
+	} else {
+		return curVer, 0
+	}
 }
 
 func (p *puller) closeFile(f protocol.FileInfo) {