Compare commits

...
Author SHA1 Message Date
Jakob BorgandGitHub 71de6fe290 lib/upnp: Exit quicker (#6339)
During NAT discovery we block for 10s (NatTimeoutS) before returning.
This is mostly noticeable when Ctrl-C:ing a Syncthing directly after
startup as we wait for those ten seconds before shutting down. This
makes it check the context a little bit more frequently.
2020-02-13 15:39:36 +01:00
Jakob BorgandGitHub 6a840a040b lib/db: Keep metadata better in sync (ref #6335) (#6337)
This adds metadata updates to the same write batch as the underlying
file change. The odds of a metadata update going missing is greatly
reduced.

Bonus change: actually commit the transaction in recalcMeta.
2020-02-13 15:23:08 +01:00
Simon FreiandGitHub c3637f2191 lib: Faster termination on exit (ref #6319) (#6329) 2020-02-13 14:43:00 +01:00
Simon FreiandGitHub ca90f4e6af lib/db: Use flags from arg not LocalFlags() updating meta (#6333) 2020-02-13 14:02:30 +01:00
Jakob BorgandGitHub 51fa36d61f lib/db: Recover sequence number and metadata on startup (fixes #6335) (#6336)
lib/db: Recover sequence number and metadata on startup (fixes #6335)

If we crashed after writing new file entries but before updating
metadata in the database the sequence number and metadata will be wrong.
This fixes that.
2020-02-13 13:05:26 +01:00
15 changed files with 206 additions and 80 deletions
+2 -1
View File
@@ -7,6 +7,7 @@
package main
import (
"context"
"crypto/tls"
"errors"
"flag"
@@ -95,7 +96,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
})
go func() {
addresses, err := disco.Lookup(deviceID)
addresses, err := disco.Lookup(context.Background(), deviceID)
res <- checkResult{addresses: addresses, error: err}
}()
+2 -1
View File
@@ -7,6 +7,7 @@
package api
import (
"context"
"time"
"github.com/syncthing/syncthing/lib/discover"
@@ -26,7 +27,7 @@ func (m *mockedCachingMux) Stop() {
// from events.Finder
func (m *mockedCachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, err error) {
func (m *mockedCachingMux) Lookup(ctx context.Context, deviceID protocol.DeviceID) (direct []string, err error) {
return nil, nil
}
+7 -1
View File
@@ -360,6 +360,12 @@ func (s *service) connect(ctx context.Context) {
var seen []string
for _, deviceCfg := range cfg.Devices {
select {
case <-ctx.Done():
return
default:
}
deviceID := deviceCfg.DeviceID
if deviceID == s.myID {
continue
@@ -380,7 +386,7 @@ func (s *service) connect(ctx context.Context) {
for _, addr := range deviceCfg.Addresses {
if addr == "dynamic" {
if s.discoverer != nil {
if t, err := s.discoverer.Lookup(deviceID); err == nil {
if t, err := s.discoverer.Lookup(ctx, deviceID); err == nil {
addrs = append(addrs, t...)
}
}
+6 -1
View File
@@ -48,10 +48,15 @@ type ReadTransaction interface {
// purposes of saving memory when transactions are in-RAM. Note that
// transactions may be checkpointed *anyway* even if this is not called, due to
// resource constraints, but this gives you a chance to decide when.
//
// Functions can be passed to Checkpoint. These are run if and only if the
// checkpoint will result in a flush, and will run before the flush. The
// transaction can be accessed via a closure. If an error is returned from
// these functions the flush will be aborted and the error bubbled.
type WriteTransaction interface {
ReadTransaction
Writer
Checkpoint() error
Checkpoint(...func() error) error
Commit() error
}
+8 -3
View File
@@ -150,8 +150,8 @@ func (t *leveldbTransaction) Put(key, val []byte) error {
return t.checkFlush(dbFlushBatchMax)
}
func (t *leveldbTransaction) Checkpoint() error {
return t.checkFlush(dbFlushBatchMin)
func (t *leveldbTransaction) Checkpoint(preFlush ...func() error) error {
return t.checkFlush(dbFlushBatchMin, preFlush...)
}
func (t *leveldbTransaction) Commit() error {
@@ -167,10 +167,15 @@ func (t *leveldbTransaction) Release() {
}
// checkFlush flushes and resets the batch if its size exceeds the given size.
func (t *leveldbTransaction) checkFlush(size int) error {
func (t *leveldbTransaction) checkFlush(size int, preFlush ...func() error) error {
if len(t.batch.Dump()) < size {
return nil
}
for _, hook := range preFlush {
if err := hook(); err != nil {
return err
}
}
return t.flush()
}
+14 -2
View File
@@ -124,11 +124,17 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI
return err
}
if err := t.Checkpoint(); err != nil {
if err := t.Checkpoint(func() error {
return meta.toDB(t, folder)
}); err != nil {
return err
}
}
if err := meta.toDB(t, folder); err != nil {
return err
}
return t.Commit()
}
@@ -227,11 +233,17 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
}
}
if err := t.Checkpoint(); err != nil {
if err := t.Checkpoint(func() error {
return meta.toDB(t, folder)
}); err != nil {
return err
}
}
if err := meta.toDB(t, folder); err != nil {
return err
}
return t.Commit()
}
+16 -12
View File
@@ -28,8 +28,8 @@ type metadataTracker struct {
}
type metaKey struct {
dev protocol.DeviceID
flags uint32
dev protocol.DeviceID
flag uint32
}
func newMetadataTracker() *metadataTracker {
@@ -62,8 +62,8 @@ func (m *metadataTracker) Marshal() ([]byte, error) {
// toDB saves the marshalled metadataTracker to the given db, under the key
// corresponding to the given folder
func (m *metadataTracker) toDB(db *Lowlevel, folder []byte) error {
key, err := db.keyer.GenerateFolderMetaKey(nil, folder)
func (m *metadataTracker) toDB(t readWriteTransaction, folder []byte) error {
key, err := t.keyer.GenerateFolderMetaKey(nil, folder)
if err != nil {
return err
}
@@ -79,7 +79,7 @@ func (m *metadataTracker) toDB(db *Lowlevel, folder []byte) error {
if err != nil {
return err
}
err = db.Put(key, bs)
err = t.Put(key, bs)
if err == nil {
m.dirty = false
}
@@ -103,14 +103,18 @@ func (m *metadataTracker) fromDB(db *Lowlevel, folder []byte) error {
// countsPtr returns a pointer to the corresponding Counts struct, if
// necessary allocating one in the process
func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flags uint32) *Counts {
func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flag uint32) *Counts {
// must be called with the mutex held
key := metaKey{dev, flags}
if bits.OnesCount32(flag) > 1 {
panic("incorrect usage: set at most one bit in flag")
}
key := metaKey{dev, flag}
idx, ok := m.indexes[key]
if !ok {
idx = len(m.counts.Counts)
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flags})
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flag})
m.indexes[key] = idx
}
return &m.counts.Counts[idx]
@@ -157,8 +161,8 @@ func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f FileIntf) {
}
}
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) {
cp := m.countsPtr(dev, flags)
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) {
cp := m.countsPtr(dev, flag)
switch {
case f.IsDeleted():
@@ -196,8 +200,8 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
}
}
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) {
cp := m.countsPtr(dev, f.FileLocalFlags())
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) {
cp := m.countsPtr(dev, flag)
switch {
case f.IsDeleted():
+67 -21
View File
@@ -71,7 +71,7 @@ func init() {
}
func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
var s = FileSet{
var s = &FileSet{
folder: folder,
fs: fs,
db: db,
@@ -79,26 +79,34 @@ func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
updateMutex: sync.NewMutex(),
}
if err := s.meta.fromDB(db, []byte(folder)); err != nil {
l.Infof("No stored folder metadata for %q: recalculating", folder)
if err := s.recalcCounts(); backend.IsClosed(err) {
return nil
} else if err != nil {
panic(err)
}
} else if age := time.Since(s.meta.Created()); age > databaseRecheckInterval {
l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, age)
if err := s.recalcCounts(); backend.IsClosed(err) {
recalc := func() *FileSet {
if err := s.recalcMeta(); backend.IsClosed(err) {
return nil
} else if err != nil {
panic(err)
}
return s
}
return &s
if err := s.meta.fromDB(db, []byte(folder)); err != nil {
l.Infof("No stored folder metadata for %q; recalculating", folder)
return recalc()
}
if metaOK := s.verifyLocalSequence(); !metaOK {
l.Infof("Stored folder metadata for %q is out of date after crash; recalculating", folder)
return recalc()
}
if age := time.Since(s.meta.Created()); age > databaseRecheckInterval {
l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, age)
return recalc()
}
return s
}
func (s *FileSet) recalcCounts() error {
func (s *FileSet) recalcMeta() error {
s.meta = newMetadataTracker()
if err := s.db.checkGlobals([]byte(s.folder), s.meta); err != nil {
@@ -109,6 +117,8 @@ func (s *FileSet) recalcCounts() error {
if err != nil {
return err
}
defer t.close()
var deviceID protocol.DeviceID
err = t.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool {
copy(deviceID[:], device)
@@ -120,7 +130,36 @@ func (s *FileSet) recalcCounts() error {
}
s.meta.SetCreated()
return s.meta.toDB(s.db, []byte(s.folder))
if err := s.meta.toDB(t, []byte(s.folder)); err != nil {
return err
}
return t.Commit()
}
// Verify the local sequence number from actual sequence entries. Returns
// true if it was all good, or false if a fixup was necessary.
func (s *FileSet) verifyLocalSequence() bool {
// Walk the sequence index from the current (supposedly) highest
// sequence number and raise the alarm if we get anything. This recovers
// from the occasion where we have written sequence entries to disk but
// not yet written new metadata to disk.
//
// Note that we can have the same thing happen for remote devices but
// there it's not a problem -- we'll simply advertise a lower sequence
// number than we've actually seen and receive some duplicate updates
// and then be in sync again.
curSeq := s.meta.Sequence(protocol.LocalDeviceID)
snap := s.Snapshot()
ok := true
snap.WithHaveSequence(curSeq+1, func(fi FileIntf) bool {
ok = false // we got something, which we should not have
return false
})
snap.Release()
return ok
}
func (s *FileSet) Drop(device protocol.DeviceID) {
@@ -150,7 +189,20 @@ func (s *FileSet) Drop(device protocol.DeviceID) {
s.meta.resetAll(device)
}
if err := s.meta.toDB(s.db, []byte(s.folder)); backend.IsClosed(err) {
t, err := s.db.newReadWriteTransaction()
if backend.IsClosed(err) {
return
} else if err != nil {
panic(err)
}
defer t.close()
if err := s.meta.toDB(t, []byte(s.folder)); backend.IsClosed(err) {
return
} else if err != nil {
panic(err)
}
if err := t.Commit(); backend.IsClosed(err) {
return
} else if err != nil {
panic(err)
@@ -168,12 +220,6 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
s.updateMutex.Lock()
defer s.updateMutex.Unlock()
defer func() {
if err := s.meta.toDB(s.db, []byte(s.folder)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}()
if device == protocol.LocalDeviceID {
// For the local device we have a bunch of metadata to track.
if err := s.db.updateLocalFiles([]byte(s.folder), fs, s.meta); err != nil && !backend.IsClosed(err) {
+3 -2
View File
@@ -7,6 +7,7 @@
package discover
import (
"context"
"sort"
stdsync "sync"
"time"
@@ -73,7 +74,7 @@ func (m *cachingMux) Add(finder Finder, cacheTime, negCacheTime time.Duration) {
// Lookup attempts to resolve the device ID using any of the added Finders,
// while obeying the cache settings.
func (m *cachingMux) Lookup(deviceID protocol.DeviceID) (addresses []string, err error) {
func (m *cachingMux) Lookup(ctx context.Context, deviceID protocol.DeviceID) (addresses []string, err error) {
m.mut.RLock()
for i, finder := range m.finders {
if cacheEntry, ok := m.caches[i].Get(deviceID); ok {
@@ -99,7 +100,7 @@ func (m *cachingMux) Lookup(deviceID protocol.DeviceID) (addresses []string, err
}
// Perform the actual lookup and cache the result.
if addrs, err := finder.Lookup(deviceID); err == nil {
if addrs, err := finder.Lookup(ctx, deviceID); err == nil {
l.Debugln("lookup for", deviceID, "at", finder)
l.Debugln(" addresses:", addrs)
addresses = append(addresses, addrs...)
+8 -5
View File
@@ -7,6 +7,7 @@
package discover
import (
"context"
"reflect"
"testing"
"time"
@@ -39,7 +40,9 @@ func TestCacheUnique(t *testing.T) {
f1 := &fakeDiscovery{addresses0}
c.Add(f1, time.Minute, 0)
addr, err := c.Lookup(protocol.LocalDeviceID)
ctx := context.Background()
addr, err := c.Lookup(ctx, protocol.LocalDeviceID)
if err != nil {
t.Fatal(err)
}
@@ -53,7 +56,7 @@ func TestCacheUnique(t *testing.T) {
f2 := &fakeDiscovery{addresses1}
c.Add(f2, time.Minute, 0)
addr, err = c.Lookup(protocol.LocalDeviceID)
addr, err = c.Lookup(ctx, protocol.LocalDeviceID)
if err != nil {
t.Fatal(err)
}
@@ -66,7 +69,7 @@ type fakeDiscovery struct {
addresses []string
}
func (f *fakeDiscovery) Lookup(deviceID protocol.DeviceID) (addresses []string, err error) {
func (f *fakeDiscovery) Lookup(_ context.Context, deviceID protocol.DeviceID) (addresses []string, err error) {
return f.addresses, nil
}
@@ -96,7 +99,7 @@ func TestCacheSlowLookup(t *testing.T) {
// Start a lookup, which will take at least a second
t0 := time.Now()
go c.Lookup(protocol.LocalDeviceID)
go c.Lookup(context.Background(), protocol.LocalDeviceID)
<-started // The slow lookup method has been called so we're inside the lock
// It should be possible to get ChildErrors while it's running
@@ -116,7 +119,7 @@ type slowDiscovery struct {
started chan struct{}
}
func (f *slowDiscovery) Lookup(deviceID protocol.DeviceID) (addresses []string, err error) {
func (f *slowDiscovery) Lookup(_ context.Context, deviceID protocol.DeviceID) (addresses []string, err error) {
close(f.started)
time.Sleep(f.delay)
return nil, nil
+2 -1
View File
@@ -7,6 +7,7 @@
package discover
import (
"context"
"time"
"github.com/syncthing/syncthing/lib/protocol"
@@ -15,7 +16,7 @@ import (
// A Finder provides lookup services of some kind.
type Finder interface {
Lookup(deviceID protocol.DeviceID) (address []string, err error)
Lookup(ctx context.Context, deviceID protocol.DeviceID) (address []string, err error)
Error() error
String() string
Cache() map[protocol.DeviceID]CacheEntry
+44 -15
View File
@@ -41,8 +41,8 @@ type globalClient struct {
}
type httpClient interface {
Get(url string) (*http.Response, error)
Post(url, ctype string, data io.Reader) (*http.Response, error)
Get(ctx context.Context, url string) (*http.Response, error)
Post(ctx context.Context, url, ctype string, data io.Reader) (*http.Response, error)
}
const (
@@ -89,7 +89,7 @@ func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, evLo
// The http.Client used for announcements. It needs to have our
// certificate to prove our identity, and may or may not verify the server
// certificate depending on the insecure setting.
var announceClient httpClient = &http.Client{
var announceClient httpClient = &contextClient{&http.Client{
Timeout: requestTimeout,
Transport: &http.Transport{
DialContext: dialer.DialContext,
@@ -99,14 +99,14 @@ func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, evLo
Certificates: []tls.Certificate{cert},
},
},
}
}}
if opts.id != "" {
announceClient = newIDCheckingHTTPClient(announceClient, devID)
}
// The http.Client used for queries. We don't need to present our
// certificate here, so lets not include it. May be insecure if requested.
var queryClient httpClient = &http.Client{
var queryClient httpClient = &contextClient{&http.Client{
Timeout: requestTimeout,
Transport: &http.Transport{
DialContext: dialer.DialContext,
@@ -115,7 +115,7 @@ func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, evLo
InsecureSkipVerify: opts.insecure,
},
},
}
}}
if opts.id != "" {
queryClient = newIDCheckingHTTPClient(queryClient, devID)
}
@@ -139,7 +139,7 @@ func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, evLo
}
// Lookup returns the list of addresses where the given device is available
func (c *globalClient) Lookup(device protocol.DeviceID) (addresses []string, err error) {
func (c *globalClient) Lookup(ctx context.Context, device protocol.DeviceID) (addresses []string, err error) {
if c.noLookup {
return nil, lookupError{
error: errors.New("lookups not supported"),
@@ -156,7 +156,7 @@ func (c *globalClient) Lookup(device protocol.DeviceID) (addresses []string, err
q.Set("device", device.String())
qURL.RawQuery = q.Encode()
resp, err := c.queryClient.Get(qURL.String())
resp, err := c.queryClient.Get(ctx, qURL.String())
if err != nil {
l.Debugln("globalClient.Lookup", qURL, err)
return nil, err
@@ -211,7 +211,7 @@ func (c *globalClient) serve(ctx context.Context) {
timer.Reset(2 * time.Second)
case <-timer.C:
c.sendAnnouncement(timer)
c.sendAnnouncement(ctx, timer)
case <-ctx.Done():
return
@@ -219,7 +219,7 @@ func (c *globalClient) serve(ctx context.Context) {
}
}
func (c *globalClient) sendAnnouncement(timer *time.Timer) {
func (c *globalClient) sendAnnouncement(ctx context.Context, timer *time.Timer) {
var ann announcement
if c.addrList != nil {
ann.Addresses = c.addrList.ExternalAddresses()
@@ -239,7 +239,7 @@ func (c *globalClient) sendAnnouncement(timer *time.Timer) {
l.Debugf("Announcement: %s", postData)
resp, err := c.announceClient.Post(c.server, "application/json", bytes.NewReader(postData))
resp, err := c.announceClient.Post(ctx, c.server, "application/json", bytes.NewReader(postData))
if err != nil {
l.Debugln("announce POST:", err)
c.setError(err)
@@ -362,8 +362,8 @@ func (c *idCheckingHTTPClient) check(resp *http.Response) error {
return nil
}
func (c *idCheckingHTTPClient) Get(url string) (*http.Response, error) {
resp, err := c.httpClient.Get(url)
func (c *idCheckingHTTPClient) Get(ctx context.Context, url string) (*http.Response, error) {
resp, err := c.httpClient.Get(ctx, url)
if err != nil {
return nil, err
}
@@ -374,8 +374,8 @@ func (c *idCheckingHTTPClient) Get(url string) (*http.Response, error) {
return resp, nil
}
func (c *idCheckingHTTPClient) Post(url, ctype string, data io.Reader) (*http.Response, error) {
resp, err := c.httpClient.Post(url, ctype, data)
func (c *idCheckingHTTPClient) Post(ctx context.Context, url, ctype string, data io.Reader) (*http.Response, error) {
resp, err := c.httpClient.Post(ctx, url, ctype, data)
if err != nil {
return nil, err
}
@@ -403,3 +403,32 @@ func (e *errorHolder) Error() error {
e.mut.Unlock()
return err
}
type contextClient struct {
*http.Client
}
func (c *contextClient) Get(ctx context.Context, url string) (*http.Response, error) {
// For <go1.13 compatibility. Use the following commented line once that
// isn't required anymore.
// req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Cancel = ctx.Done()
return c.Client.Do(req)
}
func (c *contextClient) Post(ctx context.Context, url, ctype string, data io.Reader) (*http.Response, error) {
// For <go1.13 compatibility. Use the following commented line once that
// isn't required anymore.
// req, err := http.NewRequestWithContext(ctx, "POST", url, data)
req, err := http.NewRequest("POST", url, data)
if err != nil {
return nil, err
}
req.Cancel = ctx.Done()
req.Header.Set("Content-Type", ctype)
return c.Client.Do(req)
}
+2 -1
View File
@@ -7,6 +7,7 @@
package discover
import (
"context"
"crypto/tls"
"io/ioutil"
"net"
@@ -225,7 +226,7 @@ func testLookup(url string) ([]string, error) {
go disco.Serve()
defer disco.Stop()
return disco.Lookup(protocol.LocalDeviceID)
return disco.Lookup(context.Background(), protocol.LocalDeviceID)
}
type fakeDiscoveryServer struct {
+1 -1
View File
@@ -91,7 +91,7 @@ func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister, evLogge
}
// Lookup returns a list of addresses the device is available at.
func (c *localClient) Lookup(device protocol.DeviceID) (addresses []string, err error) {
func (c *localClient) Lookup(_ context.Context, device protocol.DeviceID) (addresses []string, err error) {
if cache, ok := c.Get(device); ok {
if time.Since(cache.when) < CacheLifeTime {
addresses = cache.Addresses
+24 -13
View File
@@ -160,12 +160,6 @@ USER-AGENT: syncthing/1.0
}
defer socket.Close() // Make sure our socket gets closed
err = socket.SetDeadline(time.Now().Add(timeout))
if err != nil {
l.Debugln("UPnP discovery: setting socket deadline:", err)
return
}
l.Debugln("Sending search request for device type", deviceType, "on", intf.Name)
_, err = socket.WriteTo(search, ssdp)
@@ -178,16 +172,33 @@ USER-AGENT: syncthing/1.0
l.Debugln("Listening for UPnP response for device type", deviceType, "on", intf.Name)
// Listen for responses until a timeout is reached
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
// Listen for responses until a timeout is reached or the context is
// cancelled
resp := make([]byte, 65536)
loop:
for {
resp := make([]byte, 65536)
n, _, err := socket.ReadFrom(resp)
if err != nil {
if e, ok := err.(net.Error); !ok || !e.Timeout() {
l.Infoln("UPnP read:", err) //legitimate error, not a timeout.
}
if err := socket.SetDeadline(time.Now().Add(250 * time.Millisecond)); err != nil {
l.Infoln("UPnP socket:", err)
break
}
n, _, err := socket.ReadFrom(resp)
if err != nil {
select {
case <-ctx.Done():
break loop
default:
}
if e, ok := err.(net.Error); ok && e.Timeout() {
continue // continue reading
}
l.Infoln("UPnP read:", err) //legitimate error, not a timeout.
break
}
igds, err := parseResponse(ctx, deviceType, resp[:n])
if err != nil {
switch err.(type) {