felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: switch database engine to sqlite (fixes #9954) (#9965)

Browse Source 
Switch the database from LevelDB to SQLite, for greater stability and
simpler code.

Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl>
Co-authored-by: bt90 <btom1990@googlemail.com>
This commit is contained in:
Jakob Borg
2025-03-29 13:50:08 +01:00
committed by GitHub
co-authored by Tommy van der Vorst bt90
parent b1c8f88a44
commit 025905fcdf
146 changed files with 8315 additions and 11984 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/api/api.go
+9 -28
View File
@@ -40,10 +40,10 @@ import (
"golang.org/x/text/transform"
"golang.org/x/text/unicode/norm"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
@@ -91,7 +91,7 @@ type service struct {
startupErr error
listenerAddr net.Addr
exitChan chan *svcutil.FatalErr
miscDB *db.NamespacedKV
miscDB *db.Typed
shutdownTimeout time.Duration
guiErrors logger.Recorder
@@ -106,7 +106,7 @@ type Service interface {
WaitForStart() error
}
func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, noUpgrade bool, miscDB *db.NamespacedKV) Service {
func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, noUpgrade bool, miscDB *db.Typed) Service {
return &service{
id: id,
cfg: cfg,
@@ -984,16 +984,11 @@ func (s *service) getDBFile(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
mtimeMapping, mtimeErr := s.model.GetMtimeMapping(folder, file)
sendJSON(w, map[string]interface{}{
"global": jsonFileInfo(gf),
"local": jsonFileInfo(lf),
"availability": av,
"mtime": map[string]interface{}{
"err": mtimeErr,
"value": mtimeMapping,
},
})
}
@@ -1002,28 +997,14 @@ func (s *service) getDebugFile(w http.ResponseWriter, r *http.Request) {
folder := qs.Get("folder")
file := qs.Get("file")
snap, err := s.model.DBSnapshot(folder)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
mtimeMapping, mtimeErr := s.model.GetMtimeMapping(folder, file)
lf, _ := snap.Get(protocol.LocalDeviceID, file)
gf, _ := snap.GetGlobal(file)
av := snap.Availability(file)
vl := snap.DebugGlobalVersions(file)
lf, _, _ := s.model.CurrentFolderFile(folder, file)
gf, _, _ := s.model.CurrentGlobalFile(folder, file)
av, _ := s.model.Availability(folder, protocol.FileInfo{Name: file}, protocol.BlockInfo{})
sendJSON(w, map[string]interface{}{
"global": jsonFileInfo(gf),
"local": jsonFileInfo(lf),
"availability": av,
"globalVersions": vl.String(),
"mtime": map[string]interface{}{
"err": mtimeErr,
"value": mtimeMapping,
},
"global": jsonFileInfo(gf),
"local": jsonFileInfo(lf),
"availability": av,
})
}
lib/api/api_auth_test.go
+10 -5
View File
@@ -10,10 +10,9 @@ import (
"testing"
"time"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
)
var guiCfg config.GUIConfiguration
@@ -131,8 +130,14 @@ func (c *mockClock) wind(t time.Duration) {
func TestTokenManager(t *testing.T) {
t.Parallel()
mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
kdb := db.NewNamespacedKV(mdb, "test")
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mdb.Close()
})
kdb := db.NewMiscDB(mdb)
clock := &mockClock{now: time.Now()}
// Token manager keeps up to three tokens with a validity time of 24 hours.
lib/api/api_csrf.go
+2 -2
View File
@@ -11,7 +11,7 @@ import (
"strings"
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/internal/db"
)
const (
@@ -34,7 +34,7 @@ type apiKeyValidator interface {
// Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
// the request with 403. For / and /index.html, set a new CSRF cookie if none
// is currently set.
func newCsrfManager(unique string, prefix string, apiKeyValidator apiKeyValidator, next http.Handler, miscDB *db.NamespacedKV) *csrfManager {
func newCsrfManager(unique string, prefix string, apiKeyValidator apiKeyValidator, next http.Handler, miscDB *db.Typed) *csrfManager {
m := &csrfManager{
unique: unique,
prefix: prefix,
lib/api/api_test.go
+56 -108
View File
@@ -27,12 +27,12 @@ import (
"github.com/d4l3k/messagediff"
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/assets"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
connmocks "github.com/syncthing/syncthing/lib/connections/mocks"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
discovermocks "github.com/syncthing/syncthing/lib/discover/mocks"
"github.com/syncthing/syncthing/lib/events"
eventmocks "github.com/syncthing/syncthing/lib/events/mocks"
@@ -84,8 +84,14 @@ func TestStopAfterBrokenConfig(t *testing.T) {
}
w := config.Wrap("/dev/null", cfg, protocol.LocalDeviceID, events.NoopLogger)
mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
kdb := db.NewMiscDataNamespace(mdb)
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mdb.Close()
})
kdb := db.NewMiscDB(mdb)
srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, events.NoopLogger, nil, nil, nil, nil, nil, nil, false, kdb).(*service)
srv.started = make(chan string)
@@ -217,11 +223,7 @@ type httpTestCase struct {
func TestAPIServiceRequests(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, apiCfg)
cases := []httpTestCase{
// /rest/db
@@ -598,11 +600,7 @@ func TestHTTPLogin(t *testing.T) {
APIKey: testAPIKey,
SendBasicAuthPrompt: sendBasicAuthPrompt,
})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, cfg)
url := baseURL + path
t.Run(fmt.Sprintf("%d path", expectedOkStatus), func(t *testing.T) {
@@ -795,13 +793,9 @@ func TestHTTPLogin(t *testing.T) {
w := initConfig(initialPassword, t)
{
baseURL, cancel, err := startHTTPWithShutdownTimeout(w, shutdownTimeout)
baseURL := startHTTPWithShutdownTimeout(t, w, shutdownTimeout)
cfgPath := baseURL + "/rest/config"
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusOK {
@@ -813,12 +807,8 @@ func TestHTTPLogin(t *testing.T) {
httpRequest(http.MethodPut, cfgPath, cfg, "", "", testAPIKey, "", "", "", nil, t)
}
{
baseURL, cancel, err := startHTTP(w)
baseURL := startHTTP(t, w)
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusForbidden {
@@ -837,13 +827,9 @@ func TestHTTPLogin(t *testing.T) {
w := initConfig(initialPassword, t)
{
baseURL, cancel, err := startHTTPWithShutdownTimeout(w, shutdownTimeout)
baseURL := startHTTPWithShutdownTimeout(t, w, shutdownTimeout)
cfgPath := baseURL + "/rest/config/gui"
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusOK {
@@ -855,12 +841,8 @@ func TestHTTPLogin(t *testing.T) {
httpRequest(http.MethodPut, cfgPath, cfg.GUI, "", "", testAPIKey, "", "", "", nil, t)
}
{
baseURL, cancel, err := startHTTP(w)
baseURL := startHTTP(t, w)
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusForbidden {
@@ -885,11 +867,7 @@ func TestHtmlFormLogin(t *testing.T) {
Password: "$2a$10$IdIZTxTg/dCNuNEGlmLynOjqg4B1FvDKuIV5e0BB3pnWVHNb8.GSq", // bcrypt of "räksmörgås" in UTF-8
SendBasicAuthPrompt: false,
})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, cfg)
loginUrl := baseURL + "/rest/noauth/auth/password"
resourceUrl := baseURL + "/meta.js"
@@ -1030,11 +1008,7 @@ func TestApiCache(t *testing.T) {
RawAddress: "127.0.0.1:0",
APIKey: testAPIKey,
})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, cfg)
httpGet := func(url string, bearer string) *http.Response {
return httpGet(url, "", "", "", bearer, nil, t)
@@ -1059,11 +1033,11 @@ func TestApiCache(t *testing.T) {
})
}
func startHTTP(cfg config.Wrapper) (string, context.CancelFunc, error) {
return startHTTPWithShutdownTimeout(cfg, 0)
func startHTTP(t *testing.T, cfg config.Wrapper) string {
return startHTTPWithShutdownTimeout(t, cfg, 0)
}
func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Duration) (string, context.CancelFunc, error) {
func startHTTPWithShutdownTimeout(t *testing.T, cfg config.Wrapper, shutdownTimeout time.Duration) string {
m := new(modelmocks.Model)
assetDir := "../../gui"
eventSub := new(eventmocks.BufferedSubscription)
@@ -1086,12 +1060,18 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
// Instantiate the API service
urService := ur.New(cfg, m, connections, false)
mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
kdb := db.NewMiscDataNamespace(mdb)
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mdb.Close()
})
kdb := db.NewMiscDB(mdb)
svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, mockedSummary, errorLog, systemLog, false, kdb).(*service)
svc.started = addrChan
if shutdownTimeout > 0*time.Millisecond {
if shutdownTimeout > 0 {
svc.shutdownTimeout = shutdownTimeout
}
@@ -1101,14 +1081,14 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
})
supervisor.Add(svc)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
supervisor.ServeBackground(ctx)
// Make sure the API service is listening, and get the URL to use.
addr := <-addrChan
tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
if err != nil {
cancel()
return "", cancel, fmt.Errorf("weird address from API service: %w", err)
t.Fatal(fmt.Errorf("weird address from API service: %w", err))
}
host, _, _ := net.SplitHostPort(cfg.GUI().RawAddress)
@@ -1117,17 +1097,13 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
}
baseURL := fmt.Sprintf("http://%s", net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port)))
return baseURL, cancel, nil
return baseURL
}
func TestCSRFRequired(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal("Unexpected error from getting base URL:", err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Minute,
@@ -1245,11 +1221,7 @@ func TestCSRFRequired(t *testing.T) {
func TestRandomString(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1304,7 +1276,7 @@ func TestConfigPostOK(t *testing.T) {
]
}`))
resp, err := testConfigPost(cfg)
resp, err := testConfigPost(t, cfg)
if err != nil {
t.Fatal(err)
}
@@ -1325,7 +1297,7 @@ func TestConfigPostDupFolder(t *testing.T) {
]
}`))
resp, err := testConfigPost(cfg)
resp, err := testConfigPost(t, cfg)
if err != nil {
t.Fatal(err)
}
@@ -1334,12 +1306,10 @@ func TestConfigPostDupFolder(t *testing.T) {
}
}
func testConfigPost(data io.Reader) (*http.Response, error) {
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
return nil, err
}
defer cancel()
func testConfigPost(t *testing.T, data io.Reader) (*http.Response, error) {
t.Helper()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1356,11 +1326,7 @@ func TestHostCheck(t *testing.T) {
cfg := newMockedConfig()
cfg.GUIReturns(config.GUIConfiguration{RawAddress: "127.0.0.1:0"})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, cfg)
// A normal HTTP get to the localhost-bound service should succeed
@@ -1419,11 +1385,7 @@ func TestHostCheck(t *testing.T) {
RawAddress: "127.0.0.1:0",
InsecureSkipHostCheck: true,
})
baseURL, cancel, err = startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL = startHTTP(t, cfg)
// A request with a suspicious Host header should be allowed
@@ -1445,11 +1407,7 @@ func TestHostCheck(t *testing.T) {
cfg.GUIReturns(config.GUIConfiguration{
RawAddress: "0.0.0.0:0",
})
baseURL, cancel, err = startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL = startHTTP(t, cfg)
// A request with a suspicious Host header should be allowed
@@ -1476,11 +1434,7 @@ func TestHostCheck(t *testing.T) {
cfg.GUIReturns(config.GUIConfiguration{
RawAddress: "[::1]:0",
})
baseURL, cancel, err = startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL = startHTTP(t, cfg)
// A normal HTTP get to the localhost-bound service should succeed
@@ -1568,11 +1522,7 @@ func TestAddressIsLocalhost(t *testing.T) {
func TestAccessControlAllowOriginHeader(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1596,11 +1546,7 @@ func TestAccessControlAllowOriginHeader(t *testing.T) {
func TestOptionsRequest(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1632,8 +1578,14 @@ func TestEventMasks(t *testing.T) {
cfg := newMockedConfig()
defSub := new(eventmocks.BufferedSubscription)
diskSub := new(eventmocks.BufferedSubscription)
mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
kdb := db.NewMiscDataNamespace(mdb)
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mdb.Close()
})
kdb := db.NewMiscDB(mdb)
svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, events.NoopLogger, nil, nil, nil, nil, nil, nil, false, kdb).(*service)
if mask := svc.getEventMask(""); mask != DefaultEventMask {
@@ -1780,11 +1732,7 @@ func TestConfigChanges(t *testing.T) {
cfgCtx, cfgCancel := context.WithCancel(context.Background())
go w.Serve(cfgCtx)
defer cfgCancel()
baseURL, cancel, err := startHTTP(w)
if err != nil {
t.Fatal("Unexpected error from getting base URL:", err)
}
defer cancel()
baseURL := startHTTP(t, w)
cli := &http.Client{
Timeout: time.Minute,
lib/api/tokenmanager.go
+4 -4
View File
@@ -14,9 +14,9 @@ import (
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/gen/apiproto"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
@@ -24,7 +24,7 @@ import (
type tokenManager struct {
key string
miscDB *db.NamespacedKV
miscDB *db.Typed
lifetime time.Duration
maxItems int
@@ -35,7 +35,7 @@ type tokenManager struct {
saveTimer *time.Timer
}
func newTokenManager(key string, miscDB *db.NamespacedKV, lifetime time.Duration, maxItems int) *tokenManager {
func newTokenManager(key string, miscDB *db.Typed, lifetime time.Duration, maxItems int) *tokenManager {
var tokens apiproto.TokenSet
if bs, ok, _ := miscDB.Bytes(key); ok {
_ = proto.Unmarshal(bs, &tokens) // best effort
@@ -152,7 +152,7 @@ type tokenCookieManager struct {
tokens *tokenManager
}
func newTokenCookieManager(shortID string, guiCfg config.GUIConfiguration, evLogger events.Logger, miscDB *db.NamespacedKV) *tokenCookieManager {
func newTokenCookieManager(shortID string, guiCfg config.GUIConfiguration, evLogger events.Logger, miscDB *db.Typed) *tokenCookieManager {
return &tokenCookieManager{
cookieName: "sessionid-" + shortID,
shortID: shortID,
lib/build/build.go
+29 -1
View File
@@ -18,7 +18,7 @@ import (
"time"
)
const Codename = "Gold Grasshopper"
const Codename = "Hafnium Hornet"
var (
// Injected by build script
@@ -28,6 +28,9 @@ var (
Stamp = "0"
Tags = ""
// Added to by other packages
extraTags []string
// Set by init()
Date time.Time
IsRelease bool
@@ -43,6 +46,11 @@ var (
"STNORESTART",
"STNOUPGRADE",
}
replaceTags = map[string]string{
"sqlite_omit_load_extension": "",
"osusergo": "",
"netgo": "",
}
)
const versionExtraAllowedChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-. "
@@ -108,8 +116,23 @@ func TagsList() []string {
if Extra != "" {
tags = append(tags, Extra)
}
tags = append(tags, extraTags...)
// Replace any tag values we want to have more user friendly versions,
// or be removed
for i, tag := range tags {
if repl, ok := replaceTags[tag]; ok {
tags[i] = repl
}
}
sort.Strings(tags)
// Remove any empty tags, which will be at the front of the list now
for len(tags) > 0 && tags[0] == "" {
tags = tags[1:]
}
return tags
}
@@ -124,3 +147,8 @@ func filterString(s, allowedChars string) string {
}
return res.String()
}
func AddTag(tag string) {
extraTags = append(extraTags, tag)
LongVersion = LongVersionFor("syncthing")
}
lib/config/config_test.go
+2 -2
View File
@@ -484,7 +484,7 @@ func TestIssue1262(t *testing.T) {
t.Fatal(err)
}
actual := cfg.Folders()["test"].Filesystem(nil).URI()
actual := cfg.Folders()["test"].Filesystem().URI()
expected := `e:\`
if actual != expected {
@@ -521,7 +521,7 @@ func TestFolderPath(t *testing.T) {
Path: "~/tmp",
}
realPath := folder.Filesystem(nil).URI()
realPath := folder.Filesystem().URI()
if !filepath.IsAbs(realPath) {
t.Error(realPath, "should be absolute")
}
lib/config/folderconfiguration.go
+9 -12
View File
@@ -20,7 +20,6 @@ import (
"github.com/shirou/gopsutil/v4/disk"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -119,26 +118,24 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
// Filesystem creates a filesystem for the path and options of this folder.
// The fset parameter may be nil, in which case no mtime handling on top of
// the filesystem is provided.
func (f FolderConfiguration) Filesystem(fset *db.FileSet) fs.Filesystem {
func (f FolderConfiguration) Filesystem(extraOpts ...fs.Option) fs.Filesystem {
// This is intentionally not a pointer method, because things like
// cfg.Folders["default"].Filesystem(nil) should be valid.
opts := make([]fs.Option, 0, 3)
var opts []fs.Option
if f.FilesystemType == FilesystemTypeBasic && f.JunctionsAsDirs {
opts = append(opts, new(fs.OptionJunctionsAsDirs))
}
if !f.CaseSensitiveFS {
opts = append(opts, new(fs.OptionDetectCaseConflicts))
}
if fset != nil {
opts = append(opts, fset.MtimeOption())
}
opts = append(opts, extraOpts...)
return fs.NewFilesystem(f.FilesystemType.ToFS(), f.Path, opts...)
}
func (f FolderConfiguration) ModTimeWindow() time.Duration {
dur := time.Duration(f.RawModTimeWindowS) * time.Second
if f.RawModTimeWindowS < 1 && build.IsAndroid {
if usage, err := disk.Usage(f.Filesystem(nil).URI()); err != nil {
if usage, err := disk.Usage(f.Filesystem().URI()); err != nil {
dur = 2 * time.Second
l.Debugf(`Detecting FS at "%v" on android: Setting mtime window to 2s: err == "%v"`, f.Path, err)
} else if strings.HasPrefix(strings.ToLower(usage.Fstype), "ext2") || strings.HasPrefix(strings.ToLower(usage.Fstype), "ext3") || strings.HasPrefix(strings.ToLower(usage.Fstype), "ext4") {
@@ -162,7 +159,7 @@ func (f *FolderConfiguration) CreateMarker() error {
return nil
}
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
// Create the marker as a directory
err := ffs.Mkdir(DefaultMarkerName, 0o755)
@@ -189,7 +186,7 @@ func (f *FolderConfiguration) CreateMarker() error {
}
func (f *FolderConfiguration) RemoveMarker() error {
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
_ = ffs.Remove(filepath.Join(DefaultMarkerName, f.markerFilename()))
return ffs.Remove(DefaultMarkerName)
}
@@ -209,7 +206,7 @@ func (f *FolderConfiguration) markerContents() []byte {
// CheckPath returns nil if the folder root exists and contains the marker file
func (f *FolderConfiguration) CheckPath() error {
return f.checkFilesystemPath(f.Filesystem(nil), ".")
return f.checkFilesystemPath(f.Filesystem(), ".")
}
func (f *FolderConfiguration) checkFilesystemPath(ffs fs.Filesystem, path string) error {
@@ -252,7 +249,7 @@ func (f *FolderConfiguration) CreateRoot() (err error) {
permBits = 0o700
}
filesystem := f.Filesystem(nil)
filesystem := f.Filesystem()
if _, err = filesystem.Stat("."); fs.IsNotExist(err) {
err = filesystem.MkdirAll(".", permBits)
@@ -363,7 +360,7 @@ func (f *FolderConfiguration) CheckAvailableSpace(req uint64) error {
if val <= 0 {
return nil
}
fs := f.Filesystem(nil)
fs := f.Filesystem()
usage, err := fs.Usage(".")
if err != nil {
return nil //nolint: nilerr
lib/config/migrations.go
+4 -4
View File
@@ -208,7 +208,7 @@ func migrateToConfigV23(cfg *Configuration) {
// marker name in later versions.
for i := range cfg.Folders {
fs := cfg.Folders[i].Filesystem(nil)
fs := cfg.Folders[i].Filesystem()
// Invalid config posted, or tests.
if fs == nil {
continue
@@ -244,18 +244,18 @@ func migrateToConfigV21(cfg *Configuration) {
switch folder.Versioning.Type {
case "simple", "trashcan":
// Clean out symlinks in the known place
cleanSymlinks(folder.Filesystem(nil), ".stversions")
cleanSymlinks(folder.Filesystem(), ".stversions")
case "staggered":
versionDir := folder.Versioning.Params["versionsPath"]
if versionDir == "" {
// default place
cleanSymlinks(folder.Filesystem(nil), ".stversions")
cleanSymlinks(folder.Filesystem(), ".stversions")
} else if filepath.IsAbs(versionDir) {
// absolute
cleanSymlinks(fs.NewFilesystem(fs.FilesystemTypeBasic, versionDir), ".")
} else {
// relative to folder
cleanSymlinks(folder.Filesystem(nil), versionDir)
cleanSymlinks(folder.Filesystem(), versionDir)
}
}
}
lib/config/optionsconfiguration.go
-1
View File
@@ -61,7 +61,6 @@ type OptionsConfiguration struct {
StunKeepaliveStartS int `json:"stunKeepaliveStartS" xml:"stunKeepaliveStartS" default:"180"`
StunKeepaliveMinS int `json:"stunKeepaliveMinS" xml:"stunKeepaliveMinS" default:"20"`
RawStunServers []string `json:"stunServers" xml:"stunServer" default:"default"`
DatabaseTuning Tuning `json:"databaseTuning" xml:"databaseTuning" restart:"true"`
RawMaxCIRequestKiB int `json:"maxConcurrentIncomingRequestKiB" xml:"maxConcurrentIncomingRequestKiB"`
AnnounceLANAddresses bool `json:"announceLANAddresses" xml:"announceLANAddresses" default:"true"`
SendFullIndexOnUpgrade bool `json:"sendFullIndexOnUpgrade" xml:"sendFullIndexOnUpgrade"`
lib/config/tuning.go
-46
View File
@@ -1,46 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package config
type Tuning int32
const (
TuningAuto Tuning = 0
TuningSmall Tuning = 1
TuningLarge Tuning = 2
)
func (t Tuning) String() string {
switch t {
case TuningAuto:
return "auto"
case TuningSmall:
return "small"
case TuningLarge:
return "large"
default:
return "unknown"
}
}
func (t Tuning) MarshalText() ([]byte, error) {
return []byte(t.String()), nil
}
func (t *Tuning) UnmarshalText(bs []byte) error {
switch string(bs) {
case "auto":
*t = TuningAuto
case "small":
*t = TuningSmall
case "large":
*t = TuningLarge
default:
*t = TuningAuto
}
return nil
}
lib/config/tuning_test.go
-26
View File
@@ -1,26 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package config_test
import (
"testing"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db/backend"
)
func TestTuningMatches(t *testing.T) {
if int(config.TuningAuto) != int(backend.TuningAuto) {
t.Error("mismatch for TuningAuto")
}
if int(config.TuningSmall) != int(backend.TuningSmall) {
t.Error("mismatch for TuningSmall")
}
if int(config.TuningLarge) != int(backend.TuningLarge) {
t.Error("mismatch for TuningLarge")
}
}
lib/db/.gitignore
-2
View File
@@ -1,2 +0,0 @@
!*.zip
testdata/*.db
lib/db/backend/backend.go
-187
View File
@@ -1,187 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package backend
import (
"errors"
"sync"
)
// CommitHook is a function that is executed before a WriteTransaction is
// committed or before it is flushed to disk, e.g. on calling CheckPoint. The
// transaction can be accessed via a closure.
type CommitHook func(WriteTransaction) error
// The Reader interface specifies the read-only operations available on the
// main database and on read-only transactions (snapshots). Note that when
// called directly on the database handle these operations may take implicit
// transactions and performance may suffer.
type Reader interface {
Get(key []byte) ([]byte, error)
NewPrefixIterator(prefix []byte) (Iterator, error)
NewRangeIterator(first, last []byte) (Iterator, error)
}
// The Writer interface specifies the mutating operations available on the
// main database and on writable transactions. Note that when called
// directly on the database handle these operations may take implicit
// transactions and performance may suffer.
type Writer interface {
Put(key, val []byte) error
Delete(key []byte) error
}
// The ReadTransaction interface specifies the operations on read-only
// transactions. Every ReadTransaction must be released when no longer
// required.
type ReadTransaction interface {
Reader
Release()
}
// The WriteTransaction interface specifies the operations on writable
// transactions. Every WriteTransaction must be either committed or released
// (i.e., discarded) when no longer required. No further operations must be
// performed after release or commit (regardless of whether commit succeeded),
// with one exception -- it's fine to release an already committed or released
// transaction.
//
// A Checkpoint is a potential partial commit of the transaction so far, for
// purposes of saving memory when transactions are in-RAM. Note that
// transactions may be checkpointed *anyway* even if this is not called, due to
// resource constraints, but this gives you a chance to decide when. If, and
// only if, calling Checkpoint will result in a partial commit/flush, the
// CommitHooks passed to Backend.NewWriteTransaction are called before
// committing. If any of those returns an error, committing is aborted and the
// error bubbled.
type WriteTransaction interface {
ReadTransaction
Writer
Checkpoint() error
Commit() error
}
// The Iterator interface specifies the operations available on iterators
// returned by NewPrefixIterator and NewRangeIterator. The iterator pattern
// is to loop while Next returns true, then check Error after the loop. Next
// will return false when iteration is complete (Error() == nil) or when
// there is an error preventing iteration, which is then returned by
// Error(). For example:
//
// it, err := db.NewPrefixIterator(nil)
// if err != nil {
// // problem preventing iteration
// }
// defer it.Release()
// for it.Next() {
// // ...
// }
// if err := it.Error(); err != nil {
// // there was a database problem while iterating
// }
//
// An iterator must be Released when no longer required. The Error method
// can be called either before or after Release with the same results. If an
// iterator was created in a transaction (whether read-only or write) it
// must be released before the transaction is released (or committed).
type Iterator interface {
Next() bool
Key() []byte
Value() []byte
Error() error
Release()
}
// The Backend interface represents the main database handle. It supports
// both read/write operations and opening read-only or writable
// transactions. Depending on the actual implementation, individual
// read/write operations may be implicitly wrapped in transactions, making
// them perform quite badly when used repeatedly. For bulk operations,
// consider always using a transaction of the appropriate type. The
// transaction isolation level is "read committed" - there are no dirty
// reads.
// Location returns the path to the database, as given to Open. The returned string
// is empty for a db in memory.
type Backend interface {
Reader
Writer
NewReadTransaction() (ReadTransaction, error)
NewWriteTransaction(hooks ...CommitHook) (WriteTransaction, error)
Close() error
Compact() error
Location() string
}
type Tuning int
const (
// N.b. these constants must match those in lib/config.Tuning!
TuningAuto Tuning = iota
TuningSmall
TuningLarge
)
func Open(path string, tuning Tuning) (Backend, error) {
return OpenLevelDB(path, tuning)
}
func OpenMemory() Backend {
return OpenLevelDBMemory()
}
var (
errClosed = errors.New("database is closed")
errNotFound = errors.New("key not found")
)
func IsClosed(err error) bool { return errors.Is(err, errClosed) }
func IsNotFound(err error) bool { return errors.Is(err, errNotFound) }
// releaser manages counting on top of a waitgroup
type releaser struct {
wg *closeWaitGroup
once sync.Once
}
func newReleaser(wg *closeWaitGroup) (*releaser, error) {
if err := wg.Add(1); err != nil {
return nil, err
}
return &releaser{wg: wg}, nil
}
func (r *releaser) Release() {
// We use the Once because we may get called multiple times from
// Commit() and deferred Release().
r.once.Do(r.wg.Done)
}
// closeWaitGroup behaves just like a sync.WaitGroup, but does not require
// a single routine to do the Add and Wait calls. If Add is called after
// CloseWait, it will return an error, and both are safe to be used concurrently.
type closeWaitGroup struct {
sync.WaitGroup
closed bool
closeMut sync.RWMutex
}
func (cg *closeWaitGroup) Add(i int) error {
cg.closeMut.RLock()
defer cg.closeMut.RUnlock()
if cg.closed {
return errClosed
}
cg.WaitGroup.Add(i)
return nil
}
func (cg *closeWaitGroup) CloseWait() {
cg.closeMut.Lock()
cg.closed = true
cg.closeMut.Unlock()
cg.WaitGroup.Wait()
}
lib/db/backend/backend_test.go
-76
View File
@@ -1,76 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package backend
import "testing"
// testBackendBehavior is the generic test suite that must be fulfilled by
// every backend implementation. It should be called by each implementation
// as (part of) their test suite.
func testBackendBehavior(t *testing.T, open func() Backend) {
t.Run("WriteIsolation", func(t *testing.T) { testWriteIsolation(t, open) })
t.Run("DeleteNonexisten", func(t *testing.T) { testDeleteNonexistent(t, open) })
t.Run("IteratorClosedDB", func(t *testing.T) { testIteratorClosedDB(t, open) })
}
func testWriteIsolation(t *testing.T, open func() Backend) {
// Values written during a transaction should not be read back, our
// updateGlobal depends on this.
db := open()
defer db.Close()
// Sanity check
_ = db.Put([]byte("a"), []byte("a"))
v, _ := db.Get([]byte("a"))
if string(v) != "a" {
t.Fatal("read back should work")
}
// Now in a transaction we should still see the old value
tx, _ := db.NewWriteTransaction()
defer tx.Release()
_ = tx.Put([]byte("a"), []byte("b"))
v, _ = tx.Get([]byte("a"))
if string(v) != "a" {
t.Fatal("read in transaction should read the old value")
}
}
func testDeleteNonexistent(t *testing.T, open func() Backend) {
// Deleting a non-existent key is not an error
db := open()
defer db.Close()
err := db.Delete([]byte("a"))
if err != nil {
t.Error(err)
}
}
// Either creating the iterator or the .Error() method of the returned iterator
// should return an error and IsClosed(err) == true.
func testIteratorClosedDB(t *testing.T, open func() Backend) {
db := open()
_ = db.Put([]byte("a"), []byte("a"))
db.Close()
it, err := db.NewPrefixIterator(nil)
if err != nil {
if !IsClosed(err) {
t.Error("NewPrefixIterator: IsClosed(err) == false:", err)
}
return
}
it.Next()
if err := it.Error(); !IsClosed(err) {
t.Error("Next: IsClosed(err) == false:", err)
}
}
lib/db/backend/debug.go
-13
View File
@@ -1,13 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package backend
import (
"github.com/syncthing/syncthing/lib/logger"
)
var l = logger.DefaultLogger.NewFacility("backend", "The database backend")
lib/db/backend/leveldb_backend.go
-233
View File
@@ -1,233 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package backend
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/util"
)
const (
// Never flush transactions smaller than this, even on Checkpoint().
// This just needs to be just large enough to avoid flushing
// transactions when they are super tiny, thus creating millions of tiny
// transactions unnecessarily.
dbFlushBatchMin = 64 << KiB
// Once a transaction reaches this size, flush it unconditionally. This
// should be large enough to avoid forcing a flush between Checkpoint()
// calls in loops where we do those, so in principle just large enough
// to hold a FileInfo plus corresponding version list and metadata
// updates or two.
dbFlushBatchMax = 1 << MiB
)
// leveldbBackend implements Backend on top of a leveldb
type leveldbBackend struct {
ldb *leveldb.DB
closeWG *closeWaitGroup
location string
}
func newLeveldbBackend(ldb *leveldb.DB, location string) *leveldbBackend {
return &leveldbBackend{
ldb: ldb,
closeWG: &closeWaitGroup{},
location: location,
}
}
func (b *leveldbBackend) NewReadTransaction() (ReadTransaction, error) {
return b.newSnapshot()
}
func (b *leveldbBackend) newSnapshot() (leveldbSnapshot, error) {
rel, err := newReleaser(b.closeWG)
if err != nil {
return leveldbSnapshot{}, err
}
snap, err := b.ldb.GetSnapshot()
if err != nil {
rel.Release()
return leveldbSnapshot{}, wrapLeveldbErr(err)
}
return leveldbSnapshot{
snap: snap,
rel: rel,
}, nil
}
func (b *leveldbBackend) NewWriteTransaction(hooks ...CommitHook) (WriteTransaction, error) {
rel, err := newReleaser(b.closeWG)
if err != nil {
return nil, err
}
snap, err := b.newSnapshot()
if err != nil {
rel.Release()
return nil, err // already wrapped
}
return &leveldbTransaction{
leveldbSnapshot: snap,
ldb: b.ldb,
batch: new(leveldb.Batch),
rel: rel,
commitHooks: hooks,
inFlush: false,
}, nil
}
func (b *leveldbBackend) Close() error {
b.closeWG.CloseWait()
return wrapLeveldbErr(b.ldb.Close())
}
func (b *leveldbBackend) Get(key []byte) ([]byte, error) {
val, err := b.ldb.Get(key, nil)
return val, wrapLeveldbErr(err)
}
func (b *leveldbBackend) NewPrefixIterator(prefix []byte) (Iterator, error) {
return &leveldbIterator{b.ldb.NewIterator(util.BytesPrefix(prefix), nil)}, nil
}
func (b *leveldbBackend) NewRangeIterator(first, last []byte) (Iterator, error) {
return &leveldbIterator{b.ldb.NewIterator(&util.Range{Start: first, Limit: last}, nil)}, nil
}
func (b *leveldbBackend) Put(key, val []byte) error {
return wrapLeveldbErr(b.ldb.Put(key, val, nil))
}
func (b *leveldbBackend) Delete(key []byte) error {
return wrapLeveldbErr(b.ldb.Delete(key, nil))
}
func (b *leveldbBackend) Compact() error {
// Race is detected during testing when db is closed while compaction
// is ongoing.
err := b.closeWG.Add(1)
if err != nil {
return err
}
defer b.closeWG.Done()
return wrapLeveldbErr(b.ldb.CompactRange(util.Range{}))
}
func (b *leveldbBackend) Location() string {
return b.location
}
// leveldbSnapshot implements backend.ReadTransaction
type leveldbSnapshot struct {
snap *leveldb.Snapshot
rel *releaser
}
func (l leveldbSnapshot) Get(key []byte) ([]byte, error) {
val, err := l.snap.Get(key, nil)
return val, wrapLeveldbErr(err)
}
func (l leveldbSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) {
return l.snap.NewIterator(util.BytesPrefix(prefix), nil), nil
}
func (l leveldbSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) {
return l.snap.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
}
func (l leveldbSnapshot) Release() {
l.snap.Release()
l.rel.Release()
}
// leveldbTransaction implements backend.WriteTransaction using a batch (not
// an actual leveldb transaction)
type leveldbTransaction struct {
leveldbSnapshot
ldb *leveldb.DB
batch *leveldb.Batch
rel *releaser
commitHooks []CommitHook
inFlush bool
}
func (t *leveldbTransaction) Delete(key []byte) error {
t.batch.Delete(key)
return t.checkFlush(dbFlushBatchMax)
}
func (t *leveldbTransaction) Put(key, val []byte) error {
t.batch.Put(key, val)
return t.checkFlush(dbFlushBatchMax)
}
func (t *leveldbTransaction) Checkpoint() error {
return t.checkFlush(dbFlushBatchMin)
}
func (t *leveldbTransaction) Commit() error {
err := wrapLeveldbErr(t.flush())
t.leveldbSnapshot.Release()
t.rel.Release()
return err
}
func (t *leveldbTransaction) Release() {
t.leveldbSnapshot.Release()
t.rel.Release()
}
// checkFlush flushes and resets the batch if its size exceeds the given size.
func (t *leveldbTransaction) checkFlush(size int) error {
// Hooks might put values in the database, which triggers a checkFlush which might trigger a flush,
// which might trigger the hooks.
// Don't recurse...
if t.inFlush || len(t.batch.Dump()) < size {
return nil
}
return t.flush()
}
func (t *leveldbTransaction) flush() error {
t.inFlush = true
defer func() { t.inFlush = false }()
for _, hook := range t.commitHooks {
if err := hook(t); err != nil {
return err
}
}
if t.batch.Len() == 0 {
return nil
}
if err := t.ldb.Write(t.batch, nil); err != nil {
return wrapLeveldbErr(err)
}
t.batch.Reset()
return nil
}
type leveldbIterator struct {
iterator.Iterator
}
func (it *leveldbIterator) Error() error {
return wrapLeveldbErr(it.Iterator.Error())
}
// wrapLeveldbErr wraps errors so that the backend package can recognize them
func wrapLeveldbErr(err error) error {
switch err {
case leveldb.ErrClosed:
return errClosed
case leveldb.ErrNotFound:
return errNotFound
}
return err
}
lib/db/backend/leveldb_open.go
-231
View File
@@ -1,231 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package backend
import (
"fmt"
"os"
"strconv"
"strings"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
"github.com/syndtr/goleveldb/leveldb/util"
)
const (
dbMaxOpenFiles = 100
// A large database is > 200 MiB. It's a mostly arbitrary value, but
// it's also the case that each file is 2 MiB by default and when we
// have dbMaxOpenFiles of them we will need to start thrashing fd:s.
// Switching to large database settings causes larger files to be used
// when compacting, reducing the number.
dbLargeThreshold = dbMaxOpenFiles * (2 << MiB)
KiB = 10
MiB = 20
)
// OpenLevelDB attempts to open the database at the given location, and runs
// recovery on it if opening fails. Worst case, if recovery is not possible,
// the database is erased and created from scratch.
func OpenLevelDB(location string, tuning Tuning) (Backend, error) {
opts := optsFor(location, tuning)
ldb, err := open(location, opts)
if err != nil {
return nil, err
}
return newLeveldbBackend(ldb, location), nil
}
// OpenLevelDBAuto is OpenLevelDB with TuningAuto tuning.
func OpenLevelDBAuto(location string) (Backend, error) {
return OpenLevelDB(location, TuningAuto)
}
// OpenLevelDBRO attempts to open the database at the given location, read
// only.
func OpenLevelDBRO(location string) (Backend, error) {
opts := &opt.Options{
OpenFilesCacheCapacity: dbMaxOpenFiles,
ReadOnly: true,
}
ldb, err := open(location, opts)
if err != nil {
return nil, err
}
return newLeveldbBackend(ldb, location), nil
}
// OpenLevelDBMemory returns a new Backend referencing an in-memory database.
func OpenLevelDBMemory() Backend {
ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
return newLeveldbBackend(ldb, "")
}
// optsFor returns the database options to use when opening a database with
// the given location and tuning. Settings can be overridden by debug
// environment variables.
func optsFor(location string, tuning Tuning) *opt.Options {
large := false
switch tuning {
case TuningLarge:
large = true
case TuningAuto:
large = dbIsLarge(location)
}
var (
// Set defaults used for small databases.
defaultBlockCacheCapacity = 0 // 0 means let leveldb use default
defaultBlockSize = 0
defaultCompactionTableSize = 0
defaultCompactionTableSizeMultiplier = 0
defaultWriteBuffer = 16 << MiB // increased from leveldb default of 4 MiB
defaultCompactionL0Trigger = opt.DefaultCompactionL0Trigger // explicit because we use it as base for other stuff
)
if large {
// Change the parameters for better throughput at the price of some
// RAM and larger files. This results in larger batches of writes
// and compaction at a lower frequency.
l.Infoln("Using large-database tuning")
defaultBlockCacheCapacity = 64 << MiB
defaultBlockSize = 64 << KiB
defaultCompactionTableSize = 16 << MiB
defaultCompactionTableSizeMultiplier = 20 // 2.0 after division by ten
defaultWriteBuffer = 64 << MiB
defaultCompactionL0Trigger = 8 // number of l0 files
}
opts := &opt.Options{
BlockCacheCapacity: debugEnvValue("BlockCacheCapacity", defaultBlockCacheCapacity),
BlockCacheEvictRemoved: debugEnvValue("BlockCacheEvictRemoved", 0) != 0,
BlockRestartInterval: debugEnvValue("BlockRestartInterval", 0),
BlockSize: debugEnvValue("BlockSize", defaultBlockSize),
CompactionExpandLimitFactor: debugEnvValue("CompactionExpandLimitFactor", 0),
CompactionGPOverlapsFactor: debugEnvValue("CompactionGPOverlapsFactor", 0),
CompactionL0Trigger: debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger),
CompactionSourceLimitFactor: debugEnvValue("CompactionSourceLimitFactor", 0),
CompactionTableSize: debugEnvValue("CompactionTableSize", defaultCompactionTableSize),
CompactionTableSizeMultiplier: float64(debugEnvValue("CompactionTableSizeMultiplier", defaultCompactionTableSizeMultiplier)) / 10.0,
CompactionTotalSize: debugEnvValue("CompactionTotalSize", 0),
CompactionTotalSizeMultiplier: float64(debugEnvValue("CompactionTotalSizeMultiplier", 0)) / 10.0,
DisableBufferPool: debugEnvValue("DisableBufferPool", 0) != 0,
DisableBlockCache: debugEnvValue("DisableBlockCache", 0) != 0,
DisableCompactionBackoff: debugEnvValue("DisableCompactionBackoff", 0) != 0,
DisableLargeBatchTransaction: debugEnvValue("DisableLargeBatchTransaction", 0) != 0,
NoSync: debugEnvValue("NoSync", 0) != 0,
NoWriteMerge: debugEnvValue("NoWriteMerge", 0) != 0,
OpenFilesCacheCapacity: debugEnvValue("OpenFilesCacheCapacity", dbMaxOpenFiles),
WriteBuffer: debugEnvValue("WriteBuffer", defaultWriteBuffer),
// The write slowdown and pause can be overridden, but even if they
// are not and the compaction trigger is overridden we need to
// adjust so that we don't pause writes for L0 compaction before we
// even *start* L0 compaction...
WriteL0SlowdownTrigger: debugEnvValue("WriteL0SlowdownTrigger", 2*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
WriteL0PauseTrigger: debugEnvValue("WriteL0SlowdownTrigger", 3*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
}
return opts
}
func open(location string, opts *opt.Options) (*leveldb.DB, error) {
db, err := leveldb.OpenFile(location, opts)
if leveldbIsCorrupted(err) {
db, err = leveldb.RecoverFile(location, opts)
}
if leveldbIsCorrupted(err) {
// The database is corrupted, and we've tried to recover it but it
// didn't work. At this point there isn't much to do beyond dropping
// the database and reindexing...
l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
if err := os.RemoveAll(location); err != nil {
return nil, &errorSuggestion{err, "failed to delete corrupted database"}
}
db, err = leveldb.OpenFile(location, opts)
}
if err != nil {
return nil, &errorSuggestion{err, "is another instance of Syncthing running?"}
}
if debugEnvValue("CompactEverything", 0) != 0 {
if err := db.CompactRange(util.Range{}); err != nil {
l.Warnln("Compacting database:", err)
}
}
return db, nil
}
func debugEnvValue(key string, def int) int {
v, err := strconv.ParseInt(os.Getenv("STDEBUG_"+key), 10, 63)
if err != nil {
return def
}
return int(v)
}
// A "better" version of leveldb's errors.IsCorrupted.
func leveldbIsCorrupted(err error) bool {
switch {
case err == nil:
return false
case errors.IsCorrupted(err):
return true
case strings.Contains(err.Error(), "corrupted"):
return true
}
return false
}
// dbIsLarge returns whether the estimated size of the database at location
// is large enough to warrant optimization for large databases.
func dbIsLarge(location string) bool {
if ^uint(0)>>63 == 0 {
// We're compiled for a 32 bit architecture. We've seen trouble with
// large settings there.
// (https://forum.syncthing.net/t/many-small-ldb-files-with-database-tuning/13842)
return false
}
entries, err := os.ReadDir(location)
if err != nil {
return false
}
var size int64
for _, entry := range entries {
if entry.Name() == "LOG" {
// don't count the size
continue
}
fi, err := entry.Info()
if err != nil {
continue
}
size += fi.Size()
}
return size > dbLargeThreshold
}
type errorSuggestion struct {
inner error
suggestion string
}
func (e *errorSuggestion) Error() string {
return fmt.Sprintf("%s (%s)", e.inner.Error(), e.suggestion)
}
lib/db/backend/leveldb_test.go
-13
View File
@@ -1,13 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package backend
import "testing"
func TestLevelDBBackendBehavior(t *testing.T) {
testBackendBehavior(t, OpenLevelDBMemory)
}
lib/db/benchmark_test.go
-344
View File
@@ -1,344 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db_test
import (
"fmt"
"testing"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
)
var files, oneFile, firstHalf, secondHalf, changed100, unchanged100 []protocol.FileInfo
func lazyInitBenchFiles() {
if files != nil {
return
}
files = make([]protocol.FileInfo, 0, 1000)
for i := 0; i < 1000; i++ {
files = append(files, protocol.FileInfo{
Name: fmt.Sprintf("file%d", i),
Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}},
Blocks: genBlocks(i),
})
}
middle := len(files) / 2
firstHalf = files[:middle]
secondHalf = files[middle:]
oneFile = firstHalf[middle-1 : middle]
unchanged100 := files[100:200]
changed100 := append([]protocol.FileInfo{}, unchanged100...)
for i := range changed100 {
changed100[i].Version = changed100[i].Version.Copy().Update(myID)
}
}
func getBenchFileSet(b testing.TB) (*db.Lowlevel, *db.FileSet) {
lazyInitBenchFiles()
ldb := newLowlevelMemory(b)
benchS := newFileSet(b, "test)", ldb)
replace(benchS, remoteDevice0, files)
replace(benchS, protocol.LocalDeviceID, firstHalf)
return ldb, benchS
}
func BenchmarkReplaceAll(b *testing.B) {
ldb := newLowlevelMemory(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m := newFileSet(b, "test)", ldb)
replace(m, protocol.LocalDeviceID, files)
}
b.ReportAllocs()
}
func BenchmarkUpdateOneChanged(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
changed := make([]protocol.FileInfo, 1)
changed[0] = oneFile[0]
changed[0].Version = changed[0].Version.Copy().Update(myID)
b.ResetTimer()
for i := 0; i < b.N; i++ {
if i%2 == 0 {
benchS.Update(protocol.LocalDeviceID, changed)
} else {
benchS.Update(protocol.LocalDeviceID, oneFile)
}
}
b.ReportAllocs()
}
func BenchmarkUpdate100Changed(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if i%2 == 0 {
benchS.Update(protocol.LocalDeviceID, changed100)
} else {
benchS.Update(protocol.LocalDeviceID, unchanged100)
}
}
b.ReportAllocs()
}
func setup10Remotes(benchS *db.FileSet) {
idBase := remoteDevice1.String()[1:]
first := 'J'
for i := 0; i < 10; i++ {
id, _ := protocol.DeviceIDFromString(fmt.Sprintf("%v%s", first+rune(i), idBase))
if i%2 == 0 {
benchS.Update(id, changed100)
} else {
benchS.Update(id, unchanged100)
}
}
}
func BenchmarkUpdate100Changed10Remotes(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
setup10Remotes(benchS)
b.ResetTimer()
for i := 0; i < b.N; i++ {
if i%2 == 0 {
benchS.Update(protocol.LocalDeviceID, changed100)
} else {
benchS.Update(protocol.LocalDeviceID, unchanged100)
}
}
b.ReportAllocs()
}
func BenchmarkUpdate100ChangedRemote(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if i%2 == 0 {
benchS.Update(remoteDevice0, changed100)
} else {
benchS.Update(remoteDevice0, unchanged100)
}
}
b.ReportAllocs()
}
func BenchmarkUpdate100ChangedRemote10Remotes(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if i%2 == 0 {
benchS.Update(remoteDevice0, changed100)
} else {
benchS.Update(remoteDevice0, unchanged100)
}
}
b.ReportAllocs()
}
func BenchmarkUpdateOneUnchanged(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
benchS.Update(protocol.LocalDeviceID, oneFile)
}
b.ReportAllocs()
}
func BenchmarkNeedHalf(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
snap.Release()
if count != len(secondHalf) {
b.Errorf("wrong length %d != %d", count, len(secondHalf))
}
}
b.ReportAllocs()
}
func BenchmarkNeedHalfRemote(b *testing.B) {
ldb := newLowlevelMemory(b)
defer ldb.Close()
fset := newFileSet(b, "test)", ldb)
replace(fset, remoteDevice0, firstHalf)
replace(fset, protocol.LocalDeviceID, files)
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, fset)
snap.WithNeed(remoteDevice0, func(fi protocol.FileInfo) bool {
count++
return true
})
snap.Release()
if count != len(secondHalf) {
b.Errorf("wrong length %d != %d", count, len(secondHalf))
}
}
b.ReportAllocs()
}
func BenchmarkHave(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
snap.Release()
if count != len(firstHalf) {
b.Errorf("wrong length %d != %d", count, len(firstHalf))
}
}
b.ReportAllocs()
}
func BenchmarkGlobal(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithGlobal(func(fi protocol.FileInfo) bool {
count++
return true
})
snap.Release()
if count != len(files) {
b.Errorf("wrong length %d != %d", count, len(files))
}
}
b.ReportAllocs()
}
func BenchmarkNeedHalfTruncated(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
snap.Release()
if count != len(secondHalf) {
b.Errorf("wrong length %d != %d", count, len(secondHalf))
}
}
b.ReportAllocs()
}
func BenchmarkHaveTruncated(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
snap.Release()
if count != len(firstHalf) {
b.Errorf("wrong length %d != %d", count, len(firstHalf))
}
}
b.ReportAllocs()
}
func BenchmarkGlobalTruncated(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithGlobalTruncated(func(fi protocol.FileInfo) bool {
count++
return true
})
snap.Release()
if count != len(files) {
b.Errorf("wrong length %d != %d", count, len(files))
}
}
b.ReportAllocs()
}
func BenchmarkNeedCount(b *testing.B) {
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
benchS.Update(protocol.LocalDeviceID, changed100)
b.ResetTimer()
for i := 0; i < b.N; i++ {
snap := snapshot(b, benchS)
_ = snap.NeedSize(protocol.LocalDeviceID)
snap.Release()
}
b.ReportAllocs()
}
lib/db/blockmap.go
-64
View File
@@ -1,64 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/binary"
"fmt"
"github.com/syncthing/syncthing/lib/osutil"
)
type BlockFinder struct {
db *Lowlevel
}
func NewBlockFinder(db *Lowlevel) *BlockFinder {
return &BlockFinder{
db: db,
}
}
func (f *BlockFinder) String() string {
return fmt.Sprintf("BlockFinder@%p", f)
}
// Iterate takes an iterator function which iterates over all matching blocks
// for the given hash. The iterator function has to return either true (if
// they are happy with the block) or false to continue iterating for whatever
// reason. The iterator finally returns the result, whether or not a
// satisfying block was eventually found.
func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string, string, int32) bool) bool {
t, err := f.db.newReadOnlyTransaction()
if err != nil {
return false
}
defer t.close()
var key []byte
for _, folder := range folders {
key, err = f.db.keyer.GenerateBlockMapKey(key, []byte(folder), hash, nil)
if err != nil {
return false
}
iter, err := t.NewPrefixIterator(key)
if err != nil {
return false
}
for iter.Next() && iter.Error() == nil {
file := string(f.db.keyer.NameFromBlockMapKey(iter.Key()))
index := int32(binary.BigEndian.Uint32(iter.Value()))
if iterFn(folder, osutil.NativeFilename(file), index) {
iter.Release()
return true
}
}
iter.Release()
}
return false
}
lib/db/blockmap_test.go
-260
View File
@@ -1,260 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/binary"
"testing"
"github.com/syncthing/syncthing/lib/protocol"
)
var (
f1, f2, f3 protocol.FileInfo
folders = []string{"folder1", "folder2"}
)
func init() {
blocks := genBlocks(30)
f1 = protocol.FileInfo{
Name: "f1",
Blocks: blocks[:10],
}
f2 = protocol.FileInfo{
Name: "f2",
Blocks: blocks[10:20],
}
f3 = protocol.FileInfo{
Name: "f3",
Blocks: blocks[20:],
}
}
func setup(t testing.TB) (*Lowlevel, *BlockFinder) {
t.Helper()
db := newLowlevelMemory(t)
return db, NewBlockFinder(db)
}
func dbEmpty(db *Lowlevel) bool {
iter, err := db.NewPrefixIterator([]byte{KeyTypeBlock})
if err != nil {
panic(err)
}
defer iter.Release()
return !iter.Next()
}
func addToBlockMap(db *Lowlevel, folder []byte, fs []protocol.FileInfo) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var keyBuf []byte
blockBuf := make([]byte, 4)
for _, f := range fs {
if !f.IsDirectory() && !f.IsDeleted() && !f.IsInvalid() {
name := []byte(f.Name)
for i, block := range f.Blocks {
binary.BigEndian.PutUint32(blockBuf, uint32(i))
keyBuf, err = t.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
if err != nil {
return err
}
if err := t.Put(keyBuf, blockBuf); err != nil {
return err
}
}
}
}
return t.Commit()
}
func discardFromBlockMap(db *Lowlevel, folder []byte, fs []protocol.FileInfo) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var keyBuf []byte
for _, ef := range fs {
if !ef.IsDirectory() && !ef.IsDeleted() && !ef.IsInvalid() {
name := []byte(ef.Name)
for _, block := range ef.Blocks {
keyBuf, err = t.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
if err != nil {
return err
}
if err := t.Delete(keyBuf); err != nil {
return err
}
}
}
}
return t.Commit()
}
func TestBlockMapAddUpdateWipe(t *testing.T) {
db, f := setup(t)
defer db.Close()
if !dbEmpty(db) {
t.Fatal("db not empty")
}
folder := []byte("folder1")
f3.Type = protocol.FileInfoTypeDirectory
if err := addToBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3}); err != nil {
t.Fatal(err)
}
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
if folder != "folder1" || file != "f1" || index != 0 {
t.Fatal("Mismatch")
}
return true
})
f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
if folder != "folder1" || file != "f2" || index != 0 {
t.Fatal("Mismatch")
}
return true
})
f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
t.Fatal("Unexpected block")
return true
})
if err := discardFromBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3}); err != nil {
t.Fatal(err)
}
f1.Deleted = true
f2.LocalFlags = protocol.FlagLocalMustRescan // one of the invalid markers
if err := addToBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3}); err != nil {
t.Fatal(err)
}
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
t.Fatal("Unexpected block")
return false
})
f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
t.Fatal("Unexpected block")
return false
})
f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
if folder != "folder1" || file != "f3" || index != 0 {
t.Fatal("Mismatch")
}
return true
})
if err := db.dropFolder(folder); err != nil {
t.Fatal(err)
}
if !dbEmpty(db) {
t.Fatal("db not empty")
}
// Should not add
if err := addToBlockMap(db, folder, []protocol.FileInfo{f1, f2}); err != nil {
t.Fatal(err)
}
if !dbEmpty(db) {
t.Fatal("db not empty")
}
f1.Deleted = false
f1.LocalFlags = 0
f2.Deleted = false
f2.LocalFlags = 0
f3.Deleted = false
f3.LocalFlags = 0
}
func TestBlockFinderLookup(t *testing.T) {
db, f := setup(t)
defer db.Close()
folder1 := []byte("folder1")
folder2 := []byte("folder2")
if err := addToBlockMap(db, folder1, []protocol.FileInfo{f1}); err != nil {
t.Fatal(err)
}
if err := addToBlockMap(db, folder2, []protocol.FileInfo{f1}); err != nil {
t.Fatal(err)
}
counter := 0
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
counter++
switch counter {
case 1:
if folder != "folder1" || file != "f1" || index != 0 {
t.Fatal("Mismatch")
}
case 2:
if folder != "folder2" || file != "f1" || index != 0 {
t.Fatal("Mismatch")
}
default:
t.Fatal("Unexpected block")
}
return false
})
if counter != 2 {
t.Fatal("Incorrect count", counter)
}
if err := discardFromBlockMap(db, folder1, []protocol.FileInfo{f1}); err != nil {
t.Fatal(err)
}
f1.Deleted = true
if err := addToBlockMap(db, folder1, []protocol.FileInfo{f1}); err != nil {
t.Fatal(err)
}
counter = 0
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
counter++
switch counter {
case 1:
if folder != "folder2" || file != "f1" || index != 0 {
t.Fatal("Mismatch")
}
default:
t.Fatal("Unexpected block")
}
return false
})
if counter != 1 {
t.Fatal("Incorrect count")
}
f1.Deleted = false
}
lib/db/db_test.go
-701
View File
@@ -1,701 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"bytes"
"context"
"fmt"
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
)
func genBlocks(n int) []protocol.BlockInfo {
b := make([]protocol.BlockInfo, n)
for i := range b {
h := make([]byte, 32)
for j := range h {
h[j] = byte(i + j)
}
b[i].Size = i
b[i].Hash = h
}
return b
}
const myID = 1
var (
remoteDevice0, remoteDevice1 protocol.DeviceID
invalid = "invalid"
slashPrefixed = "/notgood"
haveUpdate0to3 map[protocol.DeviceID][]protocol.FileInfo
)
func init() {
remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
haveUpdate0to3 = map[protocol.DeviceID][]protocol.FileInfo{
protocol.LocalDeviceID: {
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: slashPrefixed, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
},
remoteDevice0: {
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), RawInvalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
},
remoteDevice1: {
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), RawInvalid: true},
protocol.FileInfo{Name: invalid, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), RawInvalid: true},
},
}
}
// TestRepairSequence checks that a few hand-crafted messed-up sequence entries get fixed.
func TestRepairSequence(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
folderStr := "test"
folder := []byte(folderStr)
id := protocol.LocalDeviceID
short := protocol.LocalDeviceID.Short()
files := []protocol.FileInfo{
{Name: "fine", Blocks: genBlocks(1)},
{Name: "duplicate", Blocks: genBlocks(2)},
{Name: "missing", Blocks: genBlocks(3)},
{Name: "overwriting", Blocks: genBlocks(4)},
{Name: "inconsistent", Blocks: genBlocks(5)},
{Name: "inconsistentNotIndirected", Blocks: genBlocks(2)},
}
for i, f := range files {
files[i].Version = f.Version.Update(short)
}
trans, err := db.newReadWriteTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.close()
addFile := func(f protocol.FileInfo, seq int64) {
dk, err := trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte(f.Name))
if err != nil {
t.Fatal(err)
}
if err := trans.putFile(dk, f); err != nil {
t.Fatal(err)
}
sk, err := trans.keyer.GenerateSequenceKey(nil, folder, seq)
if err != nil {
t.Fatal(err)
}
if err := trans.Put(sk, dk); err != nil {
t.Fatal(err)
}
}
// Plain normal entry
var seq int64 = 1
files[0].Sequence = 1
addFile(files[0], seq)
// Second entry once updated with original sequence still in place
f := files[1]
f.Sequence = int64(len(files) + 1)
addFile(f, f.Sequence)
// Original sequence entry
seq++
sk, err := trans.keyer.GenerateSequenceKey(nil, folder, seq)
if err != nil {
t.Fatal(err)
}
dk, err := trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte(f.Name))
if err != nil {
t.Fatal(err)
}
if err := trans.Put(sk, dk); err != nil {
t.Fatal(err)
}
// File later overwritten thus missing sequence entry
seq++
files[2].Sequence = seq
addFile(files[2], seq)
// File overwriting previous sequence entry (no seq bump)
seq++
files[3].Sequence = seq
addFile(files[3], seq)
// Inconistent files
seq++
files[4].Sequence = 101
addFile(files[4], seq)
seq++
files[5].Sequence = 102
addFile(files[5], seq)
// And a sequence entry pointing at nothing because why not
sk, err = trans.keyer.GenerateSequenceKey(nil, folder, 100001)
if err != nil {
t.Fatal(err)
}
dk, err = trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte("nonexisting"))
if err != nil {
t.Fatal(err)
}
if err := trans.Put(sk, dk); err != nil {
t.Fatal(err)
}
if err := trans.Commit(); err != nil {
t.Fatal(err)
}
// Loading the metadata for the first time means a "re"calculation happens,
// along which the sequences get repaired too.
db.gcMut.RLock()
_, err = db.loadMetadataTracker(folderStr)
db.gcMut.RUnlock()
if err != nil {
t.Fatal(err)
}
// Check the db
ro, err := db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
defer ro.close()
it, err := ro.NewPrefixIterator([]byte{KeyTypeDevice})
if err != nil {
t.Fatal(err)
}
defer it.Release()
for it.Next() {
fi, err := ro.unmarshalTrunc(it.Value(), true)
if err != nil {
t.Fatal(err)
}
if sk, err = ro.keyer.GenerateSequenceKey(sk, folder, fi.SequenceNo()); err != nil {
t.Fatal(err)
}
dk, err := ro.Get(sk)
if backend.IsNotFound(err) {
t.Error("Missing sequence entry for", fi.FileName())
} else if err != nil {
t.Fatal(err)
}
if !bytes.Equal(it.Key(), dk) {
t.Errorf("Wrong key for %v, expected %s, got %s", f.FileName(), it.Key(), dk)
}
}
if err := it.Error(); err != nil {
t.Fatal(err)
}
it.Release()
it, err = ro.NewPrefixIterator([]byte{KeyTypeSequence})
if err != nil {
t.Fatal(err)
}
defer it.Release()
for it.Next() {
fi, ok, err := ro.getFileTrunc(it.Value(), false)
if err != nil {
t.Fatal(err)
}
seq := ro.keyer.SequenceFromSequenceKey(it.Key())
if !ok {
t.Errorf("Sequence entry %v points at nothing", seq)
} else if fi.SequenceNo() != seq {
t.Errorf("Inconsistent sequence entry for %v: %v != %v", fi.FileName(), fi.SequenceNo(), seq)
}
if len(fi.Blocks) == 0 {
t.Error("Missing blocks in", fi.FileName())
}
}
if err := it.Error(); err != nil {
t.Fatal(err)
}
it.Release()
}
func TestDowngrade(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
// sets the min version etc
if err := UpdateSchema(db); err != nil {
t.Fatal(err)
}
// Bump the database version to something newer than we actually support
miscDB := NewMiscDataNamespace(db)
if err := miscDB.PutInt64("dbVersion", dbVersion+1); err != nil {
t.Fatal(err)
}
l.Infoln(dbVersion)
// Pretend we just opened the DB and attempt to update it again
err := UpdateSchema(db)
if err, ok := err.(*databaseDowngradeError); !ok {
t.Fatal("Expected error due to database downgrade, got", err)
} else if err.minSyncthingVersion != dbMinSyncthingVersion {
t.Fatalf("Error has %v as min Syncthing version, expected %v", err.minSyncthingVersion, dbMinSyncthingVersion)
}
}
func TestCheckGlobals(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
fs := newFileSet(t, "test", db)
// Add any file
name := "foo"
fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
{
Name: name,
Type: protocol.FileInfoTypeFile,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
},
})
// Remove just the file entry
if err := db.dropPrefix([]byte{KeyTypeDevice}); err != nil {
t.Fatal(err)
}
// Clean up global entry of the now missing file
if repaired, err := db.checkGlobals(fs.folder); err != nil {
t.Fatal(err)
} else if repaired != 1 {
t.Error("Expected 1 repaired global item, got", repaired)
}
// Check that the global entry is gone
gk, err := db.keyer.GenerateGlobalVersionKey(nil, []byte(fs.folder), []byte(name))
if err != nil {
t.Fatal(err)
}
_, err = db.Get(gk)
if !backend.IsNotFound(err) {
t.Error("Expected key missing error, got", err)
}
}
func TestDropDuplicates(t *testing.T) {
names := []string{
"foo",
"bar",
"dcxvoijnds",
"3d/dsfase/4/ss2",
}
tcs := []struct{ in, out []int }{
{[]int{0}, []int{0}},
{[]int{0, 1}, []int{0, 1}},
{[]int{0, 1, 0, 1}, []int{0, 1}},
{[]int{0, 1, 1, 1, 1}, []int{0, 1}},
{[]int{0, 0, 0, 1}, []int{0, 1}},
{[]int{0, 1, 2, 3}, []int{0, 1, 2, 3}},
{[]int{3, 2, 1, 0, 0, 1, 2, 3}, []int{0, 1, 2, 3}},
{[]int{0, 1, 1, 3, 0, 1, 0, 1, 2, 3}, []int{0, 1, 2, 3}},
}
for tci, tc := range tcs {
inp := make([]protocol.FileInfo, len(tc.in))
expSeq := make(map[string]int)
for i, j := range tc.in {
inp[i] = protocol.FileInfo{Name: names[j], Sequence: int64(i)}
expSeq[names[j]] = i
}
outp := normalizeFilenamesAndDropDuplicates(inp)
if len(outp) != len(tc.out) {
t.Errorf("tc %v: Expected %v entries, got %v", tci, len(tc.out), len(outp))
continue
}
for i, f := range outp {
if exp := names[tc.out[i]]; exp != f.Name {
t.Errorf("tc %v: Got file %v at pos %v, expected %v", tci, f.Name, i, exp)
}
if exp := int64(expSeq[outp[i].Name]); exp != f.Sequence {
t.Errorf("tc %v: Got sequence %v at pos %v, expected %v", tci, f.Sequence, i, exp)
}
}
}
}
func TestGCIndirect(t *testing.T) {
// Verify that the gcIndirect run actually removes block lists.
db := newLowlevelMemory(t)
defer db.Close()
meta := newMetadataTracker(db.keyer, events.NoopLogger)
// Add three files with different block lists
files := []protocol.FileInfo{
{Name: "a", Blocks: genBlocks(100)},
{Name: "b", Blocks: genBlocks(200)},
{Name: "c", Blocks: genBlocks(300)},
}
db.updateLocalFiles([]byte("folder"), files, meta)
// Run a GC pass
db.gcIndirect(context.Background())
// Verify that we have three different block lists
n, err := numBlockLists(db)
if err != nil {
t.Fatal(err)
}
if n != len(files) {
t.Fatal("expected each file to have a block list")
}
// Change the block lists for each file
for i := range files {
files[i].Version = files[i].Version.Update(42)
files[i].Blocks = genBlocks(len(files[i].Blocks) + 1)
}
db.updateLocalFiles([]byte("folder"), files, meta)
// Verify that we now have *six* different block lists
n, err = numBlockLists(db)
if err != nil {
t.Fatal(err)
}
if n != 2*len(files) {
t.Fatal("expected both old and new block lists to exist")
}
// Run a GC pass
db.gcIndirect(context.Background())
// Verify that we now have just the three we need, again
n, err = numBlockLists(db)
if err != nil {
t.Fatal(err)
}
if n != len(files) {
t.Fatal("expected GC to collect all but the needed ones")
}
// Double check the correctness by loading the block lists and comparing with what we stored
tr, err := db.newReadOnlyTransaction()
if err != nil {
t.Fatal()
}
defer tr.Release()
for _, f := range files {
fi, ok, err := tr.getFile([]byte("folder"), protocol.LocalDeviceID[:], []byte(f.Name))
if err != nil {
t.Fatal(err)
}
if !ok {
t.Fatal("mysteriously missing")
}
if len(fi.Blocks) != len(f.Blocks) {
t.Fatal("block list mismatch")
}
for i := range fi.Blocks {
if !bytes.Equal(fi.Blocks[i].Hash, f.Blocks[i].Hash) {
t.Fatal("hash mismatch")
}
}
}
}
func TestUpdateTo14(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
folderStr := "default"
folder := []byte(folderStr)
name := []byte("foo")
file := protocol.FileInfo{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(blocksIndirectionCutoff - 1)}
file.BlocksHash = protocol.BlocksHash(file.Blocks)
fileWOBlocks := file
fileWOBlocks.Blocks = nil
meta, err := db.loadMetadataTracker(folderStr)
if err != nil {
t.Fatal(err)
}
// Initially add the correct file the usual way, all good here.
if err := db.updateLocalFiles(folder, []protocol.FileInfo{file}, meta); err != nil {
t.Fatal(err)
}
// Simulate the previous bug, where .putFile could write a file info without
// blocks, even though the file has them (and thus a non-nil BlocksHash).
trans, err := db.newReadWriteTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.close()
key, err := db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], name)
if err != nil {
t.Fatal(err)
}
fiBs := mustMarshal(fileWOBlocks.ToWire(true))
if err := trans.Put(key, fiBs); err != nil {
t.Fatal(err)
}
if err := trans.Commit(); err != nil {
t.Fatal(err)
}
trans.close()
// Run migration, pretending were still on schema 13.
if err := (&schemaUpdater{db}).updateSchemaTo14(13); err != nil {
t.Fatal(err)
}
// checks
ro, err := db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
defer ro.close()
if f, ok, err := ro.getFileByKey(key); err != nil {
t.Fatal(err)
} else if !ok {
t.Error("file missing")
} else if !f.MustRescan() {
t.Error("file not marked as MustRescan")
}
if vl, err := ro.getGlobalVersions(nil, folder, name); err != nil {
t.Fatal(err)
} else if fv, ok := vlGetGlobal(vl); !ok {
t.Error("missing global")
} else if !fvIsInvalid(fv) {
t.Error("global not marked as invalid")
}
}
func TestFlushRecursion(t *testing.T) {
// Verify that a commit hook can write to the transaction without
// causing another flush and thus recursion.
db := newLowlevelMemory(t)
defer db.Close()
// A commit hook that writes a small piece of data to the transaction.
hookFired := 0
hook := func(tx backend.WriteTransaction) error {
err := tx.Put([]byte(fmt.Sprintf("hook-key-%d", hookFired)), []byte(fmt.Sprintf("hook-value-%d", hookFired)))
if err != nil {
t.Fatal(err)
}
hookFired++
return nil
}
// A transaction.
tx, err := db.NewWriteTransaction(hook)
if err != nil {
t.Fatal(err)
}
defer tx.Release()
// Write stuff until the transaction flushes, thus firing the hook.
i := 0
for hookFired == 0 {
err := tx.Put([]byte(fmt.Sprintf("key-%d", i)), []byte(fmt.Sprintf("value-%d", i)))
if err != nil {
t.Fatal(err)
}
i++
}
// The hook should have fired precisely once.
if hookFired != 1 {
t.Error("expect one hook fire, not", hookFired)
}
}
func TestCheckLocalNeed(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
folderStr := "test"
fs := newFileSet(t, folderStr, db)
// Add files such that we are in sync for a and b, and need c and d.
files := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
}
fs.Update(protocol.LocalDeviceID, files)
files[2].Version = files[2].Version.Update(remoteDevice0.Short())
files[3].Version = files[2].Version.Update(remoteDevice0.Short())
fs.Update(remoteDevice0, files)
checkNeed := func() {
snap := snapshot(t, fs)
defer snap.Release()
c := snap.NeedSize(protocol.LocalDeviceID)
if c.Files != 2 {
t.Errorf("Expected 2 needed files locally, got %v in meta", c.Files)
}
needed := make([]protocol.FileInfo, 0, 2)
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
needed = append(needed, fi)
return true
})
if l := len(needed); l != 2 {
t.Errorf("Expected 2 needed files locally, got %v in db", l)
} else if needed[0].Name != "c" || needed[1].Name != "d" {
t.Errorf("Expected files c and d to be needed, got %v and %v", needed[0].Name, needed[1].Name)
}
}
checkNeed()
trans, err := db.newReadWriteTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.close()
// Add "b" to needed and remove "d"
folder := []byte(folderStr)
key, err := trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[1].Name))
if err != nil {
t.Fatal(err)
}
if err = trans.Put(key, nil); err != nil {
t.Fatal(err)
}
key, err = trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[3].Name))
if err != nil {
t.Fatal(err)
}
if err = trans.Delete(key); err != nil {
t.Fatal(err)
}
if err := trans.Commit(); err != nil {
t.Fatal(err)
}
if repaired, err := db.checkLocalNeed(folder); err != nil {
t.Fatal(err)
} else if repaired != 2 {
t.Error("Expected 2 repaired local need items, got", repaired)
}
checkNeed()
}
func TestDuplicateNeedCount(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
folder := "test"
fs := newFileSet(t, folder, db)
files := []protocol.FileInfo{{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}}
fs.Update(protocol.LocalDeviceID, files)
files[0].Version = files[0].Version.Update(remoteDevice0.Short())
fs.Update(remoteDevice0, files)
db.checkRepair()
fs = newFileSet(t, folder, db)
found := false
for _, c := range fs.meta.counts.Counts {
if protocol.LocalDeviceID == c.DeviceID && c.LocalFlags == needFlag {
if found {
t.Fatal("second need count for local device encountered")
}
found = true
}
}
if !found {
t.Fatal("no need count for local device encountered")
}
}
func TestNeedAfterDropGlobal(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
folder := "test"
fs := newFileSet(t, folder, db)
// Initial:
// Three devices and a file "test": local has Version 1, remoteDevice0
// Version 2 and remoteDevice2 doesn't have it.
// All of them have "bar", just so the db knows about remoteDevice2.
files := []protocol.FileInfo{
{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1},
{Name: "bar", Version: protocol.Vector{}.Update(myID), Sequence: 2},
}
fs.Update(protocol.LocalDeviceID, files)
files[0].Version = files[0].Version.Update(myID)
fs.Update(remoteDevice0, files)
fs.Update(remoteDevice1, files[1:])
// remoteDevice1 needs one file: test
snap := snapshot(t, fs)
c := snap.NeedSize(remoteDevice1)
if c.Files != 1 {
t.Errorf("Expected 1 needed files initially, got %v", c.Files)
}
snap.Release()
// Drop remoteDevice0, i.e. remove all their files from db.
// That changes the global file, which is now what local has.
fs.Drop(remoteDevice0)
// remoteDevice1 still needs test.
snap = snapshot(t, fs)
c = snap.NeedSize(remoteDevice1)
if c.Files != 1 {
t.Errorf("Expected still 1 needed files, got %v", c.Files)
}
snap.Release()
}
func numBlockLists(db *Lowlevel) (int, error) {
it, err := db.Backend.NewPrefixIterator([]byte{KeyTypeBlockList})
if err != nil {
return 0, err
}
defer it.Release()
n := 0
for it.Next() {
n++
}
if err := it.Error(); err != nil {
return 0, err
}
return n, nil
}
lib/db/debug.go
-17
View File
@@ -1,17 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"github.com/syncthing/syncthing/lib/logger"
)
var l = logger.DefaultLogger.NewFacility("db", "The database layer")
func shouldDebug() bool {
return l.ShouldDebug("db")
}
lib/db/keyer.go
-409
View File
@@ -1,409 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/binary"
)
const (
keyPrefixLen = 1
keyFolderLen = 4 // indexed
keyDeviceLen = 4 // indexed
keySequenceLen = 8
keyHashLen = 32
maxInt64 int64 = 1<<63 - 1
)
const (
// KeyTypeDevice <int32 folder ID> <int32 device ID> <file name> = FileInfo
KeyTypeDevice byte = 0
// KeyTypeGlobal <int32 folder ID> <file name> = VersionList
KeyTypeGlobal byte = 1
// KeyTypeBlock <int32 folder ID> <32 bytes hash> <§file name> = int32 (block index)
KeyTypeBlock byte = 2
// KeyTypeDeviceStatistic <device ID as string> <some string> = some value
KeyTypeDeviceStatistic byte = 3
// KeyTypeFolderStatistic <folder ID as string> <some string> = some value
KeyTypeFolderStatistic byte = 4
// KeyTypeVirtualMtime <int32 folder ID> <file name> = mtimeMapping
KeyTypeVirtualMtime byte = 5
// KeyTypeFolderIdx <int32 id> = string value
KeyTypeFolderIdx byte = 6
// KeyTypeDeviceIdx <int32 id> = string value
KeyTypeDeviceIdx byte = 7
// KeyTypeIndexID <int32 device ID> <int32 folder ID> = protocol.IndexID
KeyTypeIndexID byte = 8
// KeyTypeFolderMeta <int32 folder ID> = CountsSet
KeyTypeFolderMeta byte = 9
// KeyTypeMiscData <some string> = some value
KeyTypeMiscData byte = 10
// KeyTypeSequence <int32 folder ID> <int64 sequence number> = KeyTypeDevice key
KeyTypeSequence byte = 11
// KeyTypeNeed <int32 folder ID> <file name> = <nothing>
KeyTypeNeed byte = 12
// KeyTypeBlockList <block list hash> = BlockList
KeyTypeBlockList byte = 13
// KeyTypeBlockListMap <int32 folder ID> <block list hash> <file name> = <nothing>
KeyTypeBlockListMap byte = 14
// KeyTypeVersion <version hash> = Vector
KeyTypeVersion byte = 15
// KeyTypePendingFolder <int32 device ID> <folder ID as string> = ObservedFolder
KeyTypePendingFolder byte = 16
// KeyTypePendingDevice <device ID in wire format> = ObservedDevice
KeyTypePendingDevice byte = 17
)
type keyer interface {
// device file key stuff
GenerateDeviceFileKey(key, folder, device, name []byte) (deviceFileKey, error)
NameFromDeviceFileKey(key []byte) []byte
DeviceFromDeviceFileKey(key []byte) ([]byte, bool)
FolderFromDeviceFileKey(key []byte) ([]byte, bool)
// global version key stuff
GenerateGlobalVersionKey(key, folder, name []byte) (globalVersionKey, error)
NameFromGlobalVersionKey(key []byte) []byte
// block map key stuff (former BlockMap)
GenerateBlockMapKey(key, folder, hash, name []byte) (blockMapKey, error)
NameFromBlockMapKey(key []byte) []byte
GenerateBlockListMapKey(key, folder, hash, name []byte) (blockListMapKey, error)
NameFromBlockListMapKey(key []byte) []byte
// file need index
GenerateNeedFileKey(key, folder, name []byte) (needFileKey, error)
// file sequence index
GenerateSequenceKey(key, folder []byte, seq int64) (sequenceKey, error)
SequenceFromSequenceKey(key []byte) int64
// index IDs
GenerateIndexIDKey(key, device, folder []byte) (indexIDKey, error)
FolderFromIndexIDKey(key []byte) ([]byte, bool)
DeviceFromIndexIDKey(key []byte) ([]byte, bool)
// Mtimes
GenerateMtimesKey(key, folder []byte) (mtimesKey, error)
// Folder metadata
GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error)
// Block lists
GenerateBlockListKey(key []byte, hash []byte) blockListKey
// Version vectors
GenerateVersionKey(key []byte, hash []byte) versionKey
// Pending (unshared) folders and devices
GeneratePendingFolderKey(key, device, folder []byte) (pendingFolderKey, error)
FolderFromPendingFolderKey(key []byte) []byte
DeviceFromPendingFolderKey(key []byte) ([]byte, bool)
GeneratePendingDeviceKey(key, device []byte) pendingDeviceKey
DeviceFromPendingDeviceKey(key []byte) []byte
}
// defaultKeyer implements our key scheme. It needs folder and device
// indexes.
type defaultKeyer struct {
folderIdx *smallIndex
deviceIdx *smallIndex
}
func newDefaultKeyer(folderIdx, deviceIdx *smallIndex) defaultKeyer {
return defaultKeyer{
folderIdx: folderIdx,
deviceIdx: deviceIdx,
}
}
type deviceFileKey []byte
func (k deviceFileKey) WithoutNameAndDevice() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
func (k deviceFileKey) WithoutName() []byte {
return k[:keyPrefixLen+keyFolderLen+keyDeviceLen]
}
func (k defaultKeyer) GenerateDeviceFileKey(key, folder, device, name []byte) (deviceFileKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
deviceID, err := k.deviceIdx.ID(device)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen+keyDeviceLen+len(name))
key[0] = KeyTypeDevice
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
binary.BigEndian.PutUint32(key[keyPrefixLen+keyFolderLen:], deviceID)
copy(key[keyPrefixLen+keyFolderLen+keyDeviceLen:], name)
return key, nil
}
func (defaultKeyer) NameFromDeviceFileKey(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
}
func (k defaultKeyer) DeviceFromDeviceFileKey(key []byte) ([]byte, bool) {
return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
}
func (k defaultKeyer) FolderFromDeviceFileKey(key []byte) ([]byte, bool) {
return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
}
type globalVersionKey []byte
func (k globalVersionKey) WithoutName() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
func (k defaultKeyer) GenerateGlobalVersionKey(key, folder, name []byte) (globalVersionKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen+len(name))
key[0] = KeyTypeGlobal
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
copy(key[keyPrefixLen+keyFolderLen:], name)
return key, nil
}
func (defaultKeyer) NameFromGlobalVersionKey(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen:]
}
type blockMapKey []byte
func (k defaultKeyer) GenerateBlockMapKey(key, folder, hash, name []byte) (blockMapKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen+keyHashLen+len(name))
key[0] = KeyTypeBlock
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
copy(key[keyPrefixLen+keyFolderLen:], hash)
copy(key[keyPrefixLen+keyFolderLen+keyHashLen:], name)
return key, nil
}
func (defaultKeyer) NameFromBlockMapKey(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen+keyHashLen:]
}
func (k blockMapKey) WithoutHashAndName() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
type blockListMapKey []byte
func (k defaultKeyer) GenerateBlockListMapKey(key, folder, hash, name []byte) (blockListMapKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen+keyHashLen+len(name))
key[0] = KeyTypeBlockListMap
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
copy(key[keyPrefixLen+keyFolderLen:], hash)
copy(key[keyPrefixLen+keyFolderLen+keyHashLen:], name)
return key, nil
}
func (defaultKeyer) NameFromBlockListMapKey(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen+keyHashLen:]
}
func (k blockListMapKey) WithoutHashAndName() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
type needFileKey []byte
func (k needFileKey) WithoutName() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
func (k defaultKeyer) GenerateNeedFileKey(key, folder, name []byte) (needFileKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen+len(name))
key[0] = KeyTypeNeed
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
copy(key[keyPrefixLen+keyFolderLen:], name)
return key, nil
}
type sequenceKey []byte
func (k sequenceKey) WithoutSequence() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
func (k defaultKeyer) GenerateSequenceKey(key, folder []byte, seq int64) (sequenceKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen+keySequenceLen)
key[0] = KeyTypeSequence
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
binary.BigEndian.PutUint64(key[keyPrefixLen+keyFolderLen:], uint64(seq))
return key, nil
}
func (defaultKeyer) SequenceFromSequenceKey(key []byte) int64 {
return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
}
type indexIDKey []byte
func (k defaultKeyer) GenerateIndexIDKey(key, device, folder []byte) (indexIDKey, error) {
deviceID, err := k.deviceIdx.ID(device)
if err != nil {
return nil, err
}
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyDeviceLen+keyFolderLen)
key[0] = KeyTypeIndexID
binary.BigEndian.PutUint32(key[keyPrefixLen:], deviceID)
binary.BigEndian.PutUint32(key[keyPrefixLen+keyDeviceLen:], folderID)
return key, nil
}
func (k defaultKeyer) FolderFromIndexIDKey(key []byte) ([]byte, bool) {
return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyDeviceLen:]))
}
func (k defaultKeyer) DeviceFromIndexIDKey(key []byte) ([]byte, bool) {
return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen : keyPrefixLen+keyDeviceLen]))
}
type mtimesKey []byte
func (k defaultKeyer) GenerateMtimesKey(key, folder []byte) (mtimesKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen)
key[0] = KeyTypeVirtualMtime
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
return key, nil
}
type folderMetaKey []byte
func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error) {
folderID, err := k.folderIdx.ID(folder)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyFolderLen)
key[0] = KeyTypeFolderMeta
binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
return key, nil
}
type blockListKey []byte
func (defaultKeyer) GenerateBlockListKey(key []byte, hash []byte) blockListKey {
key = resize(key, keyPrefixLen+len(hash))
key[0] = KeyTypeBlockList
copy(key[keyPrefixLen:], hash)
return key
}
func (k blockListKey) Hash() []byte {
return k[keyPrefixLen:]
}
type versionKey []byte
func (defaultKeyer) GenerateVersionKey(key []byte, hash []byte) versionKey {
key = resize(key, keyPrefixLen+len(hash))
key[0] = KeyTypeVersion
copy(key[keyPrefixLen:], hash)
return key
}
func (k versionKey) Hash() []byte {
return k[keyPrefixLen:]
}
type pendingFolderKey []byte
func (k defaultKeyer) GeneratePendingFolderKey(key, device, folder []byte) (pendingFolderKey, error) {
deviceID, err := k.deviceIdx.ID(device)
if err != nil {
return nil, err
}
key = resize(key, keyPrefixLen+keyDeviceLen+len(folder))
key[0] = KeyTypePendingFolder
binary.BigEndian.PutUint32(key[keyPrefixLen:], deviceID)
copy(key[keyPrefixLen+keyDeviceLen:], folder)
return key, nil
}
func (defaultKeyer) FolderFromPendingFolderKey(key []byte) []byte {
return key[keyPrefixLen+keyDeviceLen:]
}
func (k defaultKeyer) DeviceFromPendingFolderKey(key []byte) ([]byte, bool) {
return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
}
type pendingDeviceKey []byte
func (defaultKeyer) GeneratePendingDeviceKey(key, device []byte) pendingDeviceKey {
key = resize(key, keyPrefixLen+len(device))
key[0] = KeyTypePendingDevice
copy(key[keyPrefixLen:], device)
return key
}
func (defaultKeyer) DeviceFromPendingDeviceKey(key []byte) []byte {
return key[keyPrefixLen:]
}
// resize returns a byte slice of the specified size, reusing bs if possible
func resize(bs []byte, size int) []byte {
if cap(bs) < size {
return make([]byte, size)
}
return bs[:size]
}
lib/db/keyer_test.go
-80
View File
@@ -1,80 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"bytes"
"testing"
)
func TestDeviceKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
dev := []byte("device67890123456789012345678901")
name := []byte("name")
db := newLowlevelMemory(t)
defer db.Close()
key, err := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
if err != nil {
t.Fatal(err)
}
fld2, ok := db.keyer.FolderFromDeviceFileKey(key)
if !ok {
t.Fatal("unexpectedly not found")
}
if !bytes.Equal(fld2, fld) {
t.Errorf("wrong folder %q != %q", fld2, fld)
}
dev2, ok := db.keyer.DeviceFromDeviceFileKey(key)
if !ok {
t.Fatal("unexpectedly not found")
}
if !bytes.Equal(dev2, dev) {
t.Errorf("wrong device %q != %q", dev2, dev)
}
name2 := db.keyer.NameFromDeviceFileKey(key)
if !bytes.Equal(name2, name) {
t.Errorf("wrong name %q != %q", name2, name)
}
}
func TestGlobalKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
name := []byte("name")
db := newLowlevelMemory(t)
defer db.Close()
key, err := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
if err != nil {
t.Fatal(err)
}
name2 := db.keyer.NameFromGlobalVersionKey(key)
if !bytes.Equal(name2, name) {
t.Errorf("wrong name %q != %q", name2, name)
}
}
func TestSequenceKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
db := newLowlevelMemory(t)
defer db.Close()
const seq = 1234567890
key, err := db.keyer.GenerateSequenceKey(nil, fld, seq)
if err != nil {
t.Fatal(err)
}
outSeq := db.keyer.SequenceFromSequenceKey(key)
if outSeq != seq {
t.Errorf("sequence number mangled, %d != %d", outSeq, seq)
}
}
lib/db/lowlevel.go
-1453
View File
File diff suppressed because it is too large Load Diff
lib/db/meta.go
-472
View File
@@ -1,472 +0,0 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"errors"
"fmt"
"math/bits"
"time"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
var errMetaInconsistent = errors.New("inconsistent counts detected")
type countsMap struct {
counts CountsSet
indexes map[metaKey]int // device ID + local flags -> index in counts
}
// metadataTracker keeps metadata on a per device, per local flag basis.
type metadataTracker struct {
keyer keyer
countsMap
mut sync.RWMutex
dirty bool
evLogger events.Logger
}
type metaKey struct {
dev protocol.DeviceID
flag uint32
}
const needFlag uint32 = 1 << 31 // Last bit, as early ones are local flags
func newMetadataTracker(keyer keyer, evLogger events.Logger) *metadataTracker {
return &metadataTracker{
keyer: keyer,
mut: sync.NewRWMutex(),
countsMap: countsMap{
indexes: make(map[metaKey]int),
},
evLogger: evLogger,
}
}
// Unmarshal loads a metadataTracker from the corresponding protobuf
// representation
func (m *metadataTracker) Unmarshal(bs []byte) error {
var dbc dbproto.CountsSet
if err := proto.Unmarshal(bs, &dbc); err != nil {
return err
}
m.counts.Created = dbc.Created
m.counts.Counts = make([]Counts, len(dbc.Counts))
for i, c := range dbc.Counts {
m.counts.Counts[i] = countsFromWire(c)
}
// Initialize the index map
m.indexes = make(map[metaKey]int)
for i, c := range m.counts.Counts {
m.indexes[metaKey{c.DeviceID, c.LocalFlags}] = i
}
return nil
}
// protoMarshal returns the protobuf representation of the metadataTracker.
// Must be called with the read lock held.
func (m *metadataTracker) protoMarshal() ([]byte, error) {
dbc := &dbproto.CountsSet{
Counts: make([]*dbproto.Counts, len(m.counts.Counts)),
Created: m.counts.Created,
}
for i, c := range m.counts.Counts {
dbc.Counts[i] = c.toWire()
}
return proto.Marshal(dbc)
}
func (m *metadataTracker) CommitHook(folder []byte) backend.CommitHook {
return func(t backend.WriteTransaction) error {
return m.toDB(t, folder)
}
}
// toDB saves the marshalled metadataTracker to the given db, under the key
// corresponding to the given folder
func (m *metadataTracker) toDB(t backend.WriteTransaction, folder []byte) error {
key, err := m.keyer.GenerateFolderMetaKey(nil, folder)
if err != nil {
return err
}
m.mut.RLock()
defer m.mut.RUnlock()
if !m.dirty {
return nil
}
bs, err := m.protoMarshal()
if err != nil {
return err
}
err = t.Put(key, bs)
if err == nil {
m.dirty = false
}
return err
}
// fromDB initializes the metadataTracker from the marshalled data found in
// the database under the key corresponding to the given folder
func (m *metadataTracker) fromDB(db *Lowlevel, folder []byte) error {
key, err := db.keyer.GenerateFolderMetaKey(nil, folder)
if err != nil {
return err
}
bs, err := db.Get(key)
if err != nil {
return err
}
if err = m.Unmarshal(bs); err != nil {
return err
}
if m.counts.Created == 0 {
return errMetaInconsistent
}
return nil
}
// countsPtr returns a pointer to the corresponding Counts struct, if
// necessary allocating one in the process
func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flag uint32) *Counts {
// must be called with the mutex held
if bits.OnesCount32(flag) > 1 {
panic("incorrect usage: set at most one bit in flag")
}
key := metaKey{dev, flag}
idx, ok := m.indexes[key]
if !ok {
idx = len(m.counts.Counts)
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev, LocalFlags: flag})
m.indexes[key] = idx
// Need bucket must be initialized when a device first occurs in
// the metadatatracker, even if there's no change to the need
// bucket itself.
nkey := metaKey{dev, needFlag}
if _, ok := m.indexes[nkey]; !ok {
// Initially a new device needs everything, except deletes
nidx := len(m.counts.Counts)
m.counts.Counts = append(m.counts.Counts, m.allNeededCounts(dev))
m.indexes[nkey] = nidx
}
}
return &m.counts.Counts[idx]
}
// allNeeded makes sure there is a counts in case the device needs everything.
func (m *countsMap) allNeededCounts(dev protocol.DeviceID) Counts {
var counts Counts
if idx, ok := m.indexes[metaKey{protocol.GlobalDeviceID, 0}]; ok {
counts = m.counts.Counts[idx]
counts.Deleted = 0 // Don't need deletes if having nothing
}
counts.DeviceID = dev
counts.LocalFlags = needFlag
return counts
}
// addFile adds a file to the counts, adjusting the sequence number as
// appropriate
func (m *metadataTracker) addFile(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
m.updateSeqLocked(dev, f)
m.updateFileLocked(dev, f, m.addFileLocked)
}
func (m *metadataTracker) updateFileLocked(dev protocol.DeviceID, f protocol.FileInfo, fn func(protocol.DeviceID, uint32, protocol.FileInfo)) {
m.dirty = true
if f.IsInvalid() && (f.FileLocalFlags() == 0 || dev == protocol.GlobalDeviceID) {
// This is a remote invalid file or concern the global state.
// In either case invalid files are not accounted.
return
}
if flags := f.FileLocalFlags(); flags == 0 {
// Account regular files in the zero-flags bucket.
fn(dev, 0, f)
} else {
// Account in flag specific buckets.
eachFlagBit(flags, func(flag uint32) {
fn(dev, flag, f)
})
}
}
// emptyNeeded ensures that there is a need count for the given device and that it is empty.
func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) {
m.mut.Lock()
defer m.mut.Unlock()
m.dirty = true
empty := Counts{
DeviceID: dev,
LocalFlags: needFlag,
}
key := metaKey{dev, needFlag}
if idx, ok := m.indexes[key]; ok {
m.counts.Counts[idx] = empty
return
}
m.indexes[key] = len(m.counts.Counts)
m.counts.Counts = append(m.counts.Counts, empty)
}
// addNeeded adds a file to the needed counts
func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
m.dirty = true
m.addFileLocked(dev, needFlag, f)
}
func (m *metadataTracker) Sequence(dev protocol.DeviceID) int64 {
m.mut.Lock()
defer m.mut.Unlock()
return m.countsPtr(dev, 0).Sequence
}
func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f protocol.FileInfo) {
if dev == protocol.GlobalDeviceID {
return
}
if cp := m.countsPtr(dev, 0); f.SequenceNo() > cp.Sequence {
cp.Sequence = f.SequenceNo()
}
}
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileInfo) {
cp := m.countsPtr(dev, flag)
switch {
case f.IsDeleted():
cp.Deleted++
case f.IsDirectory() && !f.IsSymlink():
cp.Directories++
case f.IsSymlink():
cp.Symlinks++
default:
cp.Files++
}
cp.Bytes += f.FileSize()
}
// removeFile removes a file from the counts
func (m *metadataTracker) removeFile(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
m.updateFileLocked(dev, f, m.removeFileLocked)
}
// removeNeeded removes a file from the needed counts
func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
m.dirty = true
m.removeFileLocked(dev, needFlag, f)
}
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileInfo) {
cp := m.countsPtr(dev, flag)
switch {
case f.IsDeleted():
cp.Deleted--
case f.IsDirectory() && !f.IsSymlink():
cp.Directories--
case f.IsSymlink():
cp.Symlinks--
default:
cp.Files--
}
cp.Bytes -= f.FileSize()
// If we've run into an impossible situation, correct it for now and set
// the created timestamp to zero. Next time we start up the metadata
// will be seen as infinitely old and recalculated from scratch.
if cp.Deleted < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta deleted count for flag 0x%x dropped below zero", flag))
cp.Deleted = 0
m.counts.Created = 0
}
if cp.Files < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta files count for flag 0x%x dropped below zero", flag))
cp.Files = 0
m.counts.Created = 0
}
if cp.Directories < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta directories count for flag 0x%x dropped below zero", flag))
cp.Directories = 0
m.counts.Created = 0
}
if cp.Symlinks < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta deleted count for flag 0x%x dropped below zero", flag))
cp.Symlinks = 0
m.counts.Created = 0
}
}
// resetAll resets all metadata for the given device
func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
m.mut.Lock()
m.dirty = true
for i, c := range m.counts.Counts {
if c.DeviceID == dev {
if c.LocalFlags != needFlag {
m.counts.Counts[i] = Counts{
DeviceID: c.DeviceID,
LocalFlags: c.LocalFlags,
}
} else {
m.counts.Counts[i] = m.allNeededCounts(dev)
}
}
}
m.mut.Unlock()
}
// resetCounts resets the file, dir, etc. counters, while retaining the
// sequence number
func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
m.mut.Lock()
m.dirty = true
for i, c := range m.counts.Counts {
if c.DeviceID == dev {
m.counts.Counts[i] = Counts{
DeviceID: c.DeviceID,
Sequence: c.Sequence,
LocalFlags: c.LocalFlags,
}
}
}
m.mut.Unlock()
}
func (m *countsMap) Counts(dev protocol.DeviceID, flag uint32) Counts {
if bits.OnesCount32(flag) > 1 {
panic("incorrect usage: set at most one bit in flag")
}
idx, ok := m.indexes[metaKey{dev, flag}]
if !ok {
if flag == needFlag {
// If there's nothing about a device in the index yet,
// it needs everything.
return m.allNeededCounts(dev)
}
return Counts{}
}
return m.counts.Counts[idx]
}
// Snapshot returns a copy of the metadata for reading.
func (m *metadataTracker) Snapshot() *countsMap {
m.mut.RLock()
defer m.mut.RUnlock()
c := &countsMap{
counts: CountsSet{
Counts: make([]Counts, len(m.counts.Counts)),
Created: m.counts.Created,
},
indexes: make(map[metaKey]int, len(m.indexes)),
}
for k, v := range m.indexes {
c.indexes[k] = v
}
copy(c.counts.Counts, m.counts.Counts)
return c
}
// nextLocalSeq allocates a new local sequence number
func (m *metadataTracker) nextLocalSeq() int64 {
m.mut.Lock()
defer m.mut.Unlock()
c := m.countsPtr(protocol.LocalDeviceID, 0)
c.Sequence++
return c.Sequence
}
// devices returns the list of devices tracked, excluding the local device
// (which we don't know the ID of)
func (m *metadataTracker) devices() []protocol.DeviceID {
m.mut.RLock()
defer m.mut.RUnlock()
return m.countsMap.devices()
}
func (m *countsMap) devices() []protocol.DeviceID {
devs := make([]protocol.DeviceID, 0, len(m.counts.Counts))
for _, dev := range m.counts.Counts {
if dev.Sequence > 0 {
if dev.DeviceID == protocol.GlobalDeviceID || dev.DeviceID == protocol.LocalDeviceID {
continue
}
devs = append(devs, dev.DeviceID)
}
}
return devs
}
func (m *metadataTracker) Created() time.Time {
m.mut.RLock()
defer m.mut.RUnlock()
return time.Unix(0, m.counts.Created)
}
func (m *metadataTracker) SetCreated() {
m.mut.Lock()
m.counts.Created = time.Now().UnixNano()
m.dirty = true
m.mut.Unlock()
}
// eachFlagBit calls the function once for every bit that is set in flags
func eachFlagBit(flags uint32, fn func(flag uint32)) {
// Test each bit from the right, as long as there are bits left in the
// flag set. Clear any bits found and stop testing as soon as there are
// no more bits set.
currentBit := uint32(1 << 0)
for flags != 0 {
if flags&currentBit != 0 {
fn(currentBit)
flags &^= currentBit
}
currentBit <<= 1
}
}
lib/db/meta_test.go
-182
View File
@@ -1,182 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"math/bits"
"sort"
"testing"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestEachFlagBit(t *testing.T) {
cases := []struct {
flags uint32
iterations int
}{
{0, 0},
{1<<0 | 1<<3, 2},
{1 << 0, 1},
{1 << 31, 1},
{1<<10 | 1<<20 | 1<<30, 3},
}
for _, tc := range cases {
var flags uint32
iterations := 0
eachFlagBit(tc.flags, func(f uint32) {
iterations++
flags |= f
if bits.OnesCount32(f) != 1 {
t.Error("expected exactly one bit to be set in every call")
}
})
if flags != tc.flags {
t.Errorf("expected 0x%x flags, got 0x%x", tc.flags, flags)
}
if iterations != tc.iterations {
t.Errorf("expected %d iterations, got %d", tc.iterations, iterations)
}
}
}
func TestMetaDevices(t *testing.T) {
d1 := protocol.DeviceID{1}
d2 := protocol.DeviceID{2}
meta := newMetadataTracker(nil, events.NoopLogger)
meta.addFile(d1, protocol.FileInfo{Sequence: 1})
meta.addFile(d1, protocol.FileInfo{Sequence: 2, LocalFlags: 1})
meta.addFile(d2, protocol.FileInfo{Sequence: 1})
meta.addFile(d2, protocol.FileInfo{Sequence: 2, LocalFlags: 2})
meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 1})
// There are five device/flags combos
if l := len(meta.counts.Counts); l < 5 {
t.Error("expected at least five buckets, not", l)
}
// There are only two non-local devices
devs := meta.devices()
if l := len(devs); l != 2 {
t.Fatal("expected two devices, not", l)
}
// Check that we got the two devices we expect
sort.Slice(devs, func(a, b int) bool {
return devs[a].Compare(devs[b]) == -1
})
if devs[0] != d1 {
t.Error("first device should be d1")
}
if devs[1] != d2 {
t.Error("second device should be d2")
}
}
func TestMetaSequences(t *testing.T) {
d1 := protocol.DeviceID{1}
meta := newMetadataTracker(nil, events.NoopLogger)
meta.addFile(d1, protocol.FileInfo{Sequence: 1})
meta.addFile(d1, protocol.FileInfo{Sequence: 2, RawInvalid: true})
meta.addFile(d1, protocol.FileInfo{Sequence: 3})
meta.addFile(d1, protocol.FileInfo{Sequence: 4, RawInvalid: true})
meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 1})
meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 2})
meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 3, LocalFlags: 1})
meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 4, LocalFlags: 2})
if seq := meta.Sequence(d1); seq != 4 {
t.Error("sequence of first device should be 4, not", seq)
}
if seq := meta.Sequence(protocol.LocalDeviceID); seq != 4 {
t.Error("sequence of first device should be 4, not", seq)
}
}
func TestRecalcMeta(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
// Add some files
s1 := newFileSet(t, "test", ldb)
files := []protocol.FileInfo{
{Name: "a", Size: 1000},
{Name: "b", Size: 2000},
}
s1.Update(protocol.LocalDeviceID, files)
// Verify local/global size
snap := snapshot(t, s1)
ls := snap.LocalSize()
gs := snap.GlobalSize()
snap.Release()
if ls.Bytes != 3000 {
t.Fatalf("Wrong initial local byte count, %d != 3000", ls.Bytes)
}
if gs.Bytes != 3000 {
t.Fatalf("Wrong initial global byte count, %d != 3000", gs.Bytes)
}
// Reach into the database to make the metadata tracker intentionally
// wrong and out of date
curSeq := s1.meta.Sequence(protocol.LocalDeviceID)
tran, err := ldb.newReadWriteTransaction()
if err != nil {
t.Fatal(err)
}
s1.meta.mut.Lock()
s1.meta.countsPtr(protocol.LocalDeviceID, 0).Sequence = curSeq - 1 // too low
s1.meta.countsPtr(protocol.LocalDeviceID, 0).Bytes = 1234 // wrong
s1.meta.countsPtr(protocol.GlobalDeviceID, 0).Bytes = 1234 // wrong
s1.meta.dirty = true
s1.meta.mut.Unlock()
if err := s1.meta.toDB(tran, []byte("test")); err != nil {
t.Fatal(err)
}
if err := tran.Commit(); err != nil {
t.Fatal(err)
}
// Verify that our bad data "took"
snap = snapshot(t, s1)
ls = snap.LocalSize()
gs = snap.GlobalSize()
snap.Release()
if ls.Bytes != 1234 {
t.Fatalf("Wrong changed local byte count, %d != 1234", ls.Bytes)
}
if gs.Bytes != 1234 {
t.Fatalf("Wrong changed global byte count, %d != 1234", gs.Bytes)
}
// Create a new fileset, which will realize the inconsistency and recalculate
s2 := newFileSet(t, "test", ldb)
// Verify local/global size
snap = snapshot(t, s2)
ls = snap.LocalSize()
gs = snap.GlobalSize()
snap.Release()
if ls.Bytes != 3000 {
t.Fatalf("Wrong fixed local byte count, %d != 3000", ls.Bytes)
}
if gs.Bytes != 3000 {
t.Fatalf("Wrong fixed global byte count, %d != 3000", gs.Bytes)
}
}
func TestMetaKeyCollisions(t *testing.T) {
if protocol.LocalAllFlags&needFlag != 0 {
t.Error("Collision between need flag and protocol local file flags")
}
}
lib/db/namespaced.go
-156
View File
@@ -1,156 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/binary"
"time"
"github.com/syncthing/syncthing/lib/db/backend"
)
// NamespacedKV is a simple key-value store using a specific namespace within
// a leveldb.
type NamespacedKV struct {
db backend.Backend
prefix string
}
// NewNamespacedKV returns a new NamespacedKV that lives in the namespace
// specified by the prefix.
func NewNamespacedKV(db backend.Backend, prefix string) *NamespacedKV {
return &NamespacedKV{
db: db,
prefix: prefix,
}
}
// PutInt64 stores a new int64. Any existing value (even if of another type)
// is overwritten.
func (n *NamespacedKV) PutInt64(key string, val int64) error {
var valBs [8]byte
binary.BigEndian.PutUint64(valBs[:], uint64(val))
return n.db.Put(n.prefixedKey(key), valBs[:])
}
// Int64 returns the stored value interpreted as an int64 and a boolean that
// is false if no value was stored at the key.
func (n *NamespacedKV) Int64(key string) (int64, bool, error) {
valBs, err := n.db.Get(n.prefixedKey(key))
if err != nil {
return 0, false, filterNotFound(err)
}
val := binary.BigEndian.Uint64(valBs)
return int64(val), true, nil
}
// PutTime stores a new time.Time. Any existing value (even if of another
// type) is overwritten.
func (n *NamespacedKV) PutTime(key string, val time.Time) error {
valBs, _ := val.MarshalBinary() // never returns an error
return n.db.Put(n.prefixedKey(key), valBs)
}
// Time returns the stored value interpreted as a time.Time and a boolean
// that is false if no value was stored at the key.
func (n NamespacedKV) Time(key string) (time.Time, bool, error) {
var t time.Time
valBs, err := n.db.Get(n.prefixedKey(key))
if err != nil {
return t, false, filterNotFound(err)
}
err = t.UnmarshalBinary(valBs)
return t, err == nil, err
}
// PutString stores a new string. Any existing value (even if of another type)
// is overwritten.
func (n *NamespacedKV) PutString(key, val string) error {
return n.db.Put(n.prefixedKey(key), []byte(val))
}
// String returns the stored value interpreted as a string and a boolean that
// is false if no value was stored at the key.
func (n NamespacedKV) String(key string) (string, bool, error) {
valBs, err := n.db.Get(n.prefixedKey(key))
if err != nil {
return "", false, filterNotFound(err)
}
return string(valBs), true, nil
}
// PutBytes stores a new byte slice. Any existing value (even if of another type)
// is overwritten.
func (n *NamespacedKV) PutBytes(key string, val []byte) error {
return n.db.Put(n.prefixedKey(key), val)
}
// Bytes returns the stored value as a raw byte slice and a boolean that
// is false if no value was stored at the key.
func (n NamespacedKV) Bytes(key string) ([]byte, bool, error) {
valBs, err := n.db.Get(n.prefixedKey(key))
if err != nil {
return nil, false, filterNotFound(err)
}
return valBs, true, nil
}
// PutBool stores a new boolean. Any existing value (even if of another type)
// is overwritten.
func (n *NamespacedKV) PutBool(key string, val bool) error {
if val {
return n.db.Put(n.prefixedKey(key), []byte{0x0})
}
return n.db.Put(n.prefixedKey(key), []byte{0x1})
}
// Bool returns the stored value as a boolean and a boolean that
// is false if no value was stored at the key.
func (n NamespacedKV) Bool(key string) (bool, bool, error) {
valBs, err := n.db.Get(n.prefixedKey(key))
if err != nil {
return false, false, filterNotFound(err)
}
return valBs[0] == 0x0, true, nil
}
// Delete deletes the specified key. It is allowed to delete a nonexistent
// key.
func (n NamespacedKV) Delete(key string) error {
return n.db.Delete(n.prefixedKey(key))
}
func (n NamespacedKV) prefixedKey(key string) []byte {
return []byte(n.prefix + key)
}
// Well known namespaces that can be instantiated without knowing the key
// details.
// NewDeviceStatisticsNamespace creates a KV namespace for device statistics
// for the given device.
func NewDeviceStatisticsNamespace(db backend.Backend, device string) *NamespacedKV {
return NewNamespacedKV(db, string(KeyTypeDeviceStatistic)+device)
}
// NewFolderStatisticsNamespace creates a KV namespace for folder statistics
// for the given folder.
func NewFolderStatisticsNamespace(db backend.Backend, folder string) *NamespacedKV {
return NewNamespacedKV(db, string(KeyTypeFolderStatistic)+folder)
}
// NewMiscDataNamespace creates a KV namespace for miscellaneous metadata.
func NewMiscDataNamespace(db backend.Backend) *NamespacedKV {
return NewNamespacedKV(db, string(KeyTypeMiscData))
}
func filterNotFound(err error) error {
if backend.IsNotFound(err) {
return nil
}
return err
}
lib/db/namespaced_test.go
-177
View File
@@ -1,177 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"testing"
"time"
)
func TestNamespacedInt(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
n2 := NewNamespacedKV(ldb, "bar")
// Key is missing to start with
if v, ok, err := n1.Int64("test"); err != nil {
t.Error("Unexpected error:", err)
} else if v != 0 || ok {
t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
}
if err := n1.PutInt64("test", 42); err != nil {
t.Fatal(err)
}
// It should now exist in n1
if v, ok, err := n1.Int64("test"); err != nil {
t.Error("Unexpected error:", err)
} else if v != 42 || !ok {
t.Errorf("Incorrect return v %v != 42 || ok %v != true", v, ok)
}
// ... but not in n2, which is in a different namespace
if v, ok, err := n2.Int64("test"); err != nil {
t.Error("Unexpected error:", err)
} else if v != 0 || ok {
t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
}
if err := n1.Delete("test"); err != nil {
t.Fatal(err)
}
// It should no longer exist
if v, ok, err := n1.Int64("test"); err != nil {
t.Error("Unexpected error:", err)
} else if v != 0 || ok {
t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
}
}
func TestNamespacedTime(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
if v, ok, err := n1.Time("test"); err != nil {
t.Error("Unexpected error:", err)
} else if !v.IsZero() || ok {
t.Errorf("Incorrect return v %v != %v || ok %v != false", v, time.Time{}, ok)
}
now := time.Now()
if err := n1.PutTime("test", now); err != nil {
t.Fatal(err)
}
if v, ok, err := n1.Time("test"); err != nil {
t.Error("Unexpected error:", err)
} else if !v.Equal(now) || !ok {
t.Errorf("Incorrect return v %v != %v || ok %v != true", v, now, ok)
}
}
func TestNamespacedString(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
if v, ok, err := n1.String("test"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "" || ok {
t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
}
if err := n1.PutString("test", "yo"); err != nil {
t.Fatal(err)
}
if v, ok, err := n1.String("test"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "yo" || !ok {
t.Errorf("Incorrect return v %q != \"yo\" || ok %v != true", v, ok)
}
}
func TestNamespacedReset(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
if err := n1.PutString("test1", "yo1"); err != nil {
t.Fatal(err)
}
if err := n1.PutString("test2", "yo2"); err != nil {
t.Fatal(err)
}
if err := n1.PutString("test3", "yo3"); err != nil {
t.Fatal(err)
}
if v, ok, err := n1.String("test1"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "yo1" || !ok {
t.Errorf("Incorrect return v %q != \"yo1\" || ok %v != true", v, ok)
}
if v, ok, err := n1.String("test2"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "yo2" || !ok {
t.Errorf("Incorrect return v %q != \"yo2\" || ok %v != true", v, ok)
}
if v, ok, err := n1.String("test3"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "yo3" || !ok {
t.Errorf("Incorrect return v %q != \"yo3\" || ok %v != true", v, ok)
}
reset(n1)
if v, ok, err := n1.String("test1"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "" || ok {
t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
}
if v, ok, err := n1.String("test2"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "" || ok {
t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
}
if v, ok, err := n1.String("test3"); err != nil {
t.Error("Unexpected error:", err)
} else if v != "" || ok {
t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
}
}
// reset removes all entries in this namespace.
func reset(n *NamespacedKV) {
tr, err := n.db.NewWriteTransaction()
if err != nil {
return
}
defer tr.Release()
it, err := tr.NewPrefixIterator([]byte(n.prefix))
if err != nil {
return
}
for it.Next() {
_ = tr.Delete(it.Key())
}
it.Release()
_ = tr.Commit()
}
lib/db/observed.go
-205
View File
@@ -1,205 +0,0 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"fmt"
"time"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/protocol"
)
type ObservedFolder struct {
Time time.Time `json:"time"`
Label string `json:"label"`
ReceiveEncrypted bool `json:"receiveEncrypted"`
RemoteEncrypted bool `json:"remoteEncrypted"`
}
func (o *ObservedFolder) toWire() *dbproto.ObservedFolder {
return &dbproto.ObservedFolder{
Time: timestamppb.New(o.Time),
Label: o.Label,
ReceiveEncrypted: o.ReceiveEncrypted,
RemoteEncrypted: o.RemoteEncrypted,
}
}
func (o *ObservedFolder) fromWire(w *dbproto.ObservedFolder) {
o.Time = w.GetTime().AsTime()
o.Label = w.GetLabel()
o.ReceiveEncrypted = w.GetReceiveEncrypted()
o.RemoteEncrypted = w.GetRemoteEncrypted()
}
type ObservedDevice struct {
Time time.Time `json:"time"`
Name string `json:"name"`
Address string `json:"address"`
}
func (o *ObservedDevice) fromWire(w *dbproto.ObservedDevice) {
o.Time = w.GetTime().AsTime()
o.Name = w.GetName()
o.Address = w.GetAddress()
}
func (db *Lowlevel) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) error {
key := db.keyer.GeneratePendingDeviceKey(nil, device[:])
od := &dbproto.ObservedDevice{
Time: timestamppb.New(time.Now().Truncate(time.Second)),
Name: name,
Address: address,
}
return db.Put(key, mustMarshal(od))
}
func (db *Lowlevel) RemovePendingDevice(device protocol.DeviceID) error {
key := db.keyer.GeneratePendingDeviceKey(nil, device[:])
return db.Delete(key)
}
// PendingDevices enumerates all entries. Invalid ones are dropped from the database
// after a warning log message, as a side-effect.
func (db *Lowlevel) PendingDevices() (map[protocol.DeviceID]ObservedDevice, error) {
iter, err := db.NewPrefixIterator([]byte{KeyTypePendingDevice})
if err != nil {
return nil, err
}
defer iter.Release()
res := make(map[protocol.DeviceID]ObservedDevice)
for iter.Next() {
keyDev := db.keyer.DeviceFromPendingDeviceKey(iter.Key())
deviceID, err := protocol.DeviceIDFromBytes(keyDev)
var protoD dbproto.ObservedDevice
var od ObservedDevice
if err != nil {
goto deleteKey
}
if err = proto.Unmarshal(iter.Value(), &protoD); err != nil {
goto deleteKey
}
od.fromWire(&protoD)
res[deviceID] = od
continue
deleteKey:
// Deleting invalid entries is the only possible "repair" measure and
// appropriate for the importance of pending entries. They will come back
// soon if still relevant.
l.Infof("Invalid pending device entry, deleting from database: %x", iter.Key())
if err := db.Delete(iter.Key()); err != nil {
return nil, err
}
}
return res, nil
}
func (db *Lowlevel) AddOrUpdatePendingFolder(id string, of ObservedFolder, device protocol.DeviceID) error {
key, err := db.keyer.GeneratePendingFolderKey(nil, device[:], []byte(id))
if err != nil {
return err
}
return db.Put(key, mustMarshal(of.toWire()))
}
// RemovePendingFolderForDevice removes entries for specific folder / device combinations.
func (db *Lowlevel) RemovePendingFolderForDevice(id string, device protocol.DeviceID) error {
key, err := db.keyer.GeneratePendingFolderKey(nil, device[:], []byte(id))
if err != nil {
return err
}
return db.Delete(key)
}
// RemovePendingFolder removes all entries matching a specific folder ID.
func (db *Lowlevel) RemovePendingFolder(id string) error {
iter, err := db.NewPrefixIterator([]byte{KeyTypePendingFolder})
if err != nil {
return fmt.Errorf("creating iterator: %w", err)
}
defer iter.Release()
var iterErr error
for iter.Next() {
if id != string(db.keyer.FolderFromPendingFolderKey(iter.Key())) {
continue
}
if err = db.Delete(iter.Key()); err != nil {
if iterErr != nil {
l.Debugf("Repeat error removing pending folder: %v", err)
} else {
iterErr = err
}
}
}
return iterErr
}
// Consolidated information about a pending folder
type PendingFolder struct {
OfferedBy map[protocol.DeviceID]ObservedFolder `json:"offeredBy"`
}
func (db *Lowlevel) PendingFolders() (map[string]PendingFolder, error) {
return db.PendingFoldersForDevice(protocol.EmptyDeviceID)
}
// PendingFoldersForDevice enumerates only entries matching the given device ID, unless it
// is EmptyDeviceID. Invalid ones are dropped from the database after a info log
// message, as a side-effect.
func (db *Lowlevel) PendingFoldersForDevice(device protocol.DeviceID) (map[string]PendingFolder, error) {
var err error
prefixKey := []byte{KeyTypePendingFolder}
if device != protocol.EmptyDeviceID {
prefixKey, err = db.keyer.GeneratePendingFolderKey(nil, device[:], nil)
if err != nil {
return nil, err
}
}
iter, err := db.NewPrefixIterator(prefixKey)
if err != nil {
return nil, err
}
defer iter.Release()
res := make(map[string]PendingFolder)
for iter.Next() {
keyDev, ok := db.keyer.DeviceFromPendingFolderKey(iter.Key())
deviceID, err := protocol.DeviceIDFromBytes(keyDev)
var protoF dbproto.ObservedFolder
var of ObservedFolder
var folderID string
if !ok || err != nil {
goto deleteKey
}
if folderID = string(db.keyer.FolderFromPendingFolderKey(iter.Key())); len(folderID) < 1 {
goto deleteKey
}
if err = proto.Unmarshal(iter.Value(), &protoF); err != nil {
goto deleteKey
}
if _, ok := res[folderID]; !ok {
res[folderID] = PendingFolder{
OfferedBy: map[protocol.DeviceID]ObservedFolder{},
}
}
of.fromWire(&protoF)
res[folderID].OfferedBy[deviceID] = of
continue
deleteKey:
// Deleting invalid entries is the only possible "repair" measure and
// appropriate for the importance of pending entries. They will come back
// soon if still relevant.
l.Infof("Invalid pending folder entry, deleting from database: %x", iter.Key())
if err := db.Delete(iter.Key()); err != nil {
return nil, err
}
}
return res, nil
}
lib/db/schemaupdater.go
-271
View File
@@ -1,271 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"fmt"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/bep"
"github.com/syncthing/syncthing/lib/protocol"
)
// dbMigrationVersion is for migrations that do not change the schema and thus
// do not put restrictions on downgrades (e.g. for repairs after a bugfix).
const (
dbVersion = 14
dbMigrationVersion = 20
dbMinSyncthingVersion = "v1.9.0"
)
type migration struct {
schemaVersion int64
migrationVersion int64
minSyncthingVersion string
migration func(prevSchema int) error
}
type databaseDowngradeError struct {
minSyncthingVersion string
}
func (e *databaseDowngradeError) Error() string {
if e.minSyncthingVersion == "" {
return "newer Syncthing required"
}
return fmt.Sprintf("Syncthing %s required", e.minSyncthingVersion)
}
// UpdateSchema updates a possibly outdated database to the current schema and
// also does repairs where necessary.
func UpdateSchema(db *Lowlevel) error {
updater := &schemaUpdater{db}
return updater.updateSchema()
}
type schemaUpdater struct {
*Lowlevel
}
func (db *schemaUpdater) updateSchema() error {
// Updating the schema can touch any and all parts of the database. Make
// sure we do not run GC concurrently with schema migrations.
db.gcMut.Lock()
defer db.gcMut.Unlock()
miscDB := NewMiscDataNamespace(db.Lowlevel)
prevVersion, _, err := miscDB.Int64("dbVersion")
if err != nil {
return err
}
if prevVersion > 0 && prevVersion < 14 {
// This is a database version that is too old to be upgraded directly.
// The user will have to upgrade to an older version first.
return fmt.Errorf("database version %d is too old to be upgraded directly; step via Syncthing v1.27.0 to upgrade", prevVersion)
}
if prevVersion > dbVersion {
err := &databaseDowngradeError{}
if minSyncthingVersion, ok, dbErr := miscDB.String("dbMinSyncthingVersion"); dbErr != nil {
return dbErr
} else if ok {
err.minSyncthingVersion = minSyncthingVersion
}
return err
}
prevMigration, _, err := miscDB.Int64("dbMigrationVersion")
if err != nil {
return err
}
// Cover versions before adding `dbMigrationVersion` (== 0) and possible future weirdness.
if prevMigration < prevVersion {
prevMigration = prevVersion
}
if prevVersion == dbVersion && prevMigration >= dbMigrationVersion {
return nil
}
migrations := []migration{
{14, 14, "v1.9.0", db.updateSchemaTo14},
{14, 16, "v1.9.0", db.checkRepairMigration},
{14, 17, "v1.9.0", db.migration17},
{14, 19, "v1.9.0", db.dropAllIndexIDsMigration},
{14, 20, "v1.9.0", db.dropOutgoingIndexIDsMigration},
}
for _, m := range migrations {
if prevMigration < m.migrationVersion {
l.Infof("Running database migration %d...", m.migrationVersion)
if err := m.migration(int(prevVersion)); err != nil {
return fmt.Errorf("failed to do migration %v: %w", m.migrationVersion, err)
}
if err := db.writeVersions(m, miscDB); err != nil {
return fmt.Errorf("failed to write versions after migration %v: %w", m.migrationVersion, err)
}
}
}
if err := db.writeVersions(migration{
schemaVersion: dbVersion,
migrationVersion: dbMigrationVersion,
minSyncthingVersion: dbMinSyncthingVersion,
}, miscDB); err != nil {
return fmt.Errorf("failed to write versions after migrations: %w", err)
}
l.Infoln("Compacting database after migration...")
return db.Compact()
}
func (*schemaUpdater) writeVersions(m migration, miscDB *NamespacedKV) error {
if err := miscDB.PutInt64("dbVersion", m.schemaVersion); err != nil {
return err
}
if err := miscDB.PutString("dbMinSyncthingVersion", m.minSyncthingVersion); err != nil {
return err
}
if err := miscDB.PutInt64("dbMigrationVersion", m.migrationVersion); err != nil {
return err
}
return nil
}
func (db *schemaUpdater) updateSchemaTo14(_ int) error {
// Checks for missing blocks and marks those entries as requiring a
// rehash/being invalid. The db is checked/repaired afterwards, i.e.
// no care is taken to get metadata and sequences right.
// If the corresponding files changed on disk compared to the global
// version, this will cause a conflict.
var key, gk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
meta := newMetadataTracker(db.keyer, db.evLogger)
meta.counts.Created = 0 // Recalculate metadata afterwards
t, err := db.newReadWriteTransaction(meta.CommitHook(folder))
if err != nil {
return err
}
defer t.close()
key, err = t.keyer.GenerateDeviceFileKey(key, folder, protocol.LocalDeviceID[:], nil)
if err != nil {
return err
}
it, err := t.NewPrefixIterator(key)
if err != nil {
return err
}
defer it.Release()
for it.Next() {
var bepf bep.FileInfo
if err := proto.Unmarshal(it.Value(), &bepf); err != nil {
return err
}
fi := protocol.FileInfoFromDB(&bepf)
if len(fi.Blocks) > 0 || len(fi.BlocksHash) == 0 {
continue
}
key = t.keyer.GenerateBlockListKey(key, fi.BlocksHash)
_, err := t.Get(key)
if err == nil {
continue
}
fi.SetMustRescan()
if err = t.putFile(it.Key(), fi); err != nil {
return err
}
gk, err = t.keyer.GenerateGlobalVersionKey(gk, folder, []byte(fi.Name))
if err != nil {
return err
}
key, err = t.updateGlobal(gk, key, folder, protocol.LocalDeviceID[:], fi, meta)
if err != nil {
return err
}
}
it.Release()
if err = t.Commit(); err != nil {
return err
}
t.close()
}
return nil
}
func (db *schemaUpdater) checkRepairMigration(_ int) error {
for _, folder := range db.ListFolders() {
_, err := db.getMetaAndCheckGCLocked(folder)
if err != nil {
return err
}
}
return nil
}
// migration17 finds all files that were pulled as invalid from an invalid
// global and make sure they get scanned/pulled again.
func (db *schemaUpdater) migration17(prev int) error {
if prev < 16 {
// Issue was introduced in migration to 16
return nil
}
t, err := db.newReadOnlyTransaction()
if err != nil {
return err
}
defer t.close()
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
meta, err := db.loadMetadataTracker(folderStr)
if err != nil {
return err
}
batch := NewFileInfoBatch(func(fs []protocol.FileInfo) error {
return db.updateLocalFiles(folder, fs, meta)
})
var innerErr error
err = t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(fi protocol.FileInfo) bool {
if fi.IsInvalid() && fi.FileLocalFlags() == 0 {
fi.SetMustRescan()
fi.Version = protocol.Vector{}
batch.Append(fi)
innerErr = batch.FlushIfFull()
return innerErr == nil
}
return true
})
if innerErr != nil {
return innerErr
}
if err != nil {
return err
}
if err := batch.Flush(); err != nil {
return err
}
}
return nil
}
func (db *schemaUpdater) dropAllIndexIDsMigration(_ int) error {
return db.dropIndexIDs()
}
func (db *schemaUpdater) dropOutgoingIndexIDsMigration(_ int) error {
return db.dropOtherDeviceIndexIDs()
}
lib/db/set.go
-553
View File
@@ -1,553 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Package db provides a set type to track local/remote files with newness
// checks. We must do a certain amount of normalization in here. We will get
// fed paths with either native or wire-format separators and encodings
// depending on who calls us. We transform paths to wire-format (NFC and
// slashes) on the way to the database, and transform to native format
// (varying separator and encoding) on the way back out.
package db
import (
"bytes"
"fmt"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
type FileSet struct {
folder string
db *Lowlevel
meta *metadataTracker
updateMutex sync.Mutex // protects database updates and the corresponding metadata changes
}
// The Iterator is called with either a protocol.FileInfo or a
// FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f protocol.FileInfo) bool
func NewFileSet(folder string, db *Lowlevel) (*FileSet, error) {
select {
case <-db.oneFileSetCreated:
default:
close(db.oneFileSetCreated)
}
meta, err := db.loadMetadataTracker(folder)
if err != nil {
db.handleFailure(err)
return nil, err
}
s := &FileSet{
folder: folder,
db: db,
meta: meta,
updateMutex: sync.NewMutex(),
}
if id := s.IndexID(protocol.LocalDeviceID); id == 0 {
// No index ID set yet. We create one now.
id = protocol.NewIndexID()
err := s.db.setIndexID(protocol.LocalDeviceID[:], []byte(s.folder), id)
if err != nil && !backend.IsClosed(err) {
fatalError(err, fmt.Sprintf("%s Creating new IndexID", s.folder), s.db)
}
}
return s, nil
}
func (s *FileSet) Drop(device protocol.DeviceID) {
opStr := fmt.Sprintf("%s Drop(%v)", s.folder, device)
l.Debugf(opStr)
s.updateMutex.Lock()
defer s.updateMutex.Unlock()
if err := s.db.dropDeviceFolder(device[:], []byte(s.folder), s.meta); backend.IsClosed(err) {
return
} else if err != nil {
fatalError(err, opStr, s.db)
}
if device == protocol.LocalDeviceID {
s.meta.resetCounts(device)
// We deliberately do not reset the sequence number here. Dropping
// all files for the local device ID only happens in testing - which
// expects the sequence to be retained, like an old Replace() of all
// files would do. However, if we ever did it "in production" we
// would anyway want to retain the sequence for delta indexes to be
// happy.
} else {
// Here, on the other hand, we want to make sure that any file
// announced from the remote is newer than our current sequence
// number.
s.meta.resetAll(device)
}
t, err := s.db.newReadWriteTransaction()
if backend.IsClosed(err) {
return
} else if err != nil {
fatalError(err, opStr, s.db)
}
defer t.close()
if err := s.meta.toDB(t, []byte(s.folder)); backend.IsClosed(err) {
return
} else if err != nil {
fatalError(err, opStr, s.db)
}
if err := t.Commit(); backend.IsClosed(err) {
return
} else if err != nil {
fatalError(err, opStr, s.db)
}
}
func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
opStr := fmt.Sprintf("%s Update(%v, [%d])", s.folder, device, len(fs))
l.Debugf(opStr)
// do not modify fs in place, it is still used in outer scope
fs = append([]protocol.FileInfo(nil), fs...)
// If one file info is present multiple times, only keep the last.
// Updating the same file multiple times is problematic, because the
// previous updates won't yet be represented in the db when we update it
// again. Additionally even if that problem was taken care of, it would
// be pointless because we remove the previously added file info again
// right away.
fs = normalizeFilenamesAndDropDuplicates(fs)
s.updateMutex.Lock()
defer s.updateMutex.Unlock()
if device == protocol.LocalDeviceID {
// For the local device we have a bunch of metadata to track.
if err := s.db.updateLocalFiles([]byte(s.folder), fs, s.meta); err != nil && !backend.IsClosed(err) {
fatalError(err, opStr, s.db)
}
return
}
// Easy case, just update the files and we're done.
if err := s.db.updateRemoteFiles([]byte(s.folder), device[:], fs, s.meta); err != nil && !backend.IsClosed(err) {
fatalError(err, opStr, s.db)
}
}
func (s *FileSet) RemoveLocalItems(items []string) {
opStr := fmt.Sprintf("%s RemoveLocalItems([%d])", s.folder, len(items))
l.Debugf(opStr)
s.updateMutex.Lock()
defer s.updateMutex.Unlock()
for i := range items {
items[i] = osutil.NormalizedFilename(items[i])
}
if err := s.db.removeLocalFiles([]byte(s.folder), items, s.meta); err != nil && !backend.IsClosed(err) {
fatalError(err, opStr, s.db)
}
}
type Snapshot struct {
folder string
t readOnlyTransaction
meta *countsMap
fatalError func(error, string)
}
func (s *FileSet) Snapshot() (*Snapshot, error) {
opStr := fmt.Sprintf("%s Snapshot()", s.folder)
l.Debugf(opStr)
s.updateMutex.Lock()
defer s.updateMutex.Unlock()
t, err := s.db.newReadOnlyTransaction()
if err != nil {
s.db.handleFailure(err)
return nil, err
}
return &Snapshot{
folder: s.folder,
t: t,
meta: s.meta.Snapshot(),
fatalError: func(err error, opStr string) {
fatalError(err, opStr, s.db)
},
}, nil
}
func (s *Snapshot) Release() {
s.t.close()
}
func (s *Snapshot) WithNeed(device protocol.DeviceID, fn Iterator) {
opStr := fmt.Sprintf("%s WithNeed(%v)", s.folder, device)
l.Debugf(opStr)
if err := s.t.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *Snapshot) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
opStr := fmt.Sprintf("%s WithNeedTruncated(%v)", s.folder, device)
l.Debugf(opStr)
if err := s.t.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *Snapshot) WithHave(device protocol.DeviceID, fn Iterator) {
opStr := fmt.Sprintf("%s WithHave(%v)", s.folder, device)
l.Debugf(opStr)
if err := s.t.withHave([]byte(s.folder), device[:], nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *Snapshot) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
opStr := fmt.Sprintf("%s WithHaveTruncated(%v)", s.folder, device)
l.Debugf(opStr)
if err := s.t.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *Snapshot) WithHaveSequence(startSeq int64, fn Iterator) {
opStr := fmt.Sprintf("%s WithHaveSequence(%v)", s.folder, startSeq)
l.Debugf(opStr)
if err := s.t.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
// Except for an item with a path equal to prefix, only children of prefix are iterated.
// E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not.
func (s *Snapshot) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) {
opStr := fmt.Sprintf(`%s WithPrefixedHaveTruncated(%v, "%v")`, s.folder, device, prefix)
l.Debugf(opStr)
if err := s.t.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *Snapshot) WithGlobal(fn Iterator) {
opStr := fmt.Sprintf("%s WithGlobal()", s.folder)
l.Debugf(opStr)
if err := s.t.withGlobal([]byte(s.folder), nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *Snapshot) WithGlobalTruncated(fn Iterator) {
opStr := fmt.Sprintf("%s WithGlobalTruncated()", s.folder)
l.Debugf(opStr)
if err := s.t.withGlobal([]byte(s.folder), nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
// Except for an item with a path equal to prefix, only children of prefix are iterated.
// E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not.
func (s *Snapshot) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
opStr := fmt.Sprintf(`%s WithPrefixedGlobalTruncated("%v")`, s.folder, prefix)
l.Debugf(opStr)
if err := s.t.withGlobal([]byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *Snapshot) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
opStr := fmt.Sprintf("%s Get(%v)", s.folder, file)
l.Debugf(opStr)
f, ok, err := s.t.getFile([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
if backend.IsClosed(err) {
return protocol.FileInfo{}, false
} else if err != nil {
s.fatalError(err, opStr)
}
f.Name = osutil.NativeFilename(f.Name)
return f, ok
}
func (s *Snapshot) GetGlobal(file string) (protocol.FileInfo, bool) {
opStr := fmt.Sprintf("%s GetGlobal(%v)", s.folder, file)
l.Debugf(opStr)
_, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
if backend.IsClosed(err) {
return protocol.FileInfo{}, false
} else if err != nil {
s.fatalError(err, opStr)
}
if !ok {
return protocol.FileInfo{}, false
}
fi.Name = osutil.NativeFilename(fi.Name)
return fi, true
}
func (s *Snapshot) GetGlobalTruncated(file string) (protocol.FileInfo, bool) {
opStr := fmt.Sprintf("%s GetGlobalTruncated(%v)", s.folder, file)
l.Debugf(opStr)
_, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
if backend.IsClosed(err) {
return protocol.FileInfo{}, false
} else if err != nil {
s.fatalError(err, opStr)
}
if !ok {
return protocol.FileInfo{}, false
}
fi.Name = osutil.NativeFilename(fi.Name)
return fi, true
}
func (s *Snapshot) Availability(file string) []protocol.DeviceID {
opStr := fmt.Sprintf("%s Availability(%v)", s.folder, file)
l.Debugf(opStr)
av, err := s.t.availability([]byte(s.folder), []byte(osutil.NormalizedFilename(file)))
if backend.IsClosed(err) {
return nil
} else if err != nil {
s.fatalError(err, opStr)
}
return av
}
func (s *Snapshot) DebugGlobalVersions(file string) *DebugVersionList {
opStr := fmt.Sprintf("%s DebugGlobalVersions(%v)", s.folder, file)
l.Debugf(opStr)
vl, err := s.t.getGlobalVersions(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
if backend.IsClosed(err) || backend.IsNotFound(err) {
return nil
} else if err != nil {
s.fatalError(err, opStr)
}
return &DebugVersionList{vl}
}
func (s *Snapshot) Sequence(device protocol.DeviceID) int64 {
return s.meta.Counts(device, 0).Sequence
}
// RemoteSequences returns a map of the sequence numbers seen for each
// remote device sharing this folder.
func (s *Snapshot) RemoteSequences() map[protocol.DeviceID]int64 {
res := make(map[protocol.DeviceID]int64)
for _, device := range s.meta.devices() {
switch device {
case protocol.EmptyDeviceID, protocol.LocalDeviceID, protocol.GlobalDeviceID:
continue
default:
if seq := s.Sequence(device); seq > 0 {
res[device] = seq
}
}
}
return res
}
func (s *Snapshot) LocalSize() Counts {
local := s.meta.Counts(protocol.LocalDeviceID, 0)
return local.Add(s.ReceiveOnlyChangedSize())
}
func (s *Snapshot) ReceiveOnlyChangedSize() Counts {
return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
}
func (s *Snapshot) GlobalSize() Counts {
return s.meta.Counts(protocol.GlobalDeviceID, 0)
}
func (s *Snapshot) NeedSize(device protocol.DeviceID) Counts {
return s.meta.Counts(device, needFlag)
}
func (s *Snapshot) WithBlocksHash(hash []byte, fn Iterator) {
opStr := fmt.Sprintf(`%s WithBlocksHash("%x")`, s.folder, hash)
l.Debugf(opStr)
if err := s.t.withBlocksHash([]byte(s.folder), hash, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
s.fatalError(err, opStr)
}
}
func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
return s.meta.Sequence(device)
}
func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID {
opStr := fmt.Sprintf("%s IndexID(%v)", s.folder, device)
l.Debugf(opStr)
id, err := s.db.getIndexID(device[:], []byte(s.folder))
if backend.IsClosed(err) {
return 0
} else if err != nil {
fatalError(err, opStr, s.db)
}
return id
}
func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
if device == protocol.LocalDeviceID {
panic("do not explicitly set index ID for local device")
}
opStr := fmt.Sprintf("%s SetIndexID(%v, %v)", s.folder, device, id)
l.Debugf(opStr)
if err := s.db.setIndexID(device[:], []byte(s.folder), id); err != nil && !backend.IsClosed(err) {
fatalError(err, opStr, s.db)
}
}
func (s *FileSet) MtimeOption() fs.Option {
opStr := fmt.Sprintf("%s MtimeOption()", s.folder)
l.Debugf(opStr)
prefix, err := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
if backend.IsClosed(err) {
return nil
} else if err != nil {
fatalError(err, opStr, s.db)
}
kv := NewNamespacedKV(s.db, string(prefix))
return fs.NewMtimeOption(kv)
}
func (s *FileSet) ListDevices() []protocol.DeviceID {
return s.meta.devices()
}
func (s *FileSet) RepairSequence() (int, error) {
s.updateAndGCMutexLock() // Ensures consistent locking order
defer s.updateMutex.Unlock()
defer s.db.gcMut.RUnlock()
return s.db.repairSequenceGCLocked(s.folder, s.meta)
}
func (s *FileSet) updateAndGCMutexLock() {
s.updateMutex.Lock()
s.db.gcMut.RLock()
}
// DropFolder clears out all information related to the given folder from the
// database.
func DropFolder(db *Lowlevel, folder string) {
opStr := fmt.Sprintf("DropFolder(%v)", folder)
l.Debugf(opStr)
droppers := []func([]byte) error{
db.dropFolder,
db.dropMtimes,
db.dropFolderMeta,
db.dropFolderIndexIDs,
db.folderIdx.Delete,
}
for _, drop := range droppers {
if err := drop([]byte(folder)); backend.IsClosed(err) {
return
} else if err != nil {
fatalError(err, opStr, db)
}
}
}
// DropDeltaIndexIDs removes all delta index IDs from the database.
// This will cause a full index transmission on the next connection.
// Must be called before using FileSets, i.e. before NewFileSet is called for
// the first time.
func DropDeltaIndexIDs(db *Lowlevel) {
select {
case <-db.oneFileSetCreated:
panic("DropDeltaIndexIDs must not be called after NewFileSet for the same Lowlevel")
default:
}
opStr := "DropDeltaIndexIDs"
l.Debugf(opStr)
err := db.dropIndexIDs()
if backend.IsClosed(err) {
return
} else if err != nil {
fatalError(err, opStr, db)
}
}
func normalizeFilenamesAndDropDuplicates(fs []protocol.FileInfo) []protocol.FileInfo {
positions := make(map[string]int, len(fs))
for i, f := range fs {
norm := osutil.NormalizedFilename(f.Name)
if pos, ok := positions[norm]; ok {
fs[pos] = protocol.FileInfo{}
}
positions[norm] = i
fs[i].Name = norm
}
for i := 0; i < len(fs); {
if fs[i].Name == "" {
fs = append(fs[:i], fs[i+1:]...)
continue
}
i++
}
return fs
}
func nativeFileIterator(fn Iterator) Iterator {
return func(fi protocol.FileInfo) bool {
fi.Name = osutil.NativeFilename(fi.Name)
return fn(fi)
}
}
func fatalError(err error, opStr string, db *Lowlevel) {
db.checkErrorForRepair(err)
l.Warnf("Fatal error: %v: %v", opStr, err)
panic(ldbPathRe.ReplaceAllString(err.Error(), "$1 x: "))
}
// DebugFileVersion is the database-internal representation of a file
// version, with a nicer string representation, used only by API debug
// methods.
type DebugVersionList struct {
*dbproto.VersionList
}
func (vl DebugVersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("[")
for i, v := range vl.Versions {
if i > 0 {
b.WriteString(", ")
}
fmt.Fprintf(&b, "{Version:%v, Deleted:%v, Devices:[", protocol.VectorFromWire(v.Version), v.Deleted)
for j, dev := range v.Devices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
b.WriteString("], Invalid:[")
for j, dev := range v.InvalidDevices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
fmt.Fprint(&b, "]}")
}
b.WriteString("]")
return b.String()
}
lib/db/set_test.go
-1901
View File
File diff suppressed because it is too large Load Diff
lib/db/smallindex.go
-152
View File
@@ -1,152 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/binary"
"sort"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/sync"
)
// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
// fast lookups in both directions and persists to the database. Don't use for
// storing more items than fit comfortably in RAM.
type smallIndex struct {
db backend.Backend
prefix []byte
id2val map[uint32]string
val2id map[string]uint32
nextID uint32
mut sync.Mutex
}
func newSmallIndex(db backend.Backend, prefix []byte) *smallIndex {
idx := &smallIndex{
db: db,
prefix: prefix,
id2val: make(map[uint32]string),
val2id: make(map[string]uint32),
mut: sync.NewMutex(),
}
idx.load()
return idx
}
// load iterates over the prefix space in the database and populates the in
// memory maps.
func (i *smallIndex) load() {
it, err := i.db.NewPrefixIterator(i.prefix)
if err != nil {
panic("loading small index: " + err.Error())
}
defer it.Release()
for it.Next() {
val := string(it.Value())
id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
if val != "" {
// Empty value means the entry has been deleted.
i.id2val[id] = val
i.val2id[val] = id
}
if id >= i.nextID {
i.nextID = id + 1
}
}
}
// ID returns the index number for the given byte slice, allocating a new one
// and persisting this to the database if necessary.
func (i *smallIndex) ID(val []byte) (uint32, error) {
i.mut.Lock()
// intentionally avoiding defer here as we want this call to be as fast as
// possible in the general case (folder ID already exists). The map lookup
// with the conversion of []byte to string is compiler optimized to not
// copy the []byte, which is why we don't assign it to a temp variable
// here.
if id, ok := i.val2id[string(val)]; ok {
i.mut.Unlock()
return id, nil
}
id := i.nextID
i.nextID++
valStr := string(val)
i.val2id[valStr] = id
i.id2val[id] = valStr
key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
copy(key, i.prefix)
binary.BigEndian.PutUint32(key[len(i.prefix):], id)
if err := i.db.Put(key, val); err != nil {
i.mut.Unlock()
return 0, err
}
i.mut.Unlock()
return id, nil
}
// Val returns the value for the given index number, or (nil, false) if there
// is no such index number.
func (i *smallIndex) Val(id uint32) ([]byte, bool) {
i.mut.Lock()
val, ok := i.id2val[id]
i.mut.Unlock()
if !ok {
return nil, false
}
return []byte(val), true
}
func (i *smallIndex) Delete(val []byte) error {
i.mut.Lock()
defer i.mut.Unlock()
// Check the reverse mapping to get the ID for the value.
if id, ok := i.val2id[string(val)]; ok {
// Generate the corresponding database key.
key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
copy(key, i.prefix)
binary.BigEndian.PutUint32(key[len(i.prefix):], id)
// Put an empty value into the database. This indicates that the
// entry does not exist any more and prevents the ID from being
// reused in the future.
if err := i.db.Put(key, []byte{}); err != nil {
return err
}
// Delete reverse mapping.
delete(i.id2val, id)
}
// Delete forward mapping.
delete(i.val2id, string(val))
return nil
}
// Values returns the set of values in the index
func (i *smallIndex) Values() []string {
// In principle this method should return [][]byte because all the other
// methods deal in []byte keys. However, in practice, where it's used
// wants a []string and it's easier to just create that here rather than
// having to convert both here and there...
i.mut.Lock()
vals := make([]string, 0, len(i.val2id))
for val := range i.val2id {
vals = append(vals, val)
}
i.mut.Unlock()
sort.Strings(vals)
return vals
}
lib/db/smallindex_test.go
-62
View File
@@ -1,62 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"testing"
)
func TestSmallIndex(t *testing.T) {
db := newLowlevelMemory(t)
idx := newSmallIndex(db, []byte{12, 34})
// ID zero should be unallocated
if val, ok := idx.Val(0); ok || val != nil {
t.Fatal("Unexpected return for nonexistent ID 0")
}
// A new key should get ID zero
if id, err := idx.ID([]byte("hello")); err != nil {
t.Fatal(err)
} else if id != 0 {
t.Fatal("Expected 0, not", id)
}
// Looking up ID zero should work
if val, ok := idx.Val(0); !ok || string(val) != "hello" {
t.Fatalf(`Expected true, "hello", not %v, %q`, ok, val)
}
// Delete the key
idx.Delete([]byte("hello"))
// Next ID should be one
if id, err := idx.ID([]byte("key2")); err != nil {
t.Fatal(err)
} else if id != 1 {
t.Fatal("Expected 1, not", id)
}
// Now lets create a new index instance based on what's actually serialized to the database.
idx = newSmallIndex(db, []byte{12, 34})
// Status should be about the same as before.
if val, ok := idx.Val(0); ok || val != nil {
t.Fatal("Unexpected return for deleted ID 0")
}
if id, err := idx.ID([]byte("key2")); err != nil {
t.Fatal(err)
} else if id != 1 {
t.Fatal("Expected 1, not", id)
}
// Setting "hello" again should get us ID 2, not 0 as it was originally.
if id, err := idx.ID([]byte("hello")); err != nil {
t.Fatal(err)
} else if id != 2 {
t.Fatal("Expected 2, not", id)
}
}
lib/db/structs.go
-363
View File
@@ -1,363 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"bytes"
"fmt"
"strings"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/protocol"
)
type CountsSet struct {
Counts []Counts
Created int64 // unix nanos
}
type Counts struct {
Files int
Directories int
Symlinks int
Deleted int
Bytes int64
Sequence int64 // zero for the global state
DeviceID protocol.DeviceID // device ID for remote devices, or special values for local/global
LocalFlags uint32 // the local flag for this count bucket
}
func (c Counts) toWire() *dbproto.Counts {
return &dbproto.Counts{
Files: int32(c.Files),
Directories: int32(c.Directories),
Symlinks: int32(c.Symlinks),
Deleted: int32(c.Deleted),
Bytes: c.Bytes,
Sequence: c.Sequence,
DeviceId: c.DeviceID[:],
LocalFlags: c.LocalFlags,
}
}
func countsFromWire(w *dbproto.Counts) Counts {
return Counts{
Files: int(w.Files),
Directories: int(w.Directories),
Symlinks: int(w.Symlinks),
Deleted: int(w.Deleted),
Bytes: w.Bytes,
Sequence: w.Sequence,
DeviceID: protocol.DeviceID(w.DeviceId),
LocalFlags: w.LocalFlags,
}
}
func (c Counts) Add(other Counts) Counts {
return Counts{
Files: c.Files + other.Files,
Directories: c.Directories + other.Directories,
Symlinks: c.Symlinks + other.Symlinks,
Deleted: c.Deleted + other.Deleted,
Bytes: c.Bytes + other.Bytes,
Sequence: c.Sequence + other.Sequence,
DeviceID: protocol.EmptyDeviceID,
LocalFlags: c.LocalFlags | other.LocalFlags,
}
}
func (c Counts) TotalItems() int {
return c.Files + c.Directories + c.Symlinks + c.Deleted
}
func (c Counts) String() string {
var flags strings.Builder
if c.LocalFlags&needFlag != 0 {
flags.WriteString("Need")
}
if c.LocalFlags&protocol.FlagLocalIgnored != 0 {
flags.WriteString("Ignored")
}
if c.LocalFlags&protocol.FlagLocalMustRescan != 0 {
flags.WriteString("Rescan")
}
if c.LocalFlags&protocol.FlagLocalReceiveOnly != 0 {
flags.WriteString("Recvonly")
}
if c.LocalFlags&protocol.FlagLocalUnsupported != 0 {
flags.WriteString("Unsupported")
}
if c.LocalFlags != 0 {
flags.WriteString(fmt.Sprintf("(%x)", c.LocalFlags))
}
if flags.Len() == 0 {
flags.WriteString("---")
}
return fmt.Sprintf("{Device:%v, Files:%d, Dirs:%d, Symlinks:%d, Del:%d, Bytes:%d, Seq:%d, Flags:%s}", c.DeviceID, c.Files, c.Directories, c.Symlinks, c.Deleted, c.Bytes, c.Sequence, flags.String())
}
// Equal compares the numbers only, not sequence/dev/flags.
func (c Counts) Equal(o Counts) bool {
return c.Files == o.Files && c.Directories == o.Directories && c.Symlinks == o.Symlinks && c.Deleted == o.Deleted && c.Bytes == o.Bytes
}
// update brings the VersionList up to date with file. It returns the updated
// VersionList, a device that has the global/newest version, a device that previously
// had the global/newest version, a boolean indicating if the global version has
// changed and if any error occurred (only possible in db interaction).
func vlUpdate(vl *dbproto.VersionList, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (*dbproto.FileVersion, *dbproto.FileVersion, *dbproto.FileVersion, bool, bool, bool, error) {
if len(vl.Versions) == 0 {
nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())
vl.Versions = append(vl.Versions, nv)
return nv, nil, nil, false, false, true, nil
}
// Get the current global (before updating)
oldFV, haveOldGlobal := vlGetGlobal(vl)
oldFV = fvCopy(oldFV)
// Remove ourselves first
removedFV, haveRemoved, _ := vlPop(vl, device)
// Find position and insert the file
err := vlInsert(vl, folder, device, file, t)
if err != nil {
return nil, nil, nil, false, false, false, err
}
newFV, _ := vlGetGlobal(vl) // We just inserted something above, can't be empty
if !haveOldGlobal {
return newFV, nil, removedFV, false, haveRemoved, true, nil
}
globalChanged := true
if fvIsInvalid(oldFV) == fvIsInvalid(newFV) && protocol.VectorFromWire(oldFV.Version).Equal(protocol.VectorFromWire(newFV.Version)) {
globalChanged = false
}
return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil
}
func vlInsert(vl *dbproto.VersionList, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) error {
var added bool
var err error
i := 0
for ; i < len(vl.Versions); i++ {
// Insert our new version
added, err = vlCheckInsertAt(vl, i, folder, device, file, t)
if err != nil {
return err
}
if added {
break
}
}
if i == len(vl.Versions) {
// Append to the end
vl.Versions = append(vl.Versions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
}
return nil
}
func vlInsertAt(vl *dbproto.VersionList, i int, v *dbproto.FileVersion) {
vl.Versions = append(vl.Versions, &dbproto.FileVersion{})
copy(vl.Versions[i+1:], vl.Versions[i:])
vl.Versions[i] = v
}
// pop removes the given device from the VersionList and returns the FileVersion
// before removing the device, whether it was found/removed at all and whether
// the global changed in the process.
func vlPop(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool, bool) {
invDevice, i, j, ok := vlFindDevice(vl, device)
if !ok {
return nil, false, false
}
globalPos := vlFindGlobal(vl)
fv := vl.Versions[i]
if fvDeviceCount(fv) == 1 {
vlPopVersionAt(vl, i)
return fv, true, globalPos == i
}
oldFV := fvCopy(fv)
if invDevice {
vl.Versions[i].InvalidDevices = popDeviceAt(vl.Versions[i].InvalidDevices, j)
return oldFV, true, false
}
vl.Versions[i].Devices = popDeviceAt(vl.Versions[i].Devices, j)
// If the last valid device of the previous global was removed above,
// the global changed.
return oldFV, true, len(vl.Versions[i].Devices) == 0 && globalPos == i
}
// Get returns a FileVersion that contains the given device and whether it has
// been found at all.
func vlGet(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool) {
_, i, _, ok := vlFindDevice(vl, device)
if !ok {
return &dbproto.FileVersion{}, false
}
return vl.Versions[i], true
}
// GetGlobal returns the current global FileVersion. The returned FileVersion
// may be invalid, if all FileVersions are invalid. Returns false only if
// VersionList is empty.
func vlGetGlobal(vl *dbproto.VersionList) (*dbproto.FileVersion, bool) {
i := vlFindGlobal(vl)
if i == -1 {
return nil, false
}
return vl.Versions[i], true
}
// findGlobal returns the first version that isn't invalid, or if all versions are
// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
func vlFindGlobal(vl *dbproto.VersionList) int {
for i := range vl.Versions {
if !fvIsInvalid(vl.Versions[i]) {
return i
}
}
if len(vl.Versions) == 0 {
return -1
}
return 0
}
// findDevice returns whether the device is in InvalidVersions or Versions and
// in InvalidDevices or Devices (true for invalid), the positions in the version
// and device slices and whether it has been found at all.
func vlFindDevice(vl *dbproto.VersionList, device []byte) (bool, int, int, bool) {
for i, v := range vl.Versions {
if j := deviceIndex(v.Devices, device); j != -1 {
return false, i, j, true
}
if j := deviceIndex(v.InvalidDevices, device); j != -1 {
return true, i, j, true
}
}
return false, -1, -1, false
}
func vlPopVersionAt(vl *dbproto.VersionList, i int) {
vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
}
// checkInsertAt determines if the given device and associated file should be
// inserted into the FileVersion at position i or into a new FileVersion at
// position i.
func vlCheckInsertAt(vl *dbproto.VersionList, i int, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (bool, error) {
fv := vl.Versions[i]
ordering := protocol.VectorFromWire(fv.Version).Compare(file.FileVersion())
if ordering == protocol.Equal {
if !file.IsInvalid() {
fv.Devices = append(fv.Devices, device)
} else {
fv.InvalidDevices = append(fv.InvalidDevices, device)
}
return true, nil
}
existingDevice, _ := fvFirstDevice(fv)
insert, err := shouldInsertBefore(ordering, folder, existingDevice, fvIsInvalid(fv), file, t)
if err != nil {
return false, err
}
if insert {
vlInsertAt(vl, i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
return true, nil
}
return false, nil
}
// shouldInsertBefore determines whether the file comes before an existing
// entry, given the version ordering (existing compared to new one), existing
// device and if the existing version is invalid.
func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileInfo, t readOnlyTransaction) (bool, error) {
switch ordering {
case protocol.Lesser:
// The version at this point in the list is lesser
// ("older") than us. We insert ourselves in front of it.
return true, nil
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
// The version in conflict with us.
// Check if we can shortcut due to one being invalid.
if existingInvalid != file.IsInvalid() {
return existingInvalid, nil
}
// We must pull the actual file metadata to determine who wins.
// If we win, we insert ourselves in front of the loser here.
// (The "Lesser" and "Greater" in the condition above is just
// based on the device IDs in the version vector, which is not
// the only thing we use to determine the winner.)
of, ok, err := t.getFile(folder, existingDevice, []byte(file.FileName()))
if err != nil {
return false, err
}
// A surprise missing file entry here is counted as a win for us.
if !ok {
return true, nil
}
if file.WinsConflict(of) {
return true, nil
}
}
return false, nil
}
func deviceIndex(devices [][]byte, device []byte) int {
for i, dev := range devices {
if bytes.Equal(device, dev) {
return i
}
}
return -1
}
func popDeviceAt(devices [][]byte, i int) [][]byte {
return append(devices[:i], devices[i+1:]...)
}
func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) *dbproto.FileVersion {
fv := &dbproto.FileVersion{
Version: version.ToWire(),
Deleted: deleted,
}
if invalid {
fv.InvalidDevices = [][]byte{device}
} else {
fv.Devices = [][]byte{device}
}
return fv
}
func fvFirstDevice(fv *dbproto.FileVersion) ([]byte, bool) {
if len(fv.Devices) != 0 {
return fv.Devices[0], true
}
if len(fv.InvalidDevices) != 0 {
return fv.InvalidDevices[0], true
}
return nil, false
}
func fvIsInvalid(fv *dbproto.FileVersion) bool {
return fv == nil || len(fv.Devices) == 0
}
func fvDeviceCount(fv *dbproto.FileVersion) int {
return len(fv.Devices) + len(fv.InvalidDevices)
}
func fvCopy(fv *dbproto.FileVersion) *dbproto.FileVersion {
return proto.Clone(fv).(*dbproto.FileVersion)
}
lib/db/transactions.go
-1008
View File
File diff suppressed because it is too large Load Diff
lib/db/util_test.go
-232
View File
@@ -1,232 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/json"
"errors"
"io"
"os"
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
)
// writeJSONS serializes the database to a JSON stream that can be checked
// in to the repo and used for tests.
func writeJSONS(w io.Writer, db backend.Backend) {
it, err := db.NewPrefixIterator(nil)
if err != nil {
panic(err)
}
defer it.Release()
enc := json.NewEncoder(w)
for it.Next() {
err := enc.Encode(map[string][]byte{
"k": it.Key(),
"v": it.Value(),
})
if err != nil {
panic(err)
}
}
}
// we know this function isn't generally used, nonetheless we want it in
// here and the linter to not complain.
var _ = writeJSONS
// openJSONS reads a JSON stream file into a backend DB
func openJSONS(file string) (backend.Backend, error) {
fd, err := os.Open(file)
if err != nil {
return nil, err
}
dec := json.NewDecoder(fd)
db := backend.OpenMemory()
for {
var row map[string][]byte
err := dec.Decode(&row)
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
if err := db.Put(row["k"], row["v"]); err != nil {
return nil, err
}
}
return db, nil
}
func newLowlevel(t testing.TB, backend backend.Backend) *Lowlevel {
t.Helper()
ll, err := NewLowlevel(backend, events.NoopLogger)
if err != nil {
t.Fatal(err)
}
return ll
}
func newLowlevelMemory(t testing.TB) *Lowlevel {
return newLowlevel(t, backend.OpenMemory())
}
func newFileSet(t testing.TB, folder string, db *Lowlevel) *FileSet {
t.Helper()
fset, err := NewFileSet(folder, db)
if err != nil {
t.Fatal(err)
}
return fset
}
func snapshot(t testing.TB, fset *FileSet) *Snapshot {
t.Helper()
snap, err := fset.Snapshot()
if err != nil {
t.Fatal(err)
}
return snap
}
// The following commented tests were used to generate jsons files to stdout for
// future tests and are kept here for reference (reuse).
// TestGenerateIgnoredFilesDB generates a database with files with invalid flags,
// local and remote, in the format used in 0.14.48.
// func TestGenerateIgnoredFilesDB(t *testing.T) {
// db := OpenMemory()
// fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
// { // invalid (ignored) file
// Name: "foo",
// Type: protocol.FileInfoTypeFile,
// Invalid: true,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
// },
// { // regular file
// Name: "bar",
// Type: protocol.FileInfoTypeFile,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
// },
// })
// fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
// { // invalid file
// Name: "baz",
// Type: protocol.FileInfoTypeFile,
// Invalid: true,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
// },
// { // regular file
// Name: "quux",
// Type: protocol.FileInfoTypeFile,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
// },
// })
// writeJSONS(os.Stdout, db.DB)
// }
// TestGenerateUpdate0to3DB generates a database with files with invalid flags, prefixed
// by a slash and other files to test database migration from version 0 to 3, in the
// format used in 0.14.45.
// func TestGenerateUpdate0to3DB(t *testing.T) {
// db := OpenMemory()
// fs := newFileSet(t, update0to3Folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// for devID, files := range haveUpdate0to3 {
// fs.Update(devID, files)
// }
// writeJSONS(os.Stdout, db.DB)
// }
// func TestGenerateUpdateTo10(t *testing.T) {
// db := newLowlevelMemory(t)
// defer db.Close()
// if err := UpdateSchema(db); err != nil {
// t.Fatal(err)
// }
// fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
// files := []protocol.FileInfo{
// {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Deleted: true, Sequence: 1},
// {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2), Sequence: 2},
// {Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Deleted: true, Sequence: 3},
// }
// fs.Update(protocol.LocalDeviceID, files)
// files[1].Version = files[1].Version.Update(remoteDevice0.Short())
// files[1].Deleted = true
// files[2].Version = files[2].Version.Update(remoteDevice0.Short())
// files[2].Blocks = genBlocks(1)
// files[2].Deleted = false
// fs.Update(remoteDevice0, files)
// fd, err := os.Create("./testdata/v1.4.0-updateTo10.json")
// if err != nil {
// panic(err)
// }
// defer fd.Close()
// writeJSONS(fd, db)
// }
func TestFileInfoBatchError(t *testing.T) {
// Verify behaviour of the flush function returning an error.
var errReturn error
var called int
b := NewFileInfoBatch(func([]protocol.FileInfo) error {
called += 1
return errReturn
})
// Flush should work when the flush function error is nil
b.Append(protocol.FileInfo{Name: "test"})
if err := b.Flush(); err != nil {
t.Fatalf("expected nil, got %v", err)
}
if called != 1 {
t.Fatalf("expected 1, got %d", called)
}
// Flush should fail with an error retur
errReturn = errors.New("problem")
b.Append(protocol.FileInfo{Name: "test"})
if err := b.Flush(); err != errReturn {
t.Fatalf("expected %v, got %v", errReturn, err)
}
if called != 2 {
t.Fatalf("expected 2, got %d", called)
}
// Flush function should not be called again when it's already errored,
// same error should be returned by Flush()
if err := b.Flush(); err != errReturn {
t.Fatalf("expected %v, got %v", errReturn, err)
}
if called != 2 {
t.Fatalf("expected 2, got %d", called)
}
// Reset should clear the error (and the file list)
errReturn = nil
b.Reset()
b.Append(protocol.FileInfo{Name: "test"})
if err := b.Flush(); err != nil {
t.Fatalf("expected nil, got %v", err)
}
if called != 3 {
t.Fatalf("expected 3, got %d", called)
}
}
lib/fs/filesystem_test.go
+2 -2
View File
@@ -189,7 +189,7 @@ func TestRepro9677MissingMtimeFS(t *testing.T) {
testTime := time.Unix(1723491493, 123456789)
// Create a file with an mtime FS entry
firstFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB))
firstFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB, ""))
// Create a file, set its mtime and check that we get the expected mtime when stat-ing.
file, err := firstFS.Create(name)
@@ -231,6 +231,6 @@ func TestRepro9677MissingMtimeFS(t *testing.T) {
// be without mtime, even if requested:
NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{})
newFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB))
newFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB, ""))
checkMtime(newFS)
}
lib/fs/mtimefs.go
+30 -76
View File
@@ -7,21 +7,21 @@
package fs
import (
"errors"
"time"
)
// The database is where we store the virtual mtimes
type database interface {
Bytes(key string) (data []byte, ok bool, err error)
PutBytes(key string, data []byte) error
Delete(key string) error
GetMtime(folder, name string) (ondisk, virtual time.Time)
PutMtime(folder, name string, ondisk, virtual time.Time) error
DeleteMtime(folder, name string) error
}
type mtimeFS struct {
Filesystem
chtimes func(string, time.Time, time.Time) error
db database
folderID string
caseInsensitive bool
}
@@ -34,16 +34,18 @@ func WithCaseInsensitivity(v bool) MtimeFSOption {
}
type optionMtime struct {
db database
options []MtimeFSOption
db database
folderID string
options []MtimeFSOption
}
// NewMtimeOption makes any filesystem provide nanosecond mtime precision,
// regardless of what shenanigans the underlying filesystem gets up to.
func NewMtimeOption(db database, options ...MtimeFSOption) Option {
func NewMtimeOption(db database, folderID string, options ...MtimeFSOption) Option {
return &optionMtime{
db: db,
options: options,
db: db,
folderID: folderID,
options: options,
}
}
@@ -52,6 +54,7 @@ func (o *optionMtime) apply(fs Filesystem) Filesystem {
Filesystem: fs,
chtimes: fs.Chtimes, // for mocking it out in the tests
db: o.db,
folderID: o.folderID,
}
for _, opt := range o.options {
opt(f)
@@ -84,14 +87,11 @@ func (f *mtimeFS) Stat(name string) (FileInfo, error) {
return nil, err
}
mtimeMapping, err := f.load(name)
if err != nil {
return nil, err
}
if mtimeMapping.Real.Equal(info.ModTime()) {
ondisk, virtual := f.load(name)
if ondisk.Equal(info.ModTime()) {
info = mtimeFileInfo{
FileInfo: info,
mtime: mtimeMapping.Virtual,
mtime: virtual,
}
}
@@ -104,14 +104,11 @@ func (f *mtimeFS) Lstat(name string) (FileInfo, error) {
return nil, err
}
mtimeMapping, err := f.load(name)
if err != nil {
return nil, err
}
if mtimeMapping.Real.Equal(info.ModTime()) {
ondisk, virtual := f.load(name)
if ondisk.Equal(info.ModTime()) {
info = mtimeFileInfo{
FileInfo: info,
mtime: mtimeMapping.Virtual,
mtime: virtual,
}
}
@@ -150,43 +147,27 @@ func (*mtimeFS) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeMtime
}
func (f *mtimeFS) save(name string, real, virtual time.Time) {
func (f *mtimeFS) save(name string, ondisk, virtual time.Time) {
if f.caseInsensitive {
name = UnicodeLowercaseNormalized(name)
}
if real.Equal(virtual) {
if ondisk.Equal(virtual) {
// If the virtual time and the real on disk time are equal we don't
// need to store anything.
f.db.Delete(name)
_ = f.db.DeleteMtime(f.folderID, name)
return
}
mtime := MtimeMapping{
Real: real,
Virtual: virtual,
}
bs, _ := mtime.Marshal() // Can't fail
f.db.PutBytes(name, bs)
_ = f.db.PutMtime(f.folderID, name, ondisk, virtual)
}
func (f *mtimeFS) load(name string) (MtimeMapping, error) {
func (f *mtimeFS) load(name string) (ondisk, virtual time.Time) {
if f.caseInsensitive {
name = UnicodeLowercaseNormalized(name)
}
data, exists, err := f.db.Bytes(name)
if err != nil {
return MtimeMapping{}, err
} else if !exists {
return MtimeMapping{}, nil
}
var mtime MtimeMapping
if err := mtime.Unmarshal(data); err != nil {
return MtimeMapping{}, err
}
return mtime, nil
return f.db.GetMtime(f.folderID, name)
}
// The mtimeFileInfo is an os.FileInfo that lies about the ModTime().
@@ -211,14 +192,11 @@ func (f mtimeFile) Stat() (FileInfo, error) {
return nil, err
}
mtimeMapping, err := f.fs.load(f.Name())
if err != nil {
return nil, err
}
if mtimeMapping.Real.Equal(info.ModTime()) {
ondisk, virtual := f.fs.load(f.Name())
if ondisk.Equal(info.ModTime()) {
info = mtimeFileInfo{
FileInfo: info,
mtime: mtimeMapping.Virtual,
mtime: virtual,
}
}
@@ -230,38 +208,14 @@ func (f mtimeFile) unwrap() File {
return f.File
}
// MtimeMapping represents the mapping as stored in the database
type MtimeMapping struct {
// "Real" is the on disk timestamp
Real time.Time `json:"real"`
// "Virtual" is what want the timestamp to be
Virtual time.Time `json:"virtual"`
}
func (t *MtimeMapping) Marshal() ([]byte, error) {
bs0, _ := t.Real.MarshalBinary()
bs1, _ := t.Virtual.MarshalBinary()
return append(bs0, bs1...), nil
}
func (t *MtimeMapping) Unmarshal(bs []byte) error {
if err := t.Real.UnmarshalBinary(bs[:len(bs)/2]); err != nil {
return err
}
if err := t.Virtual.UnmarshalBinary(bs[len(bs)/2:]); err != nil {
return err
}
return nil
}
func GetMtimeMapping(fs Filesystem, file string) (MtimeMapping, error) {
func GetMtimeMapping(fs Filesystem, file string) (ondisk, virtual time.Time) {
fs, ok := unwrapFilesystem(fs, filesystemWrapperTypeMtime)
if !ok {
return MtimeMapping{}, errors.New("failed to unwrap")
return time.Time{}, time.Time{}
}
mtimeFs, ok := fs.(*mtimeFS)
if !ok {
return MtimeMapping{}, errors.New("unwrapping failed")
return time.Time{}, time.Time{}
}
return mtimeFs.load(file)
}
lib/fs/mtimefs_test.go
+9 -9
View File
@@ -226,20 +226,20 @@ func TestMtimeFSInsensitive(t *testing.T) {
// The mapStore is a simple database
type mapStore map[string][]byte
type mapStore map[string][2]time.Time
func (s mapStore) PutBytes(key string, data []byte) error {
s[key] = data
func (s mapStore) PutMtime(_, name string, real, virtual time.Time) error {
s[name] = [2]time.Time{real, virtual}
return nil
}
func (s mapStore) Bytes(key string) (data []byte, ok bool, err error) {
data, ok = s[key]
return
func (s mapStore) GetMtime(_, name string) (real, virtual time.Time) {
v := s[name]
return v[0], v[1]
}
func (s mapStore) Delete(key string) error {
delete(s, key)
func (s mapStore) DeleteMtime(_, name string) error {
delete(s, name)
return nil
}
@@ -260,7 +260,7 @@ func newMtimeFS(path string, db database, options ...MtimeFSOption) *mtimeFS {
}
func newMtimeFSWithWalk(path string, db database, options ...MtimeFSOption) (*mtimeFS, *walkFilesystem) {
fs := NewFilesystem(FilesystemTypeBasic, path, NewMtimeOption(db, options...))
fs := NewFilesystem(FilesystemTypeBasic, path, NewMtimeOption(db, "", options...))
wfs, _ := unwrapFilesystem(fs, filesystemWrapperTypeWalk)
mfs, _ := unwrapFilesystem(fs, filesystemWrapperTypeMtime)
return mfs.(*mtimeFS), wfs.(*walkFilesystem)
lib/locations/locations.go
+33 -27
View File
@@ -22,17 +22,18 @@ type LocationEnum string
// Use strings as keys to make printout and serialization of the locations map
// more meaningful.
const (
ConfigFile LocationEnum = "config"
CertFile LocationEnum = "certFile"
KeyFile LocationEnum = "keyFile"
HTTPSCertFile LocationEnum = "httpsCertFile"
HTTPSKeyFile LocationEnum = "httpsKeyFile"
Database LocationEnum = "database"
LogFile LocationEnum = "logFile"
PanicLog LocationEnum = "panicLog"
AuditLog LocationEnum = "auditLog"
GUIAssets LocationEnum = "guiAssets"
DefFolder LocationEnum = "defFolder"
ConfigFile LocationEnum = "config"
CertFile LocationEnum = "certFile"
KeyFile LocationEnum = "keyFile"
HTTPSCertFile LocationEnum = "httpsCertFile"
HTTPSKeyFile LocationEnum = "httpsKeyFile"
LegacyDatabase LocationEnum = "legacyDatabase"
Database LocationEnum = "database"
LogFile LocationEnum = "logFile"
PanicLog LocationEnum = "panicLog"
AuditLog LocationEnum = "auditLog"
GUIAssets LocationEnum = "guiAssets"
DefFolder LocationEnum = "defFolder"
)
type BaseDirEnum string
@@ -46,14 +47,15 @@ const (
// User's home directory, *not* --home flag
UserHomeBaseDir BaseDirEnum = "userHome"
LevelDBDir = "index-v0.14.0.db"
levelDBDir = "index-v0.14.0.db"
databaseName = "index-v2.db"
configFileName = "config.xml"
defaultStateDir = ".local/state/syncthing"
oldDefaultConfigDir = ".config/syncthing"
)
// Platform dependent directories
var baseDirs = make(map[BaseDirEnum]string, 3)
var baseDirs = make(map[BaseDirEnum]string)
func init() {
userHome := userHomeDir()
@@ -113,17 +115,18 @@ func GetBaseDir(baseDir BaseDirEnum) string {
// Use the variables from baseDirs here
var locationTemplates = map[LocationEnum]string{
ConfigFile: "${config}/config.xml",
CertFile: "${config}/cert.pem",
KeyFile: "${config}/key.pem",
HTTPSCertFile: "${config}/https-cert.pem",
HTTPSKeyFile: "${config}/https-key.pem",
Database: "${data}/" + LevelDBDir,
LogFile: "${data}/syncthing.log", // --logfile on Windows
PanicLog: "${data}/panic-%{timestamp}.log",
AuditLog: "${data}/audit-%{timestamp}.log",
GUIAssets: "${config}/gui",
DefFolder: "${userHome}/Sync",
ConfigFile: "${config}/config.xml",
CertFile: "${config}/cert.pem",
KeyFile: "${config}/key.pem",
HTTPSCertFile: "${config}/https-cert.pem",
HTTPSKeyFile: "${config}/https-key.pem",
LegacyDatabase: "${data}/" + levelDBDir,
Database: "${data}/" + databaseName,
LogFile: "${data}/syncthing.log", // --logfile on Windows
PanicLog: "${data}/panic-%{timestamp}.log",
AuditLog: "${data}/audit-%{timestamp}.log",
GUIAssets: "${config}/gui",
DefFolder: "${userHome}/Sync",
}
var locations = make(map[LocationEnum]string)
@@ -242,7 +245,8 @@ func unixDataDir(userHome, configDir, xdgDataHome, xdgStateHome string, fileExis
// If a database exists at the config location, use that. This is the
// most common case for both legacy (~/.config/syncthing) and current
// (~/.local/state/syncthing) setups.
if fileExists(filepath.Join(configDir, LevelDBDir)) {
if fileExists(filepath.Join(configDir, databaseName)) ||
fileExists(filepath.Join(configDir, levelDBDir)) {
return configDir
}
@@ -251,14 +255,16 @@ func unixDataDir(userHome, configDir, xdgDataHome, xdgStateHome string, fileExis
// but that's not what we did previously, so we retain the old behavior.
if xdgDataHome != "" {
candidate := filepath.Join(xdgDataHome, "syncthing")
if fileExists(filepath.Join(candidate, LevelDBDir)) {
if fileExists(filepath.Join(candidate, databaseName)) ||
fileExists(filepath.Join(candidate, levelDBDir)) {
return candidate
}
}
// Legacy: if a database exists under ~/.config/syncthing, use that
candidate := filepath.Join(userHome, oldDefaultConfigDir)
if fileExists(filepath.Join(candidate, LevelDBDir)) {
if fileExists(filepath.Join(candidate, databaseName)) ||
fileExists(filepath.Join(candidate, levelDBDir)) {
return candidate
}
lib/model/fakeconns_test.go
+3 -11
View File
@@ -12,6 +12,7 @@ import (
"sync"
"time"
"github.com/syncthing/syncthing/internal/timeutil"
"github.com/syncthing/syncthing/lib/protocol"
protocolmocks "github.com/syncthing/syncthing/lib/protocol/mocks"
"github.com/syncthing/syncthing/lib/rand"
@@ -81,7 +82,7 @@ func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol
Name: name,
Type: ftype,
Version: version,
Sequence: time.Now().UnixNano(),
Sequence: timeutil.StrictlyMonotonicNanos(),
LocalFlags: localFlags,
}
switch ftype {
@@ -108,15 +109,6 @@ func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol
f.fileData[name] = data
}
func (f *fakeConnection) addFileWithLocalFlags(name string, ftype protocol.FileInfoType, localFlags uint32) {
f.mut.Lock()
defer f.mut.Unlock()
var version protocol.Vector
version = version.Update(f.id.Short())
f.addFileLocked(name, 0, ftype, nil, version, localFlags)
}
func (f *fakeConnection) addFile(name string, flags uint32, ftype protocol.FileInfoType, data []byte) {
f.mut.Lock()
defer f.mut.Unlock()
@@ -148,7 +140,7 @@ func (f *fakeConnection) deleteFile(name string) {
fi.Deleted = true
fi.ModifiedS = time.Now().Unix()
fi.Version = fi.Version.Update(f.id.Short())
fi.Sequence = time.Now().UnixNano()
fi.Sequence = timeutil.StrictlyMonotonicNanos()
fi.Blocks = nil
f.files = append(append(f.files[:i], f.files[i+1:]...), fi)
lib/db/util.go → lib/model/fileinfobatch.go
+1 -1
View File
@@ -4,7 +4,7 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
package model
import (
"github.com/syncthing/syncthing/lib/protocol"
lib/model/fileinfobatch_test.go
+64
View File
@@ -0,0 +1,64 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package model
import (
"errors"
"testing"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestFileInfoBatchError(t *testing.T) {
// Verify behaviour of the flush function returning an error.
var errReturn error
var called int
b := NewFileInfoBatch(func([]protocol.FileInfo) error {
called += 1
return errReturn
})
// Flush should work when the flush function error is nil
b.Append(protocol.FileInfo{Name: "test"})
if err := b.Flush(); err != nil {
t.Fatalf("expected nil, got %v", err)
}
if called != 1 {
t.Fatalf("expected 1, got %d", called)
}
// Flush should fail with an error retur
errReturn = errors.New("problem")
b.Append(protocol.FileInfo{Name: "test"})
if err := b.Flush(); err != errReturn {
t.Fatalf("expected %v, got %v", errReturn, err)
}
if called != 2 {
t.Fatalf("expected 2, got %d", called)
}
// Flush function should not be called again when it's already errored,
// same error should be returned by Flush()
if err := b.Flush(); err != errReturn {
t.Fatalf("expected %v, got %v", errReturn, err)
}
if called != 2 {
t.Fatalf("expected 2, got %d", called)
}
// Reset should clear the error (and the file list)
errReturn = nil
b.Reset()
b.Append(protocol.FileInfo{Name: "test"})
if err := b.Flush(); err != nil {
t.Fatalf("expected nil, got %v", err)
}
if called != 3 {
t.Fatalf("expected 3, got %d", called)
}
}
lib/model/folder.go
+131 -118
View File
@@ -15,8 +15,9 @@ import (
"sort"
"time"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
@@ -46,7 +47,7 @@ type folder struct {
model *model
shortID protocol.ShortID
fset *db.FileSet
db db.DB
ignores *ignore.Matcher
mtimefs fs.Filesystem
modTimeWindow time.Duration
@@ -96,18 +97,18 @@ type puller interface {
pull() (bool, error) // true when successful and should not be retried
}
func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger, ioLimiter *semaphore.Semaphore, ver versioner.Versioner) folder {
func newFolder(model *model, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger, ioLimiter *semaphore.Semaphore, ver versioner.Versioner) folder {
f := folder{
stateTracker: newStateTracker(cfg.ID, evLogger),
FolderConfiguration: cfg,
FolderStatisticsReference: stats.NewFolderStatisticsReference(model.db, cfg.ID),
FolderStatisticsReference: stats.NewFolderStatisticsReference(db.NewTyped(model.sdb, "folderstats/"+cfg.ID)),
ioLimiter: ioLimiter,
model: model,
shortID: model.shortID,
fset: fset,
db: model.sdb,
ignores: ignores,
mtimefs: cfg.Filesystem(fset),
mtimefs: cfg.Filesystem(fs.NewMtimeOption(model.sdb, cfg.ID)),
modTimeWindow: cfg.ModTimeWindow(),
done: make(chan struct{}),
@@ -367,17 +368,11 @@ func (f *folder) pull() (success bool, err error) {
}()
// If there is nothing to do, don't even enter sync-waiting state.
abort := true
snap, err := f.dbSnapshot()
needCount, err := f.db.CountNeed(f.folderID, protocol.LocalDeviceID)
if err != nil {
return false, err
}
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileInfo) bool {
abort = false
return false
})
snap.Release()
if abort {
if needCount.TotalItems() == 0 {
// Clears pull failures on items that were needed before, but aren't anymore.
f.errorsMut.Lock()
f.pullErrors = nil
@@ -484,15 +479,10 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// Clean the list of subitems to ensure that we start at a known
// directory, and don't scan subdirectories of things we've already
// scanned.
snap, err := f.dbSnapshot()
if err != nil {
return err
}
subDirs = unifySubs(subDirs, func(file string) bool {
_, ok := snap.Get(protocol.LocalDeviceID, file)
return ok
_, ok, err := f.db.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file)
return err == nil && ok
})
snap.Release()
f.setState(FolderScanning)
f.clearScanErrors(subDirs)
@@ -546,7 +536,7 @@ const maxToRemove = 1000
type scanBatch struct {
f *folder
updateBatch *db.FileInfoBatch
updateBatch *FileInfoBatch
toRemove []string
}
@@ -555,7 +545,7 @@ func (f *folder) newScanBatch() *scanBatch {
f: f,
toRemove: make([]string, 0, maxToRemove),
}
b.updateBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
b.updateBatch = NewFileInfoBatch(func(fs []protocol.FileInfo) error {
if err := b.f.getHealthErrorWithoutIgnores(); err != nil {
l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err)
return err
@@ -570,46 +560,56 @@ func (b *scanBatch) Remove(item string) {
b.toRemove = append(b.toRemove, item)
}
func (b *scanBatch) flushToRemove() {
func (b *scanBatch) flushToRemove() error {
if len(b.toRemove) > 0 {
b.f.fset.RemoveLocalItems(b.toRemove)
if err := b.f.db.DropFilesNamed(b.f.folderID, protocol.LocalDeviceID, b.toRemove); err != nil {
return err
}
b.toRemove = b.toRemove[:0]
}
return nil
}
func (b *scanBatch) Flush() error {
b.flushToRemove()
if err := b.flushToRemove(); err != nil {
return err
}
return b.updateBatch.Flush()
}
func (b *scanBatch) FlushIfFull() error {
if len(b.toRemove) >= maxToRemove {
b.flushToRemove()
if err := b.flushToRemove(); err != nil {
return err
}
}
return b.updateBatch.FlushIfFull()
}
// Update adds the fileinfo to the batch for updating, and does a few checks.
// It returns false if the checks result in the file not going to be updated or removed.
func (b *scanBatch) Update(fi protocol.FileInfo, snap *db.Snapshot) bool {
func (b *scanBatch) Update(fi protocol.FileInfo) (bool, error) {
// Check for a "virtual" parent directory of encrypted files. We don't track
// it, but check if anything still exists within and delete it otherwise.
if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
if names, err := b.f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 {
b.f.mtimefs.Remove(fi.Name)
}
return false
return false, nil
}
// Resolve receive-only items which are identical with the global state or
// the global item is our own receive-only item.
switch gf, ok := snap.GetGlobal(fi.Name); {
switch gf, ok, err := b.f.db.GetGlobalFile(b.f.folderID, fi.Name); {
case err != nil:
return false, err
case !ok:
case gf.IsReceiveOnlyChanged():
if fi.IsDeleted() {
// Our item is deleted and the global item is our own receive only
// file. No point in keeping track of that.
b.Remove(fi.Name)
return true
l.Debugf("%v scanning: deleting deleted receive-only local-changed file: %v", b.f, fi)
return true, nil
}
case (b.f.Type == config.FolderTypeReceiveOnly || b.f.Type == config.FolderTypeReceiveEncrypted) &&
gf.IsEquivalentOptional(fi, protocol.FileInfoComparison{
@@ -621,20 +621,15 @@ func (b *scanBatch) Update(fi protocol.FileInfo, snap *db.Snapshot) bool {
IgnoreXattrs: !b.f.SyncXattrs && !b.f.SendXattrs,
}):
// What we have locally is equivalent to the global file.
l.Debugf("%v scanning: Merging identical locally changed item with global", b.f, fi)
l.Debugf("%v scanning: Merging identical locally changed item with global: %v", b.f, fi)
fi = gf
}
b.updateBatch.Append(fi)
return true
return true, nil
}
func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (int, error) {
changes := 0
snap, err := f.dbSnapshot()
if err != nil {
return changes, err
}
defer snap.Release()
// If we return early e.g. due to a folder health error, the scan needs
// to be cancelled.
@@ -646,7 +641,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i
Subs: subDirs,
Matcher: f.ignores,
TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour,
CurrentFiler: cFiler{snap},
CurrentFiler: cFiler{db: f.db, folder: f.folderID},
Filesystem: f.mtimefs,
IgnorePerms: f.IgnorePerms,
AutoNormalize: f.AutoNormalize,
@@ -683,15 +678,19 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i
return changes, err
}
if batch.Update(res.File, snap) {
if ok, err := batch.Update(res.File); err != nil {
return 0, err
} else if ok {
changes++
}
switch f.Type {
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
default:
if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok {
if batch.Update(nf, snap) {
if nf, ok := f.findRename(res.File, alreadyUsedOrExisting); ok {
if ok, err := batch.Update(nf); err != nil {
return 0, err
} else if ok {
changes++
}
}
@@ -705,25 +704,22 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
var toIgnore []protocol.FileInfo
ignoredParent := ""
changes := 0
snap, err := f.dbSnapshot()
if err != nil {
return 0, err
}
defer snap.Release()
outer:
for _, sub := range subDirs {
var iterError error
for fi, err := range itererr.Zip(f.db.AllLocalFilesWithPrefix(f.folderID, protocol.LocalDeviceID, sub)) {
if err != nil {
return changes, err
}
snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi protocol.FileInfo) bool {
select {
case <-f.ctx.Done():
return false
break outer
default:
}
if err := batch.FlushIfFull(); err != nil {
iterError = err
return false
return 0, err
}
if ignoredParent != "" && !fs.IsParent(fi.Name, ignoredParent) {
@@ -731,12 +727,13 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
l.Debugln("marking file as ignored", file)
nf := file
nf.SetIgnored()
if batch.Update(nf, snap) {
if ok, err := batch.Update(nf); err != nil {
return 0, err
} else if ok {
changes++
}
if err := batch.FlushIfFull(); err != nil {
iterError = err
return false
return 0, err
}
}
toIgnore = toIgnore[:0]
@@ -745,7 +742,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
switch ignored := f.ignores.Match(fi.Name).IsIgnored(); {
case fi.IsIgnored() && ignored:
return true
continue
case !fi.IsIgnored() && ignored:
// File was not ignored at last pass but has been ignored.
if fi.IsDirectory() {
@@ -756,13 +753,15 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
// this path as the "highest" ignored parent
ignoredParent = fi.Name
}
return true
continue
}
l.Debugln("marking file as ignored", fi)
nf := fi
nf.SetIgnored()
if batch.Update(nf, snap) {
if ok, err := batch.Update(nf); err != nil {
return 0, err
} else if ok {
changes++
}
@@ -781,7 +780,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
toIgnore = toIgnore[:0]
ignoredParent = ""
}
return true
continue
}
nf := fi
nf.SetDeleted(f.shortID)
@@ -793,13 +792,17 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
nf.Version = protocol.Vector{}
}
l.Debugln("marking file as deleted", nf)
if batch.Update(nf, snap) {
if ok, err := batch.Update(nf); err != nil {
return 0, err
} else if ok {
changes++
}
case fi.IsDeleted() && fi.IsReceiveOnlyChanged():
switch f.Type {
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
switch gf, ok := snap.GetGlobal(fi.Name); {
switch gf, ok, err := f.db.GetGlobalFile(f.folderID, fi.Name); {
case err != nil:
return 0, err
case !ok:
case gf.IsReceiveOnlyChanged():
l.Debugln("removing deleted, receive-only item that is globally receive-only from db", fi)
@@ -810,7 +813,9 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
// pretend it is a normal deleted file (nobody cares about that).
l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, fi.Name)
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
if batch.Update(fi, snap) {
if ok, err := batch.Update(fi); err != nil {
return 0, err
} else if ok {
changes++
}
}
@@ -819,14 +824,14 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
// deleted and just the folder type/local flags changed.
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
l.Debugln("removing receive-only flag on deleted item", fi)
if batch.Update(fi, snap) {
if ok, err := batch.Update(fi); err != nil {
return 0, err
} else if ok {
changes++
}
}
}
return true
})
}
select {
case <-f.ctx.Done():
@@ -834,30 +839,28 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
default:
}
if iterError == nil && len(toIgnore) > 0 {
if len(toIgnore) > 0 {
for _, file := range toIgnore {
l.Debugln("marking file as ignored", file)
nf := file
nf.SetIgnored()
if batch.Update(nf, snap) {
if ok, err := batch.Update(nf); err != nil {
return 0, err
} else if ok {
changes++
}
if iterError = batch.FlushIfFull(); iterError != nil {
break
if err := batch.FlushIfFull(); err != nil {
return 0, err
}
}
toIgnore = toIgnore[:0]
}
if iterError != nil {
return changes, iterError
}
}
return changes, nil
}
func (f *folder) findRename(snap *db.Snapshot, file protocol.FileInfo, alreadyUsedOrExisting map[string]struct{}) (protocol.FileInfo, bool) {
func (f *folder) findRename(file protocol.FileInfo, alreadyUsedOrExisting map[string]struct{}) (protocol.FileInfo, bool) {
if len(file.Blocks) == 0 || file.Size == 0 {
return protocol.FileInfo{}, false
}
@@ -865,49 +868,58 @@ func (f *folder) findRename(snap *db.Snapshot, file protocol.FileInfo, alreadyUs
found := false
nf := protocol.FileInfo{}
snap.WithBlocksHash(file.BlocksHash, func(fi protocol.FileInfo) bool {
loop:
for fi, err := range itererr.Zip(f.db.AllLocalFilesWithBlocksHash(f.folderID, file.BlocksHash)) {
if err != nil {
return protocol.FileInfo{}, false
}
select {
case <-f.ctx.Done():
return false
break loop
default:
}
if fi.Name == file.Name {
alreadyUsedOrExisting[fi.Name] = struct{}{}
return true
continue
}
if _, ok := alreadyUsedOrExisting[fi.Name]; ok {
return true
continue
}
if fi.ShouldConflict() {
return true
continue
}
if f.ignores.Match(fi.Name).IsIgnored() {
return true
continue
}
// Only check the size.
// No point checking block equality, as that uses BlocksHash comparison if that is set (which it will be).
// No point checking BlocksHash comparison as WithBlocksHash already does that.
if file.Size != fi.Size {
return true
continue
}
alreadyUsedOrExisting[fi.Name] = struct{}{}
if !osutil.IsDeleted(f.mtimefs, fi.Name) {
return true
continue
}
nf = fi
var ok bool
nf, ok, err = f.db.GetDeviceFile(f.folderID, protocol.LocalDeviceID, fi.Name)
if err != nil || !ok || nf.Sequence != fi.Sequence {
continue
}
nf.SetDeleted(f.shortID)
nf.LocalFlags = f.localFlags
found = true
return false
})
break
}
return nf, found
}
@@ -1216,20 +1228,26 @@ func (f *folder) ScheduleForceRescan(path string) {
}
}
func (f *folder) updateLocalsFromScanning(fs []protocol.FileInfo) {
f.updateLocals(fs)
func (f *folder) updateLocalsFromScanning(fs []protocol.FileInfo) error {
if err := f.updateLocals(fs); err != nil {
return err
}
f.emitDiskChangeEvents(fs, events.LocalChangeDetected)
return nil
}
func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) {
f.updateLocals(fs)
func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) error {
if err := f.updateLocals(fs); err != nil {
return err
}
f.emitDiskChangeEvents(fs, events.RemoteChangeDetected)
return nil
}
func (f *folder) updateLocals(fs []protocol.FileInfo) {
f.fset.Update(protocol.LocalDeviceID, fs)
func (f *folder) updateLocals(fs []protocol.FileInfo) error {
if err := f.db.Update(f.folderID, protocol.LocalDeviceID, fs); err != nil {
return err
}
filenames := make([]string, len(fs))
f.forcedRescanPathsMut.Lock()
@@ -1240,7 +1258,10 @@ func (f *folder) updateLocals(fs []protocol.FileInfo) {
}
f.forcedRescanPathsMut.Unlock()
seq := f.fset.Sequence(protocol.LocalDeviceID)
seq, err := f.db.GetDeviceSequence(f.folderID, protocol.LocalDeviceID)
if err != nil {
return err
}
f.evLogger.Log(events.LocalIndexUpdated, map[string]interface{}{
"folder": f.ID,
"items": len(fs),
@@ -1248,6 +1269,7 @@ func (f *folder) updateLocals(fs []protocol.FileInfo) {
"sequence": seq,
"version": seq, // legacy for sequence
})
return nil
}
func (f *folder) emitDiskChangeEvents(fs []protocol.FileInfo, typeOfEvent events.EventType) {
@@ -1294,23 +1316,19 @@ func (f *folder) handleForcedRescans() error {
return nil
}
batch := db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
f.fset.Update(protocol.LocalDeviceID, fs)
return nil
batch := NewFileInfoBatch(func(fs []protocol.FileInfo) error {
return f.db.Update(f.folderID, protocol.LocalDeviceID, fs)
})
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
for _, path := range paths {
if err := batch.FlushIfFull(); err != nil {
return err
}
fi, ok := snap.Get(protocol.LocalDeviceID, path)
fi, ok, err := f.db.GetDeviceFile(f.folderID, protocol.LocalDeviceID, path)
if err != nil {
return err
}
if !ok {
continue
}
@@ -1318,23 +1336,13 @@ func (f *folder) handleForcedRescans() error {
batch.Append(fi)
}
if err = batch.Flush(); err != nil {
if err := batch.Flush(); err != nil {
return err
}
return f.scanSubdirs(paths)
}
// dbSnapshots gets a snapshot from the fileset, and wraps any error
// in a svcutil.FatalErr.
func (f *folder) dbSnapshot() (*db.Snapshot, error) {
snap, err := f.fset.Snapshot()
if err != nil {
return nil, svcutil.AsFatalErr(err, svcutil.ExitError)
}
return snap, nil
}
// The exists function is expected to return true for all known paths
// (excluding "" and ".")
func unifySubs(dirs []string, exists func(dir string) bool) []string {
@@ -1370,10 +1378,15 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string {
}
type cFiler struct {
*db.Snapshot
db db.DB
folder string
}
// Implements scanner.CurrentFiler
func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
return cf.Get(protocol.LocalDeviceID, file)
fi, ok, err := cf.db.GetDeviceFile(cf.folder, protocol.LocalDeviceID, file)
if err != nil || !ok {
return protocol.FileInfo{}, false
}
return fi, true
}
lib/model/folder_recvenc.go
+17 -25
View File
@@ -10,8 +10,8 @@ import (
"fmt"
"sort"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
@@ -28,8 +28,8 @@ type receiveEncryptedFolder struct {
*sendReceiveFolder
}
func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
f := &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
func newReceiveEncryptedFolder(model *model, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
f := &receiveEncryptedFolder{newSendReceiveFolder(model, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
f.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return f
}
@@ -44,30 +44,27 @@ func (f *receiveEncryptedFolder) revert() error {
f.setState(FolderScanning)
defer f.setState(FolderIdle)
batch := db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
batch := NewFileInfoBatch(func(fs []protocol.FileInfo) error {
f.updateLocalsFromScanning(fs)
return nil
})
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
var iterErr error
var dirs []string
snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
if iterErr = batch.FlushIfFull(); iterErr != nil {
return false
for fi, err := range itererr.Zip(f.db.AllLocalFiles(f.folderID, protocol.LocalDeviceID)) {
if err != nil {
return err
}
if err := batch.FlushIfFull(); err != nil {
return err
}
if !fi.IsReceiveOnlyChanged() || fi.IsDeleted() {
return true
continue
}
if fi.IsDirectory() {
dirs = append(dirs, fi.Name)
return true
continue
}
if err := f.inWritableDir(f.mtimefs.Remove, fi.Name); err != nil && !fs.IsNotExist(err) {
@@ -84,15 +81,10 @@ func (f *receiveEncryptedFolder) revert() error {
// deleted, it will not show up as an unexpected file in the UI
// anymore.
batch.Append(fi)
return true
})
f.revertHandleDirs(dirs, snap)
if iterErr != nil {
return iterErr
}
f.revertHandleDirs(dirs)
if err := batch.Flush(); err != nil {
return err
}
@@ -103,7 +95,7 @@ func (f *receiveEncryptedFolder) revert() error {
return nil
}
func (f *receiveEncryptedFolder) revertHandleDirs(dirs []string, snap *db.Snapshot) {
func (f *receiveEncryptedFolder) revertHandleDirs(dirs []string) {
if len(dirs) == 0 {
return
}
@@ -114,7 +106,7 @@ func (f *receiveEncryptedFolder) revertHandleDirs(dirs []string, snap *db.Snapsh
sort.Sort(sort.Reverse(sort.StringSlice(dirs)))
for _, dir := range dirs {
if err := f.deleteDirOnDisk(dir, snap, scanChan); err != nil {
if err := f.deleteDirOnDisk(dir, scanChan); err != nil {
f.newScanError(dir, fmt.Errorf("deleting unexpected dir: %w", err))
}
scanChan <- dir
lib/model/folder_recvonly.go
+30 -29
View File
@@ -10,8 +10,8 @@ import (
"sort"
"time"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
@@ -57,8 +57,8 @@ type receiveOnlyFolder struct {
*sendReceiveFolder
}
func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)
func newReceiveOnlyFolder(model *model, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
sr := newSendReceiveFolder(model, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)
sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return &receiveOnlyFolder{sr}
}
@@ -83,30 +83,31 @@ func (f *receiveOnlyFolder) revert() error {
scanChan: scanChan,
}
batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
batch := NewFileInfoBatch(func(files []protocol.FileInfo) error {
f.updateLocalsFromScanning(files)
return nil
})
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
for fi, err := range itererr.Zip(f.db.AllLocalFiles(f.folderID, protocol.LocalDeviceID)) {
if err != nil {
return err
}
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in
// receive only mode.
return true
continue
}
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
switch gf, ok := snap.GetGlobal(fi.Name); {
switch gf, ok, err := f.db.GetGlobalFile(f.folderID, fi.Name); {
case err != nil:
return err
case !ok:
msg := "Unexpected global file that we have locally"
msg := "Unexpectedly missing global file that we have locally"
l.Debugf("%v revert: %v: %v", f, msg, fi.Name)
f.evLogger.Log(events.Failure, msg)
return true
continue
case gf.IsReceiveOnlyChanged():
// The global file is our own. A revert then means to delete it.
// We'll delete files directly, directories get queued and
@@ -115,13 +116,13 @@ func (f *receiveOnlyFolder) revert() error {
fi.Version = protocol.Vector{} // if this file ever resurfaces anywhere we want our delete to be strictly older
break
}
handled, err := delQueue.handle(fi, snap)
l.Debugf("Revert: deleting %s: %v\n", fi.Name, err)
handled, err := delQueue.handle(fi)
if err != nil {
l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
return true // continue
continue
}
if !handled {
return true // continue
continue
}
fi.SetDeleted(f.shortID)
fi.Version = protocol.Vector{} // if this file ever resurfaces anywhere we want our delete to be strictly older
@@ -144,13 +145,13 @@ func (f *receiveOnlyFolder) revert() error {
batch.Append(fi)
_ = batch.FlushIfFull()
return true
})
_ = batch.Flush()
}
if err := batch.Flush(); err != nil {
return err
}
// Handle any queued directories
deleted, err := delQueue.flush(snap)
deleted, err := delQueue.flush()
if err != nil {
l.Infoln("Revert:", err)
}
@@ -179,15 +180,15 @@ func (f *receiveOnlyFolder) revert() error {
// directories for last.
type deleteQueue struct {
handler interface {
deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error
deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error
deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) error
deleteDirOnDisk(dir string, scanChan chan<- string) error
}
ignores *ignore.Matcher
dirs []string
scanChan chan<- string
}
func (q *deleteQueue) handle(fi protocol.FileInfo, snap *db.Snapshot) (bool, error) {
func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
// Things that are ignored but not marked deletable are not processed.
ign := q.ignores.Match(fi.Name)
if ign.IsIgnored() && !ign.IsDeletable() {
@@ -201,11 +202,11 @@ func (q *deleteQueue) handle(fi protocol.FileInfo, snap *db.Snapshot) (bool, err
}
// Kill it.
err := q.handler.deleteItemOnDisk(fi, snap, q.scanChan)
err := q.handler.deleteItemOnDisk(fi, q.scanChan)
return true, err
}
func (q *deleteQueue) flush(snap *db.Snapshot) ([]string, error) {
func (q *deleteQueue) flush() ([]string, error) {
// Process directories from the leaves inward.
sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))
@@ -213,7 +214,7 @@ func (q *deleteQueue) flush(snap *db.Snapshot) ([]string, error) {
var deleted []string
for _, dir := range q.dirs {
if err := q.handler.deleteDirOnDisk(dir, snap, q.scanChan); err == nil {
if err := q.handler.deleteDirOnDisk(dir, q.scanChan); err == nil {
deleted = append(deleted, dir)
} else if firstError == nil {
firstError = err
lib/model/folder_recvonly_test.go
+64 -58
View File
@@ -13,6 +13,7 @@ import (
"testing"
"time"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
@@ -28,7 +29,7 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
m, f, wcfgCancel := setupROFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
defer cleanupModel(m)
conn := addFakeConn(m, device1, f.ID)
@@ -46,9 +47,11 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
// Send and index update for the known stuff
must(t, m.Index(conn, &protocol.Index{Folder: "ro", Files: knownFiles}))
f.updateLocalsFromScanning(knownFiles)
if err := f.updateLocalsFromScanning(knownFiles); err != nil {
t.Fatal(err)
}
size := globalSize(t, m, "ro")
size := mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
@@ -59,15 +62,15 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
// We should now have two files and two directories, with global state unchanged.
size = globalSize(t, m, "ro")
size = mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
}
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
@@ -92,11 +95,11 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
// We should now have one file and directory again.
size = globalSize(t, m, "ro")
size = mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
}
@@ -110,7 +113,7 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
m, f, wcfgCancel := setupROFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
defer cleanupModel(m)
conn := addFakeConn(m, device1, f.ID)
@@ -131,19 +134,19 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
// Everything should be in sync.
size := globalSize(t, m, "ro")
size := mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
@@ -159,20 +162,20 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
// We now have a newer file than the rest of the cluster. Global state should reflect this.
size = globalSize(t, m, "ro")
size = mustV(m.GlobalSize("ro"))
const sizeOfDir = 128
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
t.Fatalf("Global: expected no change due to the new file: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the new file to be reflected: %+v", size)
}
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
@@ -181,15 +184,15 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
m.Revert("ro")
size = globalSize(t, m, "ro")
size = mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
t.Fatalf("Global: expected the global size to revert: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the local size to remain: %+v", size)
}
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Bytes != int64(len(oldData)) {
t.Fatalf("Local: expected to need the old file data: %+v", size)
}
@@ -200,7 +203,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
m, f, wcfgCancel := setupROFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
defer cleanupModel(m)
conn := addFakeConn(m, device1, f.ID)
@@ -221,19 +224,19 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
// Everything should be in sync.
size := globalSize(t, m, "ro")
size := mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
@@ -246,7 +249,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
must(t, m.ScanFolder("ro"))
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files != 2 {
t.Fatalf("Receive only: expected 2 files: %+v", size)
}
@@ -259,7 +262,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
must(t, m.ScanFolder("ro"))
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories+size.Deleted != 0 {
t.Fatalf("Receive only: expected all zero: %+v", size)
}
@@ -270,7 +273,7 @@ func TestRecvOnlyDeletedRemoteDrop(t *testing.T) {
m, f, wcfgCancel := setupROFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
defer cleanupModel(m)
conn := addFakeConn(m, device1, f.ID)
@@ -291,19 +294,19 @@ func TestRecvOnlyDeletedRemoteDrop(t *testing.T) {
// Everything should be in sync.
size := globalSize(t, m, "ro")
size := mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
@@ -314,17 +317,17 @@ func TestRecvOnlyDeletedRemoteDrop(t *testing.T) {
must(t, m.ScanFolder("ro"))
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Deleted != 1 {
t.Fatalf("Receive only: expected 1 deleted: %+v", size)
}
// Drop the remote
f.fset.Drop(device1)
f.db.DropAllFiles("ro", device1)
must(t, m.ScanFolder("ro"))
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Deleted != 0 {
t.Fatalf("Receive only: expected no deleted: %+v", size)
}
@@ -335,7 +338,7 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) {
m, f, wcfgCancel := setupROFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
defer cleanupModel(m)
conn := addFakeConn(m, device1, f.ID)
@@ -356,19 +359,19 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) {
// Everything should be in sync.
size := globalSize(t, m, "ro")
size := mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
@@ -382,7 +385,7 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) {
must(t, m.ScanFolder("ro"))
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files != 2 {
t.Fatalf("Receive only: expected 2 files: %+v", size)
}
@@ -390,17 +393,17 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) {
// Do the same changes on the remote
files := make([]protocol.FileInfo, 0, 2)
snap := fsetSnapshot(t, f.fset)
snap.WithHave(protocol.LocalDeviceID, func(f protocol.FileInfo) bool {
for f, err := range itererr.Zip(f.db.AllLocalFiles("ro", protocol.LocalDeviceID)) {
if err != nil {
t.Fatal(err)
}
if f.Name != file && f.Name != knownFile {
return true
continue
}
f.LocalFlags = 0
f.Version = protocol.Vector{}.Update(device1.Short())
files = append(files, f)
return true
})
snap.Release()
}
must(t, m.IndexUpdate(conn, &protocol.IndexUpdate{Folder: "ro", Files: files}))
// Ensure the pull to resolve conflicts (content identical) happened
@@ -409,7 +412,10 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) {
return nil
}))
size = receiveOnlyChangedSize(t, m, "ro")
size, err := m.ReceiveOnlySize("ro")
if err != nil {
t.Fatal(err)
}
if size.Files+size.Directories+size.Deleted != 0 {
t.Fatalf("Receive only: expected all zero: %+v", size)
}
@@ -424,7 +430,7 @@ func TestRecvOnlyRevertOwnID(t *testing.T) {
m, f, wcfgCancel := setupROFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
defer cleanupModel(m)
conn := addFakeConn(m, device1, f.ID)
@@ -484,7 +490,7 @@ func TestRecvOnlyLocalChangeDoesNotCauseConflict(t *testing.T) {
m, f, wcfgCancel := setupROFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
defer cleanupModel(m)
conn := addFakeConn(m, device1, f.ID)
@@ -505,19 +511,19 @@ func TestRecvOnlyLocalChangeDoesNotCauseConflict(t *testing.T) {
// Everything should be in sync.
size := globalSize(t, m, "ro")
size := mustV(m.GlobalSize("ro"))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = localSize(t, m, "ro")
size = mustV(m.LocalSize("ro", protocol.LocalDeviceID))
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
@@ -528,7 +534,7 @@ func TestRecvOnlyLocalChangeDoesNotCauseConflict(t *testing.T) {
must(t, m.ScanFolder("ro"))
size = receiveOnlyChangedSize(t, m, "ro")
size = mustV(m.ReceiveOnlySize("ro"))
if size.Files != 1 {
t.Fatalf("Receive only: expected 1 file: %+v", size)
}
@@ -541,7 +547,7 @@ func TestRecvOnlyLocalChangeDoesNotCauseConflict(t *testing.T) {
must(t, m.ScanFolder("ro"))
size = needSizeLocal(t, m, "ro")
size = mustV(m.NeedSize("ro", protocol.LocalDeviceID))
if size.Files != 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
@@ -577,7 +583,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
ModifiedS: fi.ModTime().Unix(),
ModifiedNs: int32(fi.ModTime().Nanosecond()),
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
Sequence: 42,
Sequence: 43,
Blocks: blocks,
},
}
lib/model/folder_sendonly.go
+40 -37
View File
@@ -7,8 +7,8 @@
package model
import (
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
@@ -24,9 +24,9 @@ type sendOnlyFolder struct {
folder
}
func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
func newSendOnlyFolder(model *model, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
f := &sendOnlyFolder{
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil),
folder: newFolder(model, ignores, cfg, evLogger, ioLimiter, nil),
}
f.folder.puller = f
return f
@@ -38,36 +38,36 @@ func (*sendOnlyFolder) PullErrors() []FileError {
// pull checks need for files that only differ by metadata (no changes on disk)
func (f *sendOnlyFolder) pull() (bool, error) {
batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
batch := NewFileInfoBatch(func(files []protocol.FileInfo) error {
f.updateLocalsFromPulling(files)
return nil
})
snap, err := f.dbSnapshot()
if err != nil {
return false, err
}
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(file protocol.FileInfo) bool {
batch.FlushIfFull()
for file, err := range itererr.Zip(f.db.AllNeededGlobalFiles(f.folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0)) {
if err != nil {
return false, err
}
if err := batch.FlushIfFull(); err != nil {
return false, err
}
if f.ignores.Match(file.FileName()).IsIgnored() {
file.SetIgnored()
batch.Append(file)
l.Debugln(f, "Handling ignored file", file)
return true
continue
}
curFile, ok := snap.Get(protocol.LocalDeviceID, file.FileName())
curFile, ok, err := f.db.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.FileName())
if err != nil {
return false, err
}
if !ok {
if file.IsInvalid() {
// Global invalid file just exists for need accounting
if file.IsInvalid() || file.IsDeleted() {
// Accept the file for accounting purposes
batch.Append(file)
} else if file.IsDeleted() {
l.Debugln("Should never get a deleted file as needed when we don't have it")
f.evLogger.Log(events.Failure, "got deleted file that doesn't exist locally as needed when pulling on send-only")
}
return true
continue
}
if !file.IsEquivalentOptional(curFile, protocol.FileInfoComparison{
@@ -76,14 +76,12 @@ func (f *sendOnlyFolder) pull() (bool, error) {
IgnoreOwnership: !f.SyncOwnership,
IgnoreXattrs: !f.SyncXattrs,
}) {
return true
continue
}
batch.Append(file)
l.Debugln(f, "Merging versions of identical file", file)
return true
})
}
batch.Flush()
@@ -100,25 +98,31 @@ func (f *sendOnlyFolder) override() error {
f.setState(FolderScanning)
defer f.setState(FolderIdle)
batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
batch := NewFileInfoBatch(func(files []protocol.FileInfo) error {
f.updateLocalsFromScanning(files)
return nil
})
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(need protocol.FileInfo) bool {
_ = batch.FlushIfFull()
have, ok := snap.Get(protocol.LocalDeviceID, need.Name)
for need, err := range itererr.Zip(f.db.AllNeededGlobalFiles(f.folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0)) {
if err != nil {
return err
}
if err := batch.FlushIfFull(); err != nil {
return err
}
have, haveOk, err := f.db.GetDeviceFile(f.folderID, protocol.LocalDeviceID, need.Name)
if err != nil {
return err
}
// Don't override files that are in a bad state (ignored,
// unsupported, must rescan, ...).
if ok && have.IsInvalid() {
return true
if haveOk && have.IsInvalid() {
continue
}
if !ok || have.Name != need.Name {
if !haveOk || have.Name != need.Name {
// We are missing the file
need.SetDeleted(f.shortID)
} else {
@@ -128,7 +132,6 @@ func (f *sendOnlyFolder) override() error {
}
need.Sequence = 0
batch.Append(need)
return true
})
}
return batch.Flush()
}
lib/model/folder_sendrecv.go
+147 -124
View File
@@ -19,9 +19,9 @@ import (
"strings"
"time"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
@@ -129,9 +129,9 @@ type sendReceiveFolder struct {
tempPullErrors map[string]string // pull errors that might be just transient
}
func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
func newSendReceiveFolder(model *model, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
f := &sendReceiveFolder{
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, ver),
folder: newFolder(model, ignores, cfg, evLogger, ioLimiter, ver),
queue: newJobQueue(),
blockPullReorderer: newBlockPullReorderer(cfg.BlockPullOrder, model.id, cfg.DeviceIDs()),
writeLimiter: semaphore.New(cfg.MaxConcurrentWrites),
@@ -240,12 +240,6 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) (int, error)
f.tempPullErrors = make(map[string]string)
f.errorsMut.Unlock()
snap, err := f.dbSnapshot()
if err != nil {
return 0, err
}
defer snap.Release()
pullChan := make(chan pullBlockState)
copyChan := make(chan copyBlocksState)
finisherChan := make(chan *sharedPullerState)
@@ -277,18 +271,18 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) (int, error)
pullWg.Add(1)
go func() {
// pullerRoutine finishes when pullChan is closed
f.pullerRoutine(snap, pullChan, finisherChan)
f.pullerRoutine(pullChan, finisherChan)
pullWg.Done()
}()
doneWg.Add(1)
// finisherRoutine finishes when finisherChan is closed
go func() {
f.finisherRoutine(snap, finisherChan, dbUpdateChan, scanChan)
f.finisherRoutine(finisherChan, dbUpdateChan, scanChan)
doneWg.Done()
}()
changed, fileDeletions, dirDeletions, err := f.processNeeded(snap, dbUpdateChan, copyChan, scanChan)
changed, fileDeletions, dirDeletions, err := f.processNeeded(dbUpdateChan, copyChan, scanChan)
// Signal copy and puller routines that we are done with the in data for
// this iteration. Wait for them to finish.
@@ -303,7 +297,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) (int, error)
doneWg.Wait()
if err == nil {
f.processDeletions(fileDeletions, dirDeletions, snap, dbUpdateChan, scanChan)
f.processDeletions(fileDeletions, dirDeletions, dbUpdateChan, scanChan)
}
// Wait for db updates and scan scheduling to complete
@@ -315,7 +309,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) (int, error)
return changed, err
}
func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
changed := 0
var dirDeletions []protocol.FileInfo
fileDeletions := map[string]protocol.FileInfo{}
@@ -325,16 +319,20 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
// Regular files to pull goes into the file queue, everything else
// (directories, symlinks and deletes) goes into the "process directly"
// pile.
snap.WithNeed(protocol.LocalDeviceID, func(file protocol.FileInfo) bool {
loop:
for file, err := range itererr.Zip(f.model.sdb.AllNeededGlobalFiles(f.folderID, protocol.LocalDeviceID, f.Order, 0, 0)) {
if err != nil {
return changed, nil, nil, err
}
select {
case <-f.ctx.Done():
return false
break loop
default:
}
if f.IgnoreDelete && file.IsDeleted() {
l.Debugln(f, "ignore file deletion (config)", file.FileName())
return true
continue
}
changed++
@@ -366,9 +364,12 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
// files to delete inside them before we get to that point.
dirDeletions = append(dirDeletions, file)
} else if file.IsSymlink() {
f.deleteFile(file, snap, dbUpdateChan, scanChan)
f.deleteFile(file, dbUpdateChan, scanChan)
} else {
df, ok := snap.Get(protocol.LocalDeviceID, file.Name)
df, ok, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
if err != nil {
return changed, nil, nil, err
}
// Local file can be already deleted, but with a lower version
// number, hence the deletion coming in again as part of
// WithNeed, furthermore, the file can simply be of the wrong
@@ -384,7 +385,10 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
}
case file.Type == protocol.FileInfoTypeFile:
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
curFile, hasCurFile, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
if err != nil {
return changed, nil, nil, err
}
if hasCurFile && file.BlocksEqual(curFile) {
// We are supposed to copy the entire file, and then fetch nothing. We
// are only updating metadata, so we don't actually *need* to make the
@@ -396,7 +400,7 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
}
case (build.IsWindows || build.IsAndroid) && file.IsSymlink():
if err := f.handleSymlinkCheckExisting(file, snap, scanChan); err != nil {
if err := f.handleSymlinkCheckExisting(file, scanChan); err != nil {
f.newPullError(file.Name, fmt.Errorf("handling unsupported symlink: %w", err))
break
}
@@ -407,22 +411,20 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
case file.IsDirectory() && !file.IsSymlink():
l.Debugln(f, "Handling directory", file.Name)
if f.checkParent(file.Name, scanChan) {
f.handleDir(file, snap, dbUpdateChan, scanChan)
f.handleDir(file, dbUpdateChan, scanChan)
}
case file.IsSymlink():
l.Debugln(f, "Handling symlink", file.Name)
if f.checkParent(file.Name, scanChan) {
f.handleSymlink(file, snap, dbUpdateChan, scanChan)
f.handleSymlink(file, dbUpdateChan, scanChan)
}
default:
l.Warnln(file)
panic("unhandleable item type, can't happen")
}
return true
})
}
select {
case <-f.ctx.Done():
@@ -430,23 +432,6 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
default:
}
// Now do the file queue. Reorder it according to configuration.
switch f.Order {
case config.PullOrderRandom:
f.queue.Shuffle()
case config.PullOrderAlphabetic:
// The queue is already in alphabetic order.
case config.PullOrderSmallestFirst:
f.queue.SortSmallestFirst()
case config.PullOrderLargestFirst:
f.queue.SortLargestFirst()
case config.PullOrderOldestFirst:
f.queue.SortOldestFirst()
case config.PullOrderNewestFirst:
f.queue.SortNewestFirst()
}
// Process the file queue.
nextFile:
@@ -462,7 +447,10 @@ nextFile:
break
}
fi, ok := snap.GetGlobal(fileName)
fi, ok, err := f.model.sdb.GetGlobalFile(f.folderID, fileName)
if err != nil {
return changed, nil, nil, err
}
if !ok {
// File is no longer in the index. Mark it as done and drop it.
f.queue.Done(fileName)
@@ -489,7 +477,7 @@ nextFile:
// desired state with the delete bit set is in the deletion
// map.
desired := fileDeletions[candidate.Name]
if err := f.renameFile(candidate, desired, fi, snap, dbUpdateChan, scanChan); err != nil {
if err := f.renameFile(candidate, desired, fi, dbUpdateChan, scanChan); err != nil {
l.Debugf("rename shortcut for %s failed: %s", fi.Name, err.Error())
// Failed to rename, try next one.
continue
@@ -502,9 +490,11 @@ nextFile:
continue nextFile
}
devices := f.model.fileAvailability(f.FolderConfiguration, snap, fi)
devices := f.model.fileAvailability(f.FolderConfiguration, fi)
if len(devices) > 0 {
f.handleFile(fi, snap, copyChan)
if err := f.handleFile(fi, copyChan); err != nil {
f.newPullError(fileName, err)
}
continue
}
f.newPullError(fileName, errNotAvailable)
@@ -524,7 +514,7 @@ func popCandidate(buckets map[string][]protocol.FileInfo, key string) (protocol.
return cands[0], true
}
func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
for _, file := range fileDeletions {
select {
case <-f.ctx.Done():
@@ -532,7 +522,7 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F
default:
}
f.deleteFile(file, snap, dbUpdateChan, scanChan)
f.deleteFile(file, dbUpdateChan, scanChan)
}
// Process in reverse order to delete depth first
@@ -545,12 +535,12 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F
dir := dirDeletions[len(dirDeletions)-i-1]
l.Debugln(f, "Deleting dir", dir.Name)
f.deleteDir(dir, snap, dbUpdateChan, scanChan)
f.deleteDir(dir, dbUpdateChan, scanChan)
}
}
// handleDir creates or updates the given directory
func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@@ -578,7 +568,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
}
if shouldDebug() {
curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
curFile, _, _ := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
}
@@ -589,7 +579,11 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
// that don't result in a conflict.
case err == nil && !info.IsDir():
// Check that it is what we have in the database.
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
curFile, hasCurFile, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
if err != nil {
f.newPullError(file.Name, fmt.Errorf("handling dir: %w", err))
return
}
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, false, scanChan); err != nil {
f.newPullError(file.Name, fmt.Errorf("handling dir: %w", err))
return
@@ -606,7 +600,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
}, curFile.Name)
} else {
err = f.deleteItemOnDisk(curFile, snap, scanChan)
err = f.deleteItemOnDisk(curFile, scanChan)
}
if err != nil {
f.newPullError(file.Name, err)
@@ -715,7 +709,7 @@ func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) boo
}
// handleSymlink creates or updates the given symlink
func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@@ -738,8 +732,8 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
}()
if shouldDebug() {
curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
l.Debugf("need symlink\n\t%v\n\t%v", file, curFile)
curFile, ok, _ := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
l.Debugf("need symlink\n\t%v\n\t%v", file, curFile, ok)
}
if len(file.SymlinkTarget) == 0 {
@@ -749,7 +743,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
return
}
if err = f.handleSymlinkCheckExisting(file, snap, scanChan); err != nil {
if err = f.handleSymlinkCheckExisting(file, scanChan); err != nil {
f.newPullError(file.Name, fmt.Errorf("handling symlink: %w", err))
return
}
@@ -770,7 +764,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
}
}
func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error {
func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, scanChan chan<- string) error {
// If there is already something under that name, we need to handle that.
info, err := f.mtimefs.Lstat(file.Name)
if err != nil {
@@ -780,7 +774,10 @@ func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, s
return err
}
// Check that it is what we have in the database.
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
curFile, hasCurFile, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
if err != nil {
return err
}
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, false, scanChan); err != nil {
return err
}
@@ -796,12 +793,12 @@ func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, s
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
}, curFile.Name)
} else {
return f.deleteItemOnDisk(curFile, snap, scanChan)
return f.deleteItemOnDisk(curFile, scanChan)
}
}
// deleteDir attempts to remove a directory that was deleted on a remote
func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@@ -826,7 +823,10 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot,
})
}()
cur, hasCur := snap.Get(protocol.LocalDeviceID, file.Name)
cur, hasCur, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
if err != nil {
return
}
if err = f.checkToBeDeleted(file, cur, hasCur, scanChan); err != nil {
if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
@@ -836,7 +836,7 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot,
return
}
if err = f.deleteDirOnDisk(file.Name, snap, scanChan); err != nil {
if err = f.deleteDirOnDisk(file.Name, scanChan); err != nil {
return
}
@@ -844,8 +844,12 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot,
}
// deleteFile attempts to delete the given file
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
cur, hasCur := snap.Get(protocol.LocalDeviceID, file.Name)
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
cur, hasCur, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
if err != nil {
f.newPullError(file.Name, fmt.Errorf("delete file: %w", err))
return
}
f.deleteFileWithCurrent(file, cur, hasCur, dbUpdateChan, scanChan)
}
@@ -924,7 +928,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
// renameFile attempts to rename an existing file to a destination
// and set the right attributes on it.
func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@@ -966,7 +970,10 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, sn
return err
}
// Check that the target corresponds to what we have in the DB
curTarget, ok := snap.Get(protocol.LocalDeviceID, target.Name)
curTarget, ok, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, target.Name)
if err != nil {
return err
}
switch stat, serr := f.mtimefs.Lstat(target.Name); {
case serr != nil:
var caseErr *fs.ErrCaseConflict
@@ -1040,7 +1047,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, sn
// of the source and the creation of the target temp file. Fix-up the metadata,
// update the local index of the target file and rename from temp to real name.
if err = f.performFinish(target, curTarget, true, tempName, snap, dbUpdateChan, scanChan); err != nil {
if err = f.performFinish(target, curTarget, true, tempName, dbUpdateChan, scanChan); err != nil {
return err
}
@@ -1085,8 +1092,11 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, sn
// handleFile queues the copies and pulls as necessary for a single new or
// changed file.
func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot, copyChan chan<- copyBlocksState) {
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState) error {
curFile, hasCurFile, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
if err != nil {
return err
}
have, _ := blockDiff(curFile.Blocks, file.Blocks)
@@ -1130,6 +1140,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot
have: len(have),
}
copyChan <- cs
return nil
}
func (f *sendReceiveFolder) reuseBlocks(blocks []protocol.BlockInfo, reused []int, file protocol.FileInfo, tempName string) ([]protocol.BlockInfo, []int) {
@@ -1284,7 +1295,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
// Hope that it's usually in the same folder, so start with that one.
folders := []string{f.folderID}
for folder, cfg := range f.model.cfg.Folders() {
folderFilesystems[folder] = cfg.Filesystem(nil)
folderFilesystems[folder] = cfg.Filesystem()
if folder != f.folderID {
folders = append(folders, folder)
}
@@ -1333,49 +1344,61 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
buf = protocol.BufferPool.Upgrade(buf, int(block.Size))
found := f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
ffs := folderFilesystems[folder]
fd, err := ffs.Open(path)
found := false
for e, err := range itererr.Zip(f.model.sdb.AllLocalBlocksWithHash(block.Hash)) {
if err != nil {
return false
break
}
defer fd.Close()
srcOffset := int64(state.file.BlockSize()) * int64(index)
_, err = fd.ReadAt(buf, srcOffset)
if err != nil {
return false
}
// Hash is not SHA256 as it's an encrypted hash token. In that
// case we can't verify the block integrity so we'll take it on
// trust. (The other side can and will verify.)
if f.Type != config.FolderTypeReceiveEncrypted {
if err := f.verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
return false
it, errFn := f.model.sdb.AllLocalFilesWithBlocksHashAnyFolder(e.BlocklistHash)
for folderID, fi := range it {
ffs := folderFilesystems[folderID]
fd, err := ffs.Open(fi.Name)
if err != nil {
continue
}
}
defer fd.Close()
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
err = f.withLimiter(func() error {
dstFd.mut.Lock()
defer dstFd.mut.Unlock()
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
})
} else {
err = f.limitedWriteAt(dstFd, buf, block.Offset)
_, err = fd.ReadAt(buf, e.Offset)
if err != nil {
fd.Close()
continue
}
// Hash is not SHA256 as it's an encrypted hash token. In that
// case we can't verify the block integrity so we'll take it on
// trust. (The other side can and will verify.)
if f.Type != config.FolderTypeReceiveEncrypted {
if err := f.verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
continue
}
}
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
err = f.withLimiter(func() error {
dstFd.mut.Lock()
defer dstFd.mut.Unlock()
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, e.Offset, block.Offset, int64(block.Size))
})
} else {
err = f.limitedWriteAt(dstFd, buf, block.Offset)
}
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
break
}
if fi.Name == state.file.Name {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromElsewhere(block.Size)
}
found = true
break
}
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
if err := errFn(); err != nil {
l.Warnln(err)
}
if path == state.file.Name {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromElsewhere(block.Size)
}
return true
})
}
if state.failed() != nil {
break
@@ -1410,7 +1433,7 @@ func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) err
return nil
}
func (f *sendReceiveFolder) pullerRoutine(snap *db.Snapshot, in <-chan pullBlockState, out chan<- *sharedPullerState) {
func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
requestLimiter := semaphore.New(f.PullerMaxPendingKiB * 1024)
wg := sync.NewWaitGroup()
@@ -1441,13 +1464,13 @@ func (f *sendReceiveFolder) pullerRoutine(snap *db.Snapshot, in <-chan pullBlock
defer wg.Done()
defer requestLimiter.Give(bytes)
f.pullBlock(state, snap, out)
f.pullBlock(state, out)
}()
}
wg.Wait()
}
func (f *sendReceiveFolder) pullBlock(state pullBlockState, snap *db.Snapshot, out chan<- *sharedPullerState) {
func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPullerState) {
// Get an fd to the temporary file. Technically we don't need it until
// after fetching the block, but if we run into an error here there is
// no point in issuing the request to the network.
@@ -1466,7 +1489,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, snap *db.Snapshot, o
}
var lastError error
candidates := f.model.blockAvailability(f.FolderConfiguration, snap, state.file, state.block)
candidates := f.model.blockAvailability(f.FolderConfiguration, state.file, state.block)
loop:
for {
select {
@@ -1531,7 +1554,7 @@ loop:
out <- state.sharedPullerState
}
func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
// Set the correct permission bits on the new file
if !f.IgnorePerms && !file.NoPermissions {
if err := f.mtimefs.Chmod(tempName, fs.FileMode(file.Permissions&0o777)); err != nil {
@@ -1562,7 +1585,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
}, curFile.Name)
} else {
err = f.deleteItemOnDisk(curFile, snap, scanChan)
err = f.deleteItemOnDisk(curFile, scanChan)
}
if err != nil {
return fmt.Errorf("moving for conflict: %w", err)
@@ -1585,7 +1608,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
return nil
}
func (f *sendReceiveFolder) finisherRoutine(snap *db.Snapshot, in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
for state := range in {
if closed, err := state.finalClose(); closed {
l.Debugln(f, "closing", state.file.Name)
@@ -1593,7 +1616,7 @@ func (f *sendReceiveFolder) finisherRoutine(snap *db.Snapshot, in <-chan *shared
f.queue.Done(state.file.Name)
if err == nil {
err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, snap, dbUpdateChan, scanChan)
err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, dbUpdateChan, scanChan)
}
if err != nil {
@@ -1646,7 +1669,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine(dbUpdateChan <-chan dbUpdateJob) {
var lastFile protocol.FileInfo
tick := time.NewTicker(maxBatchTime)
defer tick.Stop()
batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
batch := NewFileInfoBatch(func(files []protocol.FileInfo) error {
// sync directories
for dir := range changedDirs {
delete(changedDirs, dir)
@@ -1830,7 +1853,7 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
}
// deleteItemOnDisk deletes the file represented by old that is about to be replaced by new.
func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) (err error) {
func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) (err error) {
defer func() {
if err != nil {
err = fmt.Errorf("%s: %w", contextRemovingOldItem, err)
@@ -1841,7 +1864,7 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Sn
case item.IsDirectory():
// Directories aren't archived and need special treatment due
// to potential children.
return f.deleteDirOnDisk(item.Name, snap, scanChan)
return f.deleteDirOnDisk(item.Name, scanChan)
case !item.IsSymlink() && f.versioner != nil:
// If we should use versioning, let the versioner archive the
@@ -1857,12 +1880,12 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Sn
// deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside
// the directory and removes them if possible or returns an error if it fails
func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error {
func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string) error {
if err := osutil.TraversesSymlink(f.mtimefs, filepath.Dir(dir)); err != nil {
return err
}
if err := f.deleteDirOnDiskHandleChildren(dir, snap, scanChan); err != nil {
if err := f.deleteDirOnDiskHandleChildren(dir, scanChan); err != nil {
return err
}
@@ -1882,7 +1905,7 @@ func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanC
return err
}
func (f *sendReceiveFolder) deleteDirOnDiskHandleChildren(dir string, snap *db.Snapshot, scanChan chan<- string) error {
func (f *sendReceiveFolder) deleteDirOnDiskHandleChildren(dir string, scanChan chan<- string) error {
var dirsToDelete []string
var hasIgnored, hasKnown, hasToBeScanned, hasReceiveOnlyChanged bool
var delErr error
@@ -1909,7 +1932,7 @@ func (f *sendReceiveFolder) deleteDirOnDiskHandleChildren(dir string, snap *db.S
hasIgnored = true
return nil
}
cf, ok := snap.Get(protocol.LocalDeviceID, path)
cf, ok, err := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, path)
switch {
case !ok || cf.IsDeleted():
// Something appeared in the dir that we either are not
lib/model/folder_sendrecv_test.go
+56 -66
View File
@@ -19,6 +19,7 @@ import (
"testing"
"time"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
@@ -149,7 +150,7 @@ func TestHandleFile(t *testing.T) {
copyChan := make(chan copyBlocksState, 1)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(requiredFile, copyChan)
// Receive the results
toCopy := <-copyChan
@@ -189,13 +190,13 @@ func TestHandleFileWithTemp(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t, existingFile)
defer wcfgCancel()
if _, err := prepareTmpFile(f.Filesystem(nil)); err != nil {
if _, err := prepareTmpFile(f.Filesystem()); err != nil {
t.Fatal(err)
}
copyChan := make(chan copyBlocksState, 1)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(requiredFile, copyChan)
// Receive the results
toCopy := <-copyChan
@@ -239,7 +240,7 @@ func TestCopierFinder(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t, existingFile)
defer wcfgCancel()
if _, err := prepareTmpFile(f.Filesystem(nil)); err != nil {
if _, err := prepareTmpFile(f.Filesystem()); err != nil {
t.Fatal(err)
}
@@ -251,7 +252,7 @@ func TestCopierFinder(t *testing.T) {
go f.copierRoutine(copyChan, pullChan, finisherChan)
defer close(copyChan)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(requiredFile, copyChan)
timeout := time.After(10 * time.Second)
pulls := make([]pullBlockState, 4)
@@ -272,8 +273,9 @@ func TestCopierFinder(t *testing.T) {
defer cleanupSharedPullerState(finish)
select {
case <-pullChan:
t.Fatal("Pull channel has data to be read")
case v := <-pullChan:
t.Logf("%+v\n", v)
t.Fatal("Pull channel had data to be read")
case <-finisherChan:
t.Fatal("Finisher channel has data to be read")
default:
@@ -299,7 +301,7 @@ func TestCopierFinder(t *testing.T) {
}
// Verify that the fetched blocks have actually been written to the temp file
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil)
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(), tempFile, protocol.MinBlockSize, nil)
if err != nil {
t.Log(err)
}
@@ -313,10 +315,6 @@ func TestCopierFinder(t *testing.T) {
// Test that updating a file removes its old blocks from the blockmap
func TestCopierCleanup(t *testing.T) {
iterFn := func(folder, file string, index int32) bool {
return true
}
// Create a file
file := setupFile("test", []int{0})
file.Size = 1
@@ -328,11 +326,11 @@ func TestCopierCleanup(t *testing.T) {
// Update index (removing old blocks)
f.updateLocalsFromScanning([]protocol.FileInfo{file})
if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[0].Hash)); err != nil || len(vals) > 0 {
t.Error("Unexpected block found")
}
if !m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[1].Hash)); err != nil || len(vals) == 0 {
t.Error("Expected block not found")
}
@@ -341,11 +339,11 @@ func TestCopierCleanup(t *testing.T) {
// Update index (removing old blocks)
f.updateLocalsFromScanning([]protocol.FileInfo{file})
if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[0].Hash)); err != nil || len(vals) == 0 {
t.Error("Unexpected block found")
}
if m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[1].Hash)); err != nil || len(vals) > 0 {
t.Error("Expected block not found")
}
}
@@ -371,10 +369,9 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState, 1)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := fsetSnapshot(t, f.fset)
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
defer func() {
close(copyChan)
@@ -384,7 +381,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
close(finisherChan)
}()
f.handleFile(file, snap, copyChan)
f.handleFile(file, copyChan)
// Receive a block at puller, to indicate that at least a single copier
// loop has been performed.
@@ -471,16 +468,15 @@ func TestDeregisterOnFailInPull(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := fsetSnapshot(t, f.fset)
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
pullWg := sync.NewWaitGroup()
pullWg.Add(1)
go func() {
f.pullerRoutine(snap, pullChan, finisherBufferChan)
f.pullerRoutine(pullChan, finisherBufferChan)
pullWg.Done()
}()
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
defer func() {
// Unblock copier and puller
go func() {
@@ -495,7 +491,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
close(finisherChan)
}()
f.handleFile(file, snap, copyChan)
f.handleFile(file, copyChan)
// Receive at finisher, we should error out as puller has nowhere to pull
// from.
@@ -558,7 +554,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
func TestIssue3164(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
ignDir := filepath.Join("issue3164", "oktodelete")
subDir := filepath.Join(ignDir, "foobar")
@@ -577,7 +573,7 @@ func TestIssue3164(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteDir(file, fsetSnapshot(t, f.fset), dbUpdateChan, make(chan string))
f.deleteDir(file, dbUpdateChan, make(chan string))
if _, err := ffs.Stat("issue3164"); !fs.IsNotExist(err) {
t.Fatal(err)
@@ -648,7 +644,7 @@ func TestDiffEmpty(t *testing.T) {
func TestDeleteIgnorePerms(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
f.IgnorePerms = true
name := "deleteIgnorePerms"
@@ -692,9 +688,6 @@ func TestCopyOwner(t *testing.T) {
f.folder.FolderConfiguration = newFolderConfiguration(m.cfg, f.ID, f.Label, config.FilesystemTypeFake, "/TestCopyOwner")
f.folder.FolderConfiguration.CopyOwnershipFromParent = true
f.fset = newFileSet(t, f.ID, m.db)
f.mtimefs = f.Filesystem(f.fset)
// Create a parent dir with a certain owner/group.
f.mtimefs.Mkdir("foo", 0o755)
@@ -712,7 +705,7 @@ func TestCopyOwner(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string)
defer close(dbUpdateChan)
f.handleDir(dir, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.handleDir(dir, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan: // empty the channel for later
case toScan := <-scanChan:
@@ -742,17 +735,16 @@ func TestCopyOwner(t *testing.T) {
// but it's the way data is passed around. When the database update
// comes the finisher is done.
snap := fsetSnapshot(t, f.fset)
finisherChan := make(chan *sharedPullerState)
copierChan, copyWg := startCopier(f, nil, finisherChan)
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, nil)
go f.finisherRoutine(finisherChan, dbUpdateChan, nil)
defer func() {
close(copierChan)
copyWg.Wait()
close(finisherChan)
}()
f.handleFile(file, snap, copierChan)
f.handleFile(file, copierChan)
<-dbUpdateChan
info, err = f.mtimefs.Lstat("foo/bar/baz")
@@ -771,7 +763,7 @@ func TestCopyOwner(t *testing.T) {
SymlinkTarget: []byte("over the rainbow"),
}
f.handleSymlink(symlink, snap, dbUpdateChan, scanChan)
f.handleSymlink(symlink, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan:
case toScan := <-scanChan:
@@ -792,7 +784,7 @@ func TestCopyOwner(t *testing.T) {
func TestSRConflictReplaceFileByDir(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
@@ -810,7 +802,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleDir(file, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.handleDir(file, dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@@ -824,7 +816,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
func TestSRConflictReplaceFileByLink(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
@@ -843,7 +835,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleSymlink(file, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.handleSymlink(file, dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@@ -857,7 +849,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
func TestDeleteBehindSymlink(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
link := "link"
linkFile := filepath.Join(link, "file")
@@ -873,7 +865,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
fi.Version = fi.Version.Update(device1.Short())
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteFile(fi, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.deleteFile(fi, dbUpdateChan, scanChan)
select {
case f := <-scanChan:
t.Fatalf("Received %v on scanChan", f)
@@ -903,7 +895,7 @@ func TestPullCtxCancel(t *testing.T) {
var cancel context.CancelFunc
f.ctx, cancel = context.WithCancel(context.Background())
go f.pullerRoutine(fsetSnapshot(t, f.fset), pullChan, finisherChan)
go f.pullerRoutine(pullChan, finisherChan)
defer close(pullChan)
emptyState := func() pullBlockState {
@@ -938,7 +930,7 @@ func TestPullCtxCancel(t *testing.T) {
func TestPullDeleteUnscannedDir(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
dir := "foobar"
must(t, ffs.MkdirAll(dir, 0o777))
@@ -949,7 +941,7 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteDir(fi, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.deleteDir(fi, dbUpdateChan, scanChan)
if _, err := ffs.Stat(dir); fs.IsNotExist(err) {
t.Error("directory has been deleted")
@@ -967,7 +959,7 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
func TestPullCaseOnlyPerformFinish(t *testing.T) {
m, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
contents := []byte("contents")
@@ -976,16 +968,17 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) {
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
it, errFn := m.LocalFiles(f.ID, protocol.LocalDeviceID)
for i := range it {
if hasCur {
t.Fatal("got more than one file")
}
cur = i
hasCur = true
return true
})
}
if err := errFn(); err != nil {
t.Fatal(err)
}
if !hasCur {
t.Fatal("file is missing")
}
@@ -999,7 +992,7 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) {
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
err := f.performFinish(remote, cur, hasCur, temp, snap, dbUpdateChan, scanChan)
err := f.performFinish(remote, cur, hasCur, temp, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan: // boring case sensitive filesystem
@@ -1029,7 +1022,7 @@ func TestPullCaseOnlySymlink(t *testing.T) {
func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
m, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
if dir {
@@ -1041,16 +1034,17 @@ func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
must(t, f.scanSubdirs(nil))
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
it, errFn := m.LocalFiles(f.ID, protocol.LocalDeviceID)
for i := range it {
if hasCur {
t.Fatal("got more than one file")
}
cur = i
hasCur = true
return true
})
}
if err := errFn(); err != nil {
t.Fatal(err)
}
if !hasCur {
t.Fatal("file is missing")
}
@@ -1063,9 +1057,9 @@ func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
remote.Name = strings.ToUpper(cur.Name)
if dir {
f.handleDir(remote, snap, dbUpdateChan, scanChan)
f.handleDir(remote, dbUpdateChan, scanChan)
} else {
f.handleSymlink(remote, snap, dbUpdateChan, scanChan)
f.handleSymlink(remote, dbUpdateChan, scanChan)
}
select {
@@ -1100,7 +1094,7 @@ func TestPullTempFileCaseConflict(t *testing.T) {
fd.Close()
}
f.handleFile(file, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(file, copyChan)
cs := <-copyChan
if _, err := cs.tempFile(); err != nil {
@@ -1142,9 +1136,7 @@ func TestPullCaseOnlyRename(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 2)
scanChan := make(chan string, 2)
snap := fsetSnapshot(t, f.fset)
defer snap.Release()
if err := f.renameFile(cur, deleted, confl, snap, dbUpdateChan, scanChan); err != nil {
if err := f.renameFile(cur, deleted, confl, dbUpdateChan, scanChan); err != nil {
t.Error(err)
}
}
@@ -1219,9 +1211,7 @@ func TestPullDeleteCaseConflict(t *testing.T) {
t.Error("Missing db update for file")
}
snap := fsetSnapshot(t, f.fset)
defer snap.Release()
f.deleteDir(fi, snap, dbUpdateChan, scanChan)
f.deleteDir(fi, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan:
default:
@@ -1249,7 +1239,7 @@ func TestPullDeleteIgnoreChildDir(t *testing.T) {
scanChan := make(chan string, 2)
err := f.deleteDirOnDisk(parent, fsetSnapshot(t, f.fset), scanChan)
err := f.deleteDirOnDisk(parent, scanChan)
if err == nil {
t.Error("no error")
}
lib/model/folder_summary.go
+7 -11
View File
@@ -17,8 +17,8 @@ import (
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/svcutil"
@@ -127,16 +127,12 @@ func (c *folderSummaryService) Summary(folder string) (*FolderSummary, error) {
var remoteSeq map[protocol.DeviceID]int64
errors, err := c.model.FolderErrors(folder)
if err == nil {
var snap *db.Snapshot
if snap, err = c.model.DBSnapshot(folder); err == nil {
global = snap.GlobalSize()
local = snap.LocalSize()
need = snap.NeedSize(protocol.LocalDeviceID)
ro = snap.ReceiveOnlyChangedSize()
ourSeq = snap.Sequence(protocol.LocalDeviceID)
remoteSeq = snap.RemoteSequences()
snap.Release()
}
global, _ = c.model.GlobalSize(folder)
local, _ = c.model.LocalSize(folder, protocol.LocalDeviceID)
need, _ = c.model.NeedSize(folder, protocol.LocalDeviceID)
ro, _ = c.model.ReceiveOnlySize(folder)
ourSeq, _ = c.model.Sequence(folder, protocol.LocalDeviceID)
remoteSeq, _ = c.model.RemoteSequences(folder)
}
// For API backwards compatibility (SyncTrayzor needs it) an empty folder
// summary is returned for not running folders, an error might actually be
lib/model/indexhandler.go
+83 -77
View File
@@ -8,12 +8,14 @@ package model
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/svcutil"
@@ -45,13 +47,19 @@ type indexHandler struct {
cond *sync.Cond
paused bool
fset *db.FileSet
sdb db.DB
runner service
}
func newIndexHandler(conn protocol.Connection, downloads *deviceDownloadState, folder config.FolderConfiguration, fset *db.FileSet, runner service, startInfo *clusterConfigDeviceInfo, evLogger events.Logger) *indexHandler {
myIndexID := fset.IndexID(protocol.LocalDeviceID)
mySequence := fset.Sequence(protocol.LocalDeviceID)
func newIndexHandler(conn protocol.Connection, downloads *deviceDownloadState, folder config.FolderConfiguration, sdb db.DB, runner service, startInfo *clusterConfigDeviceInfo, evLogger events.Logger) (*indexHandler, error) {
myIndexID, err := sdb.GetIndexID(folder.ID, protocol.LocalDeviceID)
if err != nil {
return nil, err
}
mySequence, err := sdb.GetDeviceSequence(folder.ID, protocol.LocalDeviceID)
if err != nil {
return nil, err
}
var startSequence int64
// This is the other side's description of what it knows
@@ -91,14 +99,14 @@ func newIndexHandler(conn protocol.Connection, downloads *deviceDownloadState, f
// otherwise we drop our old index data and expect to get a
// completely new set.
theirIndexID := fset.IndexID(conn.DeviceID())
theirIndexID, _ := sdb.GetIndexID(folder.ID, conn.DeviceID())
if startInfo.remote.IndexID == 0 {
// They're not announcing an index ID. This means they
// do not support delta indexes and we should clear any
// information we have from them before accepting their
// index, which will presumably be a full index.
l.Debugf("Device %v folder %s does not announce an index ID", conn.DeviceID().Short(), folder.Description())
fset.Drop(conn.DeviceID())
sdb.DropAllFiles(folder.ID, conn.DeviceID())
} else if startInfo.remote.IndexID != theirIndexID {
// The index ID we have on file is not what they're
// announcing. They must have reset their database and
@@ -106,8 +114,8 @@ func newIndexHandler(conn protocol.Connection, downloads *deviceDownloadState, f
// information we have and remember this new index ID
// instead.
l.Infof("Device %v folder %s has a new index ID (%v)", conn.DeviceID().Short(), folder.Description(), startInfo.remote.IndexID)
fset.Drop(conn.DeviceID())
fset.SetIndexID(conn.DeviceID(), startInfo.remote.IndexID)
sdb.DropAllFiles(folder.ID, conn.DeviceID())
sdb.SetIndexID(folder.ID, conn.DeviceID(), startInfo.remote.IndexID)
}
return &indexHandler{
@@ -119,27 +127,27 @@ func newIndexHandler(conn protocol.Connection, downloads *deviceDownloadState, f
sentPrevSequence: startSequence,
evLogger: evLogger,
fset: fset,
sdb: sdb,
runner: runner,
cond: sync.NewCond(new(sync.Mutex)),
}
}, nil
}
// waitForFileset waits for the handler to resume and fetches the current fileset.
func (s *indexHandler) waitForFileset(ctx context.Context) (*db.FileSet, error) {
// waitWhilePaused waits for the handler to resume
func (s *indexHandler) waitWhilePaused(ctx context.Context) error {
s.cond.L.Lock()
defer s.cond.L.Unlock()
for s.paused {
select {
case <-ctx.Done():
return nil, ctx.Err()
return ctx.Err()
default:
s.cond.Wait()
}
}
return s.fset, nil
return nil
}
func (s *indexHandler) Serve(ctx context.Context) (err error) {
@@ -162,11 +170,10 @@ func (s *indexHandler) Serve(ctx context.Context) (err error) {
}()
// We need to send one index, regardless of whether there is something to send or not
fset, err := s.waitForFileset(ctx)
if err != nil {
if err := s.waitWhilePaused(ctx); err != nil {
return err
}
err = s.sendIndexTo(ctx, fset)
err = s.sendIndexTo(ctx)
// Subscribe to LocalIndexUpdated (we have new information to send) and
// DeviceDisconnected (it might be us who disconnected, so we should
@@ -179,8 +186,7 @@ func (s *indexHandler) Serve(ctx context.Context) (err error) {
defer ticker.Stop()
for err == nil {
fset, err = s.waitForFileset(ctx)
if err != nil {
if err := s.waitWhilePaused(ctx); err != nil {
return err
}
@@ -188,7 +194,11 @@ func (s *indexHandler) Serve(ctx context.Context) (err error) {
// currently in the database, wait for the local index to update. The
// local index may update for other folders than the one we are
// sending for.
if fset.Sequence(protocol.LocalDeviceID) <= s.localPrevSequence {
seq, err := s.sdb.GetDeviceSequence(s.folder, protocol.LocalDeviceID)
if err != nil {
return err
}
if seq <= s.localPrevSequence {
select {
case <-ctx.Done():
return ctx.Err()
@@ -198,7 +208,7 @@ func (s *indexHandler) Serve(ctx context.Context) (err error) {
continue
}
err = s.sendIndexTo(ctx, fset)
err = s.sendIndexTo(ctx)
// Wait a short amount of time before entering the next loop. If there
// are continuous changes happening to the local index, this gives us
@@ -215,10 +225,9 @@ func (s *indexHandler) Serve(ctx context.Context) (err error) {
// resume might be called because the folder was actually resumed, or just
// because the folder config changed (and thus the runner and potentially fset).
func (s *indexHandler) resume(fset *db.FileSet, runner service) {
func (s *indexHandler) resume(runner service) {
s.cond.L.Lock()
s.paused = false
s.fset = fset
s.runner = runner
s.cond.Broadcast()
s.cond.L.Unlock()
@@ -230,7 +239,6 @@ func (s *indexHandler) pause() {
s.evLogger.Log(events.Failure, "index handler got paused while already paused")
}
s.paused = true
s.fset = nil
s.runner = nil
s.cond.Broadcast()
s.cond.L.Unlock()
@@ -238,9 +246,9 @@ func (s *indexHandler) pause() {
// sendIndexTo sends file infos with a sequence number higher than prevSequence and
// returns the highest sent sequence number.
func (s *indexHandler) sendIndexTo(ctx context.Context, fset *db.FileSet) error {
func (s *indexHandler) sendIndexTo(ctx context.Context) error {
initial := s.localPrevSequence == 0
batch := db.NewFileInfoBatch(nil)
batch := NewFileInfoBatch(nil)
var batchError error
batch.SetFlushFunc(func(fs []protocol.FileInfo) error {
select {
@@ -284,21 +292,26 @@ func (s *indexHandler) sendIndexTo(ctx context.Context, fset *db.FileSet) error
return nil
})
var err error
var f protocol.FileInfo
snap, err := fset.Snapshot()
if err != nil {
return svcutil.AsFatalErr(err, svcutil.ExitError)
}
defer snap.Release()
previousWasDelete := false
snap.WithHaveSequence(s.localPrevSequence+1, func(fi protocol.FileInfo) bool {
t0 := time.Now()
for fi, err := range itererr.Zip(s.sdb.AllLocalFilesBySequence(s.folder, protocol.LocalDeviceID, s.localPrevSequence+1, 5000)) {
if err != nil {
return err
}
// This is to make sure that renames (which is an add followed by a delete) land in the same batch.
// Even if the batch is full, we allow a last delete to slip in, we do this by making sure that
// the batch ends with a non-delete, or that the last item in the batch is already a delete
if batch.Full() && (!fi.IsDeleted() || previousWasDelete) {
if err = batch.Flush(); err != nil {
return false
if err := batch.Flush(); err != nil {
return err
}
if time.Since(t0) > 5*time.Second {
// minor hack -- avoid very long running read transactions
// during index transmission, to help prevent excessive
// growth of database WAL file
break
}
}
@@ -307,6 +320,7 @@ func (s *indexHandler) sendIndexTo(ctx context.Context, fset *db.FileSet) error
"sequence": fi.SequenceNo(),
"start": s.localPrevSequence + 1,
})
return errors.New("database misbehaved")
}
if f.Sequence > 0 && fi.SequenceNo() <= f.Sequence {
@@ -315,27 +329,17 @@ func (s *indexHandler) sendIndexTo(ctx context.Context, fset *db.FileSet) error
"start": s.localPrevSequence + 1,
"previous": f.Sequence,
})
// Abort this round of index sending - the next one will pick
// up from the last successful one with the repeaired db.
defer func() {
if fixed, dbErr := fset.RepairSequence(); dbErr != nil {
l.Warnln("Failed repairing sequence entries:", dbErr)
panic("Failed repairing sequence entries")
} else {
s.evLogger.Log(events.Failure, "detected and repaired non-increasing sequence")
l.Infof("Repaired %v sequence entries in database", fixed)
}
}()
return false
return errors.New("database misbehaved")
}
f = fi
s.localPrevSequence = f.Sequence
// If this is a folder receiving encrypted files only, we
// mustn't ever send locally changed file infos. Those aren't
// encrypted and thus would be a protocol error at the remote.
if s.folderIsReceiveEncrypted && fi.IsReceiveOnlyChanged() {
return true
continue
}
f = prepareFileInfoForIndex(f)
@@ -343,23 +347,11 @@ func (s *indexHandler) sendIndexTo(ctx context.Context, fset *db.FileSet) error
previousWasDelete = f.IsDeleted()
batch.Append(f)
return true
})
if err != nil {
return err
}
if err := batch.Flush(); err != nil {
return err
}
// Use the sequence of the snapshot we iterated as a starting point for the
// next run. Previously we used the sequence of the last file we sent,
// however it's possible that a higher sequence exists, just doesn't need to
// be sent (e.g. in a receive-only folder, when a local change was
// reverted). No point trying to send nothing again.
s.localPrevSequence = snap.Sequence(protocol.LocalDeviceID)
return nil
}
@@ -368,7 +360,6 @@ func (s *indexHandler) receive(fs []protocol.FileInfo, update bool, op string, p
s.cond.L.Lock()
paused := s.paused
fset := s.fset
runner := s.runner
s.cond.L.Unlock()
@@ -382,13 +373,19 @@ func (s *indexHandler) receive(fs []protocol.FileInfo, update bool, op string, p
s.downloads.Update(s.folder, makeForgetUpdate(fs))
if !update {
fset.Drop(deviceID)
if err := s.sdb.DropAllFiles(s.folder, deviceID); err != nil {
return err
}
}
l.Debugf("Received %d files for %s from %s, prevSeq=%d, lastSeq=%d", len(fs), s.folder, deviceID.Short(), prevSequence, lastSequence)
// Verify that the previous sequence number matches what we expected
if exp := fset.Sequence(deviceID); prevSequence > 0 && prevSequence != exp {
exp, err := s.sdb.GetDeviceSequence(s.folder, deviceID)
if err != nil {
return err
}
if prevSequence > 0 && prevSequence != exp {
s.logSequenceAnomaly("index update with unexpected sequence", map[string]any{
"prevSeq": prevSequence,
"lastSeq": lastSequence,
@@ -444,8 +441,13 @@ func (s *indexHandler) receive(fs []protocol.FileInfo, update bool, op string, p
})
}
fset.Update(deviceID, fs)
seq := fset.Sequence(deviceID)
if err := s.sdb.Update(s.folder, deviceID, fs); err != nil {
return err
}
seq, err := s.sdb.GetDeviceSequence(s.folder, deviceID)
if err != nil {
return err
}
// Check that the sequence we get back is what we put in...
if lastSequence > 0 && len(fs) > 0 && seq != lastSequence {
@@ -508,6 +510,7 @@ func (s *indexHandler) String() string {
type indexHandlerRegistry struct {
evLogger events.Logger
conn protocol.Connection
sdb db.DB
downloads *deviceDownloadState
indexHandlers *serviceMap[string, *indexHandler]
startInfos map[string]*clusterConfigDeviceInfo
@@ -517,14 +520,14 @@ type indexHandlerRegistry struct {
type indexHandlerFolderState struct {
cfg config.FolderConfiguration
fset *db.FileSet
runner service
}
func newIndexHandlerRegistry(conn protocol.Connection, downloads *deviceDownloadState, evLogger events.Logger) *indexHandlerRegistry {
func newIndexHandlerRegistry(conn protocol.Connection, sdb db.DB, downloads *deviceDownloadState, evLogger events.Logger) *indexHandlerRegistry {
r := &indexHandlerRegistry{
evLogger: evLogger,
conn: conn,
sdb: sdb,
downloads: downloads,
indexHandlers: newServiceMap[string, *indexHandler](evLogger),
startInfos: make(map[string]*clusterConfigDeviceInfo),
@@ -544,15 +547,19 @@ func (r *indexHandlerRegistry) Serve(ctx context.Context) error {
return r.indexHandlers.Serve(ctx)
}
func (r *indexHandlerRegistry) startLocked(folder config.FolderConfiguration, fset *db.FileSet, runner service, startInfo *clusterConfigDeviceInfo) {
func (r *indexHandlerRegistry) startLocked(folder config.FolderConfiguration, runner service, startInfo *clusterConfigDeviceInfo) error {
r.indexHandlers.RemoveAndWait(folder.ID, 0)
delete(r.startInfos, folder.ID)
is := newIndexHandler(r.conn, r.downloads, folder, fset, runner, startInfo, r.evLogger)
is, err := newIndexHandler(r.conn, r.downloads, folder, r.sdb, runner, startInfo, r.evLogger)
if err != nil {
return err
}
r.indexHandlers.Add(folder.ID, is)
// This new connection might help us get in sync.
runner.SchedulePull()
return nil
}
// AddIndexInfo starts an index handler for given folder, unless it is paused.
@@ -572,7 +579,7 @@ func (r *indexHandlerRegistry) AddIndexInfo(folder string, startInfo *clusterCon
r.startInfos[folder] = startInfo
return
}
r.startLocked(folderState.cfg, folderState.fset, folderState.runner, startInfo)
_ = r.startLocked(folderState.cfg, folderState.runner, startInfo) // XXX error handling...
}
// Remove stops a running index handler or removes one pending to be started.
@@ -612,7 +619,7 @@ func (r *indexHandlerRegistry) RemoveAllExcept(except map[string]remoteFolderSta
// RegisterFolderState must be called whenever something about the folder
// changes. The exception being if the folder is removed entirely, then call
// Remove. The fset and runner arguments may be nil, if given folder is paused.
func (r *indexHandlerRegistry) RegisterFolderState(folder config.FolderConfiguration, fset *db.FileSet, runner service) {
func (r *indexHandlerRegistry) RegisterFolderState(folder config.FolderConfiguration, runner service) {
if !folder.SharedWith(r.conn.DeviceID()) {
r.Remove(folder.ID)
return
@@ -622,7 +629,7 @@ func (r *indexHandlerRegistry) RegisterFolderState(folder config.FolderConfigura
if folder.Paused {
r.folderPausedLocked(folder.ID)
} else {
r.folderRunningLocked(folder, fset, runner)
r.folderRunningLocked(folder, runner)
}
r.mut.Unlock()
}
@@ -643,10 +650,9 @@ func (r *indexHandlerRegistry) folderPausedLocked(folder string) {
// folderRunningLocked resumes an already running index handler or starts it, if it
// was added while paused.
// It is a noop if the folder isn't known.
func (r *indexHandlerRegistry) folderRunningLocked(folder config.FolderConfiguration, fset *db.FileSet, runner service) {
func (r *indexHandlerRegistry) folderRunningLocked(folder config.FolderConfiguration, runner service) {
r.folderStates[folder.ID] = &indexHandlerFolderState{
cfg: folder,
fset: fset,
runner: runner,
}
@@ -656,12 +662,12 @@ func (r *indexHandlerRegistry) folderRunningLocked(folder config.FolderConfigura
r.indexHandlers.RemoveAndWait(folder.ID, 0)
l.Debugf("Removed index handler for device %v and folder %v in resume", r.conn.DeviceID().Short(), folder.ID)
}
r.startLocked(folder, fset, runner, info)
_ = r.startLocked(folder, runner, info) // XXX error handling...
delete(r.startInfos, folder.ID)
l.Debugf("Started index handler for device %v and folder %v in resume", r.conn.DeviceID().Short(), folder.ID)
} else if isOk {
l.Debugf("Resuming index handler for device %v and folder %v", r.conn.DeviceID().Short(), folder)
is.resume(fset, runner)
is.resume(runner)
} else {
l.Debugf("Not resuming index handler for device %v and folder %v as none is paused and there is no start info", r.conn.DeviceID().Short(), folder.ID)
}
lib/model/indexhandler_test.go
+2 -2
View File
@@ -13,7 +13,7 @@ import (
"sync"
"testing"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/model/mocks"
"github.com/syncthing/syncthing/lib/protocol"
protomock "github.com/syncthing/syncthing/lib/protocol/mocks"
@@ -63,7 +63,7 @@ func TestIndexhandlerConcurrency(t *testing.T) {
return nil
})
b1 := db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
b1 := model.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
return c1.IndexUpdate(ctx, &protocol.IndexUpdate{Folder: "foo", Files: fs})
})
sentEntries := 0
lib/model/mocks/model.go
+725 -162
View File
File diff suppressed because it is too large Load Diff
lib/model/model.go
+197 -233
View File
@@ -16,11 +16,13 @@ import (
"errors"
"fmt"
"io"
"iter"
"net"
"os"
"path/filepath"
"reflect"
"runtime"
"slices"
"strings"
stdsync "sync"
"sync/atomic"
@@ -28,10 +30,11 @@ import (
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
@@ -93,7 +96,16 @@ type Model interface {
GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]error, error)
DBSnapshot(folder string) (*db.Snapshot, error)
LocalFiles(folder string, device protocol.DeviceID) (iter.Seq[protocol.FileInfo], func() error)
LocalFilesSequenced(folder string, device protocol.DeviceID, startSet int64) (iter.Seq[protocol.FileInfo], func() error)
LocalSize(folder string, device protocol.DeviceID) (db.Counts, error)
GlobalSize(folder string) (db.Counts, error)
NeedSize(folder string, device protocol.DeviceID) (db.Counts, error)
ReceiveOnlySize(folder string) (db.Counts, error)
Sequence(folder string, device protocol.DeviceID) (int64, error)
AllGlobalFiles(folder string) (iter.Seq[db.FileMetadata], func() error)
RemoteSequences(folder string) (map[protocol.DeviceID]int64, error)
NeedFolderFiles(folder string, page, perpage int) ([]protocol.FileInfo, []protocol.FileInfo, []protocol.FileInfo, error)
RemoteNeedFolderFiles(folder string, device protocol.DeviceID, page, perpage int) ([]protocol.FileInfo, error)
LocalChangedFolderFiles(folder string, page, perpage int) ([]protocol.FileInfo, error)
@@ -101,7 +113,6 @@ type Model interface {
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error)
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error)
GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error)
Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) ([]Availability, error)
Completion(device protocol.DeviceID, folder string) (FolderCompletion, error)
@@ -127,12 +138,11 @@ type model struct {
// constructor parameters
cfg config.Wrapper
id protocol.DeviceID
db *db.Lowlevel
sdb db.DB
protectedFiles []string
evLogger events.Logger
// constant or concurrency safe fields
finder *db.BlockFinder
progressEmitter *ProgressEmitter
shortID protocol.ShortID
// globalRequestLimiter limits the amount of data in concurrent incoming
@@ -145,11 +155,11 @@ type model struct {
started chan struct{}
keyGen *protocol.KeyGenerator
promotionTimer *time.Timer
observed *db.ObservedDB
// fields protected by mut
mut sync.RWMutex
folderCfgs map[string]config.FolderConfiguration // folder -> cfg
folderFiles map[string]*db.FileSet // folder -> files
deviceStatRefs map[protocol.DeviceID]*stats.DeviceStatisticsReference // deviceID -> statsRef
folderIgnores map[string]*ignore.Matcher // folder -> matcher object
folderRunners *serviceMap[string, service] // folder -> puller or scanner
@@ -173,7 +183,7 @@ type model struct {
var _ config.Verifier = &model{}
type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *semaphore.Semaphore) service
type folderFactory func(*model, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *semaphore.Semaphore) service
var folderFactories = make(map[config.FolderType]folderFactory)
@@ -202,7 +212,7 @@ var (
// NewModel creates and starts a new model. The model starts in read-only mode,
// where it sends index information to connected peers and responds to requests
// for file data without altering the local folder in any way.
func NewModel(cfg config.Wrapper, id protocol.DeviceID, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger, keyGen *protocol.KeyGenerator) Model {
func NewModel(cfg config.Wrapper, id protocol.DeviceID, sdb db.DB, protectedFiles []string, evLogger events.Logger, keyGen *protocol.KeyGenerator) Model {
spec := svcutil.SpecWithDebugLogger(l)
m := &model{
Supervisor: suture.New("model", spec),
@@ -210,12 +220,11 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, ldb *db.Lowlevel, protec
// constructor parameters
cfg: cfg,
id: id,
db: ldb,
sdb: sdb,
protectedFiles: protectedFiles,
evLogger: evLogger,
// constant or concurrency safe fields
finder: db.NewBlockFinder(ldb),
progressEmitter: NewProgressEmitter(cfg, evLogger),
shortID: id.Short(),
globalRequestLimiter: semaphore.New(1024 * cfg.Options().MaxConcurrentIncomingRequestKiB()),
@@ -224,11 +233,11 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, ldb *db.Lowlevel, protec
started: make(chan struct{}),
keyGen: keyGen,
promotionTimer: time.NewTimer(0),
observed: db.NewObservedDB(sdb),
// fields protected by mut
mut: sync.NewRWMutex(),
folderCfgs: make(map[string]config.FolderConfiguration),
folderFiles: make(map[string]*db.FileSet),
deviceStatRefs: make(map[protocol.DeviceID]*stats.DeviceStatisticsReference),
folderIgnores: make(map[string]*ignore.Matcher),
folderRunners: newServiceMap[string, service](evLogger),
@@ -246,7 +255,7 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, ldb *db.Lowlevel, protec
indexHandlers: newServiceMap[protocol.DeviceID, *indexHandlerRegistry](evLogger),
}
for devID, cfg := range cfg.Devices() {
m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID)
m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(db.NewTyped(sdb, "devicestats/"+devID.String()))
m.setConnRequestLimitersLocked(cfg)
}
m.Add(m.folderRunners)
@@ -327,21 +336,20 @@ func (m *model) fatal(err error) {
}
// Need to hold lock on m.mut when calling this.
func (m *model) addAndStartFolderLocked(cfg config.FolderConfiguration, fset *db.FileSet, cacheIgnoredFiles bool) {
ignores := ignore.New(cfg.Filesystem(nil), ignore.WithCache(cacheIgnoredFiles))
func (m *model) addAndStartFolderLocked(cfg config.FolderConfiguration, cacheIgnoredFiles bool) {
ignores := ignore.New(cfg.Filesystem(), ignore.WithCache(cacheIgnoredFiles))
if cfg.Type != config.FolderTypeReceiveEncrypted {
if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
l.Warnln("Loading ignores:", err)
}
}
m.addAndStartFolderLockedWithIgnores(cfg, fset, ignores)
m.addAndStartFolderLockedWithIgnores(cfg, ignores)
}
// Only needed for testing, use addAndStartFolderLocked instead.
func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguration, fset *db.FileSet, ignores *ignore.Matcher) {
func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguration, ignores *ignore.Matcher) {
m.folderCfgs[cfg.ID] = cfg
m.folderFiles[cfg.ID] = fset
m.folderIgnores[cfg.ID] = ignores
_, ok := m.folderRunners.Get(cfg.ID)
@@ -360,16 +368,19 @@ func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguratio
// Find any devices for which we hold the index in the db, but the folder
// is not shared, and drop it.
expected := mapDevices(cfg.DeviceIDs())
for _, available := range fset.ListDevices() {
devs, _ := m.sdb.ListDevicesForFolder(cfg.ID)
for _, available := range devs {
if _, ok := expected[available]; !ok {
l.Debugln("dropping", folder, "state for", available)
fset.Drop(available)
m.sdb.DropAllFiles(folder, available)
}
}
v, ok := fset.Sequence(protocol.LocalDeviceID), true
indexHasFiles := ok && v > 0
if !indexHasFiles {
seq, err := m.sdb.GetDeviceSequence(folder, protocol.LocalDeviceID)
if err != nil {
panic(fmt.Errorf("error getting sequence number: %w", err))
}
if seq == 0 {
// It's a blank folder, so this may the first time we're looking at
// it. Attempt to create and tag with our marker as appropriate. We
// don't really do anything with errors at this point except warn -
@@ -392,7 +403,7 @@ func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguratio
}
// These are our metadata files, and they should always be hidden.
ffs := cfg.Filesystem(nil)
ffs := cfg.Filesystem()
_ = ffs.Hide(config.DefaultMarkerName)
_ = ffs.Hide(versioner.DefaultPath)
_ = ffs.Hide(".stignore")
@@ -409,7 +420,7 @@ func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguratio
m.warnAboutOverwritingProtectedFiles(cfg, ignores)
p := folderFactory(m, fset, ignores, cfg, ver, m.evLogger, m.folderIOLimiter)
p := folderFactory(m, ignores, cfg, ver, m.evLogger, m.folderIOLimiter)
m.folderRunners.Add(folder, p)
l.Infof("Ready to synchronize %s (%s)", cfg.Description(), cfg.Type)
@@ -421,7 +432,7 @@ func (m *model) warnAboutOverwritingProtectedFiles(cfg config.FolderConfiguratio
}
// This is a bit of a hack.
ffs := cfg.Filesystem(nil)
ffs := cfg.Filesystem()
if ffs.Type() != fs.FilesystemTypeBasic {
return
}
@@ -468,7 +479,7 @@ func (m *model) removeFolder(cfg config.FolderConfiguration) {
// otherwise not removable) Syncthing-specific marker files.
if err := cfg.RemoveMarker(); err != nil && !errors.Is(err, os.ErrNotExist) {
moved := config.DefaultMarkerName + time.Now().Format(".removed-20060102-150405")
fs := cfg.Filesystem(nil)
fs := cfg.Filesystem()
_ = fs.Rename(config.DefaultMarkerName, moved)
}
}
@@ -482,7 +493,7 @@ func (m *model) removeFolder(cfg config.FolderConfiguration) {
m.mut.Unlock()
// Remove it from the database
db.DropFolder(m.db, cfg.ID)
m.sdb.DropFolder(cfg.ID)
}
// Need to hold lock on m.mut when calling this.
@@ -490,7 +501,6 @@ func (m *model) cleanupFolderLocked(cfg config.FolderConfiguration) {
// clear up our config maps
m.folderRunners.Remove(cfg.ID)
delete(m.folderCfgs, cfg.ID)
delete(m.folderFiles, cfg.ID)
delete(m.folderIgnores, cfg.ID)
delete(m.folderVersioners, cfg.ID)
delete(m.folderEncryptionPasswordTokens, cfg.ID)
@@ -525,28 +535,14 @@ func (m *model) restartFolder(from, to config.FolderConfiguration, cacheIgnoredF
m.mut.Lock()
defer m.mut.Unlock()
// Cache the (maybe) existing fset before it's removed by cleanupFolderLocked
fset := m.folderFiles[folder]
fsetNil := fset == nil
m.cleanupFolderLocked(from)
if !to.Paused {
if fsetNil {
// Create a new fset. Might take a while and we do it under
// locking, but it's unsafe to create fset:s concurrently so
// that's the price we pay.
var err error
fset, err = db.NewFileSet(folder, m.db)
if err != nil {
return fmt.Errorf("restarting %v: %w", to.Description(), err)
}
}
m.addAndStartFolderLocked(to, fset, cacheIgnoredFiles)
m.addAndStartFolderLocked(to, cacheIgnoredFiles)
}
runner, _ := m.folderRunners.Get(to.ID)
m.indexHandlers.Each(func(_ protocol.DeviceID, r *indexHandlerRegistry) error {
r.RegisterFolderState(to, fset, runner)
r.RegisterFolderState(to, runner)
return nil
})
@@ -568,22 +564,14 @@ func (m *model) newFolder(cfg config.FolderConfiguration, cacheIgnoredFiles bool
m.mut.Lock()
defer m.mut.Unlock()
// Creating the fileset can take a long time (metadata calculation), but
// nevertheless should happen inside the lock (same as when restarting
// a folder).
fset, err := db.NewFileSet(cfg.ID, m.db)
if err != nil {
return fmt.Errorf("adding %v: %w", cfg.Description(), err)
}
m.addAndStartFolderLocked(cfg, fset, cacheIgnoredFiles)
m.addAndStartFolderLocked(cfg, cacheIgnoredFiles)
// Cluster configs might be received and processed before reaching this
// point, i.e. before the folder is started. If that's the case, start
// index senders here.
m.indexHandlers.Each(func(_ protocol.DeviceID, r *indexHandlerRegistry) error {
runner, _ := m.folderRunners.Get(cfg.ID)
r.RegisterFolderState(cfg, fset, runner)
r.RegisterFolderState(cfg, runner)
return nil
})
@@ -923,46 +911,78 @@ func (m *model) Completion(device protocol.DeviceID, folder string) (FolderCompl
func (m *model) folderCompletion(device protocol.DeviceID, folder string) (FolderCompletion, error) {
m.mut.RLock()
err := m.checkFolderRunningRLocked(folder)
rf := m.folderFiles[folder]
m.mut.RUnlock()
if err != nil {
return FolderCompletion{}, err
}
snap, err := rf.Snapshot()
if err != nil {
return FolderCompletion{}, err
}
defer snap.Release()
m.mut.RLock()
state := m.remoteFolderStates[device][folder]
downloaded := m.deviceDownloads[device].BytesDownloaded(folder)
m.mut.RUnlock()
need := snap.NeedSize(device)
need, err := m.sdb.CountNeed(folder, device)
if err != nil {
return FolderCompletion{}, err
}
need.Bytes -= downloaded
// This might be more than it really is, because some blocks can be of a smaller size.
if need.Bytes < 0 {
need.Bytes = 0
}
comp := newFolderCompletion(snap.GlobalSize(), need, snap.Sequence(device), state)
seq, err := m.sdb.GetDeviceSequence(folder, device)
if err != nil {
return FolderCompletion{}, err
}
glob, err := m.sdb.CountGlobal(folder)
if err != nil {
return FolderCompletion{}, err
}
comp := newFolderCompletion(glob, need, seq, state)
l.Debugf("%v Completion(%s, %q): %v", m, device, folder, comp.Map())
return comp, nil
}
// DBSnapshot returns a snapshot of the database content relevant to the given folder.
func (m *model) DBSnapshot(folder string) (*db.Snapshot, error) {
m.mut.RLock()
err := m.checkFolderRunningRLocked(folder)
rf := m.folderFiles[folder]
m.mut.RUnlock()
if err != nil {
return nil, err
}
return rf.Snapshot()
func (m *model) LocalFiles(folder string, device protocol.DeviceID) (iter.Seq[protocol.FileInfo], func() error) {
return m.sdb.AllLocalFiles(folder, device)
}
func (m *model) LocalFilesSequenced(folder string, device protocol.DeviceID, startSeq int64) (iter.Seq[protocol.FileInfo], func() error) {
return m.sdb.AllLocalFilesBySequence(folder, device, startSeq, 0)
}
func (m *model) AllForBlocksHash(folder string, h []byte) (iter.Seq[db.FileMetadata], func() error) {
return m.sdb.AllLocalFilesWithBlocksHash(folder, h)
}
func (m *model) LocalSize(folder string, device protocol.DeviceID) (db.Counts, error) {
return m.sdb.CountLocal(folder, device)
}
func (m *model) GlobalSize(folder string) (db.Counts, error) {
return m.sdb.CountGlobal(folder)
}
func (m *model) NeedSize(folder string, device protocol.DeviceID) (db.Counts, error) {
return m.sdb.CountNeed(folder, device)
}
func (m *model) ReceiveOnlySize(folder string) (db.Counts, error) {
return m.sdb.CountReceiveOnlyChanged(folder)
}
func (m *model) Sequence(folder string, device protocol.DeviceID) (int64, error) {
return m.sdb.GetDeviceSequence(folder, device)
}
func (m *model) AllGlobalFiles(folder string) (iter.Seq[db.FileMetadata], func() error) {
return m.sdb.AllGlobalFiles(folder)
}
func (m *model) RemoteSequences(folder string) (map[protocol.DeviceID]int64, error) {
return m.sdb.RemoteSequences(folder)
}
func (m *model) FolderProgressBytesCompleted(folder string) int64 {
@@ -973,20 +993,14 @@ func (m *model) FolderProgressBytesCompleted(folder string) int64 {
// progress, queued, and to be queued on next puller iteration.
func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]protocol.FileInfo, []protocol.FileInfo, []protocol.FileInfo, error) {
m.mut.RLock()
rf, rfOk := m.folderFiles[folder]
runner, runnerOk := m.folderRunners.Get(folder)
cfg := m.folderCfgs[folder]
cfg, cfgOK := m.folderCfgs[folder]
m.mut.RUnlock()
if !rfOk {
if !cfgOK {
return nil, nil, nil, ErrFolderMissing
}
snap, err := rf.Snapshot()
if err != nil {
return nil, nil, nil, err
}
defer snap.Release()
var progress, queued, rest []protocol.FileInfo
var seen map[string]struct{}
@@ -1000,14 +1014,14 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]protocol.Fi
seen = make(map[string]struct{}, len(progressNames)+len(queuedNames))
for i, name := range progressNames {
if f, ok := snap.GetGlobalTruncated(name); ok {
if f, ok, err := m.sdb.GetGlobalFile(folder, name); err == nil && ok {
progress[i] = f
seen[name] = struct{}{}
}
}
for i, name := range queuedNames {
if f, ok := snap.GetGlobalTruncated(name); ok {
if f, ok, err := m.sdb.GetGlobalFile(folder, name); err == nil && ok {
queued[i] = f
seen[name] = struct{}{}
}
@@ -1020,21 +1034,29 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]protocol.Fi
p.toSkip -= skipped
}
rest = make([]protocol.FileInfo, 0, perpage)
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileInfo) bool {
if cfg.IgnoreDelete && f.IsDeleted() {
return true
}
if p.get > 0 {
rest = make([]protocol.FileInfo, 0, p.get)
it, errFn := m.sdb.AllNeededGlobalFiles(folder, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0)
for f := range it {
if cfg.IgnoreDelete && f.IsDeleted() {
continue
}
if p.skip() {
return true
if p.skip() {
continue
}
if _, ok := seen[f.Name]; !ok {
rest = append(rest, f)
p.get--
}
if p.get == 0 {
break
}
}
if _, ok := seen[f.Name]; !ok {
rest = append(rest, f)
p.get--
if err := errFn(); err != nil {
return nil, nil, nil, err
}
return p.get > 0
})
}
return progress, queued, rest, nil
}
@@ -1043,63 +1065,56 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]protocol.Fi
// remote device to become synced with a folder.
func (m *model) RemoteNeedFolderFiles(folder string, device protocol.DeviceID, page, perpage int) ([]protocol.FileInfo, error) {
m.mut.RLock()
rf, ok := m.folderFiles[folder]
_, ok := m.folderCfgs[folder]
m.mut.RUnlock()
if !ok {
return nil, ErrFolderMissing
}
snap, err := rf.Snapshot()
if err != nil {
return nil, err
}
defer snap.Release()
files := make([]protocol.FileInfo, 0, perpage)
p := newPager(page, perpage)
snap.WithNeedTruncated(device, func(f protocol.FileInfo) bool {
if p.skip() {
return true
}
files = append(files, f)
return !p.done()
})
return files, nil
it, errFn := m.sdb.AllNeededGlobalFiles(folder, device, config.PullOrderAlphabetic, perpage, (page-1)*perpage)
files := slices.Collect(it)
return files, errFn()
}
func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]protocol.FileInfo, error) {
m.mut.RLock()
rf, ok := m.folderFiles[folder]
_, ok := m.folderCfgs[folder]
m.mut.RUnlock()
if !ok {
return nil, ErrFolderMissing
}
snap, err := rf.Snapshot()
ros, err := m.sdb.CountReceiveOnlyChanged(folder)
if err != nil {
return nil, err
}
defer snap.Release()
if snap.ReceiveOnlyChangedSize().TotalItems() == 0 {
if ros.TotalItems() == 0 {
return nil, nil
}
p := newPager(page, perpage)
files := make([]protocol.FileInfo, 0, perpage)
snap.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileInfo) bool {
// This could be made more efficient with a specifically targeted DB
// call
it, errFn := m.sdb.AllLocalFiles(folder, protocol.LocalDeviceID)
for f := range it {
if !f.IsReceiveOnlyChanged() {
return true
continue
}
if p.skip() {
return true
continue
}
files = append(files, f)
return !p.done()
})
if p.done() {
break
}
}
if err := errFn(); err != nil {
return nil, err
}
return files, nil
}
@@ -1343,11 +1358,11 @@ func (m *model) ensureIndexHandler(conn protocol.Connection) *indexHandlerRegist
}
// Create a new index handler for this device.
indexHandlerRegistry = newIndexHandlerRegistry(conn, m.deviceDownloads[deviceID], m.evLogger)
indexHandlerRegistry = newIndexHandlerRegistry(conn, m.sdb, m.deviceDownloads[deviceID], m.evLogger)
for id, fcfg := range m.folderCfgs {
l.Debugln("Registering folder", id, "for", deviceID.Short())
runner, _ := m.folderRunners.Get(id)
indexHandlerRegistry.RegisterFolderState(fcfg, m.folderFiles[id], runner)
indexHandlerRegistry.RegisterFolderState(fcfg, runner)
}
m.indexHandlers.Add(deviceID, indexHandlerRegistry)
@@ -1376,7 +1391,7 @@ func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.Devi
seenFolders := make(map[string]remoteFolderState, len(folders))
updatedPending := make([]updatedPendingFolder, 0, len(folders))
deviceID := deviceCfg.DeviceID
expiredPending, err := m.db.PendingFoldersForDevice(deviceID)
expiredPending, err := m.observed.PendingFoldersForDevice(deviceID)
if err != nil {
l.Infof("Could not get pending folders for cleanup: %v", err)
}
@@ -1398,7 +1413,7 @@ func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.Devi
of.Label = folder.Label
of.ReceiveEncrypted = len(ccDeviceInfos[folder.ID].local.EncryptionPasswordToken) > 0
of.RemoteEncrypted = len(ccDeviceInfos[folder.ID].remote.EncryptionPasswordToken) > 0
if err := m.db.AddOrUpdatePendingFolder(folder.ID, of, deviceID); err != nil {
if err := m.observed.AddOrUpdatePendingFolder(folder.ID, of, deviceID); err != nil {
l.Warnf("Failed to persist pending folder entry to database: %v", err)
}
if !folder.Paused {
@@ -1485,7 +1500,7 @@ func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.Devi
expiredPendingList := make([]map[string]string, 0, len(expiredPending))
for folder := range expiredPending {
if err = m.db.RemovePendingFolderForDevice(folder, deviceID); err != nil {
if err = m.observed.RemovePendingFolderForDevice(folder, deviceID); err != nil {
msg := "Failed to remove pending folder-device entry"
l.Warnf("%v (%v, %v): %v", msg, folder, deviceID, err)
m.evLogger.Log(events.Failure, msg)
@@ -2015,7 +2030,7 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr
// Grab the FS after limiting, as it causes I/O and we want to minimize
// the race time between the symlink check and the read.
folderFs := folderCfg.Filesystem(nil)
folderFs := folderCfg.Filesystem()
if err := osutil.TraversesSymlink(folderFs, filepath.Dir(req.Name)); err != nil {
l.Debugf("%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d", m, err, deviceID.Short(), req.Folder, req.Name, req.Offset, req.Size)
@@ -2138,46 +2153,11 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, off
}
func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error) {
m.mut.RLock()
fs, ok := m.folderFiles[folder]
m.mut.RUnlock()
if !ok {
return protocol.FileInfo{}, false, ErrFolderMissing
}
snap, err := fs.Snapshot()
if err != nil {
return protocol.FileInfo{}, false, err
}
f, ok := snap.Get(protocol.LocalDeviceID, file)
snap.Release()
return f, ok, nil
return m.sdb.GetDeviceFile(folder, protocol.LocalDeviceID, file)
}
func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) {
m.mut.RLock()
ffs, ok := m.folderFiles[folder]
m.mut.RUnlock()
if !ok {
return protocol.FileInfo{}, false, ErrFolderMissing
}
snap, err := ffs.Snapshot()
if err != nil {
return protocol.FileInfo{}, false, err
}
f, ok := snap.GetGlobal(file)
snap.Release()
return f, ok, nil
}
func (m *model) GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error) {
m.mut.RLock()
ffs, ok := m.folderFiles[folder]
fcfg := m.folderCfgs[folder]
m.mut.RUnlock()
if !ok {
return fs.MtimeMapping{}, ErrFolderMissing
}
return fs.GetMtimeMapping(fcfg.Filesystem(ffs), file)
return m.sdb.GetGlobalFile(folder, file)
}
// Connection returns if we are connected to the given device.
@@ -2208,7 +2188,7 @@ func (m *model) LoadIgnores(folder string) ([]string, []string, error) {
}
if !ignoresOk {
ignores = ignore.New(cfg.Filesystem(nil))
ignores = ignore.New(cfg.Filesystem())
}
err := ignores.Load(".stignore")
@@ -2263,7 +2243,7 @@ func (m *model) setIgnores(cfg config.FolderConfiguration, content []string) err
return err
}
if err := ignore.WriteIgnores(cfg.Filesystem(nil), ".stignore", content); err != nil {
if err := ignore.WriteIgnores(cfg.Filesystem(), ".stignore", content); err != nil {
l.Warnln("Saving .stignore:", err)
return err
}
@@ -2282,7 +2262,7 @@ func (m *model) setIgnores(cfg config.FolderConfiguration, content []string) err
// and add it to a list of known devices ahead of any checks.
func (m *model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protocol.Hello) error {
if _, ok := m.cfg.Device(remoteID); !ok {
if err := m.db.AddOrUpdatePendingDevice(remoteID, hello.DeviceName, addr.String()); err != nil {
if err := m.observed.AddOrUpdatePendingDevice(remoteID, hello.DeviceName, addr.String()); err != nil {
l.Warnf("Failed to persist pending device entry to database: %v", err)
}
m.evLogger.Log(events.PendingDevicesChanged, map[string][]interface{}{
@@ -2611,13 +2591,11 @@ func (m *model) generateClusterConfigRLocked(device protocol.DeviceID) (*protoco
DisableTempIndexes: folderCfg.DisableTempIndexes,
}
fs := m.folderFiles[folderCfg.ID]
// Even if we aren't paused, if we haven't started the folder yet
// pretend we are. Otherwise the remote might get confused about
// the missing index info (and drop all the info). We will send
// another cluster config once the folder is started.
protocolFolder.Paused = folderCfg.Paused || fs == nil
protocolFolder.Paused = folderCfg.Paused
for _, folderDevice := range folderCfg.Devices {
deviceCfg, _ := m.cfg.Device(folderDevice.DeviceID)
@@ -2640,14 +2618,12 @@ func (m *model) generateClusterConfigRLocked(device protocol.DeviceID) (*protoco
}
}
if fs != nil {
if deviceCfg.DeviceID == m.id {
protocolDevice.IndexID = fs.IndexID(protocol.LocalDeviceID)
protocolDevice.MaxSequence = fs.Sequence(protocol.LocalDeviceID)
} else {
protocolDevice.IndexID = fs.IndexID(deviceCfg.DeviceID)
protocolDevice.MaxSequence = fs.Sequence(deviceCfg.DeviceID)
}
if deviceCfg.DeviceID == m.id {
protocolDevice.IndexID, _ = m.sdb.GetIndexID(folderCfg.ID, protocol.LocalDeviceID)
protocolDevice.MaxSequence, _ = m.sdb.GetDeviceSequence(folderCfg.ID, protocol.LocalDeviceID)
} else {
protocolDevice.IndexID, _ = m.sdb.GetIndexID(folderCfg.ID, deviceCfg.DeviceID)
protocolDevice.MaxSequence, _ = m.sdb.GetDeviceSequence(folderCfg.ID, deviceCfg.DeviceID)
}
protocolFolder.Devices = append(protocolFolder.Devices, protocolDevice)
@@ -2744,7 +2720,7 @@ func findByName(slice []*TreeEntry, name string) *TreeEntry {
func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly bool) ([]*TreeEntry, error) {
m.mut.RLock()
files, ok := m.folderFiles[folder]
_, ok := m.folderCfgs[folder]
m.mut.RUnlock()
if !ok {
return nil, ErrFolderMissing
@@ -2760,15 +2736,14 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly
prefix = prefix + sep
}
snap, err := files.Snapshot()
if err != nil {
return nil, err
}
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(f protocol.FileInfo) bool {
for f, err := range itererr.Zip(m.sdb.AllGlobalFilesPrefix(folder, prefix)) {
if err != nil {
return nil, err
}
// Don't include the prefix itself.
if f.IsInvalid() || f.IsDeleted() || strings.HasPrefix(prefix, f.Name) {
return true
if f.Invalid || f.Deleted || strings.HasPrefix(prefix, f.Name) {
continue
}
f.Name = strings.Replace(f.Name, prefix, "", 1)
@@ -2777,7 +2752,7 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly
base := filepath.Base(f.Name)
if levels > -1 && strings.Count(f.Name, sep) > levels {
return true
continue
}
parent := root
@@ -2785,28 +2760,22 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly
for _, path := range strings.Split(dir, sep) {
child := findByName(parent.Children, path)
if child == nil {
err = fmt.Errorf("could not find child '%s' for path '%s' in parent '%s'", path, f.Name, parent.Name)
return false
return nil, fmt.Errorf("could not find child '%s' for path '%s' in parent '%s'", path, f.Name, parent.Name)
}
parent = child
}
}
if dirsOnly && !f.IsDirectory() {
return true
continue
}
parent.Children = append(parent.Children, &TreeEntry{
Name: base,
Type: f.Type.String(),
ModTime: f.ModTime(),
Size: f.FileSize(),
Size: f.Size,
})
return true
})
if err != nil {
return nil, err
}
return root.Children, nil
@@ -2860,46 +2829,42 @@ func (m *model) Availability(folder string, file protocol.FileInfo, block protoc
m.mut.RLock()
defer m.mut.RUnlock()
fs, ok := m.folderFiles[folder]
cfg := m.folderCfgs[folder]
cfg, ok := m.folderCfgs[folder]
if !ok {
return nil, ErrFolderMissing
}
snap, err := fs.Snapshot()
if err != nil {
return nil, err
}
defer snap.Release()
return m.blockAvailabilityRLocked(cfg, snap, file, block), nil
return m.blockAvailabilityRLocked(cfg, file, block), nil
}
func (m *model) blockAvailability(cfg config.FolderConfiguration, snap *db.Snapshot, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
func (m *model) blockAvailability(cfg config.FolderConfiguration, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
m.mut.RLock()
defer m.mut.RUnlock()
return m.blockAvailabilityRLocked(cfg, snap, file, block)
return m.blockAvailabilityRLocked(cfg, file, block)
}
func (m *model) blockAvailabilityRLocked(cfg config.FolderConfiguration, snap *db.Snapshot, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
func (m *model) blockAvailabilityRLocked(cfg config.FolderConfiguration, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
var candidates []Availability
candidates = append(candidates, m.fileAvailabilityRLocked(cfg, snap, file)...)
candidates = append(candidates, m.fileAvailabilityRLocked(cfg, file)...)
candidates = append(candidates, m.blockAvailabilityFromTemporaryRLocked(cfg, file, block)...)
return candidates
}
func (m *model) fileAvailability(cfg config.FolderConfiguration, snap *db.Snapshot, file protocol.FileInfo) []Availability {
func (m *model) fileAvailability(cfg config.FolderConfiguration, file protocol.FileInfo) []Availability {
m.mut.RLock()
defer m.mut.RUnlock()
return m.fileAvailabilityRLocked(cfg, snap, file)
return m.fileAvailabilityRLocked(cfg, file)
}
func (m *model) fileAvailabilityRLocked(cfg config.FolderConfiguration, snap *db.Snapshot, file protocol.FileInfo) []Availability {
func (m *model) fileAvailabilityRLocked(cfg config.FolderConfiguration, file protocol.FileInfo) []Availability {
var availabilities []Availability
for _, device := range snap.Availability(file.Name) {
devs, err := m.sdb.GetGlobalAvailability(cfg.ID, file.Name)
if err != nil {
return nil
}
for _, device := range devs {
if _, ok := m.remoteFolderStates[device]; !ok {
continue
}
@@ -2936,15 +2901,14 @@ func (m *model) BringToFront(folder, file string) {
}
func (m *model) ResetFolder(folder string) error {
m.mut.RLock()
defer m.mut.RUnlock()
m.mut.Lock()
defer m.mut.Unlock()
_, ok := m.folderRunners.Get(folder)
if ok {
return errors.New("folder must be paused when resetting")
}
l.Infof("Cleaning metadata for reset folder %q", folder)
db.DropFolder(m.db, folder)
return nil
return m.sdb.DropFolder(folder)
}
func (m *model) String() string {
@@ -3058,7 +3022,7 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
for deviceID, toCfg := range toDevices {
fromCfg, ok := fromDevices[deviceID]
if !ok {
sr := stats.NewDeviceStatisticsReference(m.db, deviceID)
sr := stats.NewDeviceStatisticsReference(db.NewTyped(m.sdb, "devicestats/"+deviceID.String()))
m.mut.Lock()
m.deviceStatRefs[deviceID] = sr
m.mut.Unlock()
@@ -3151,7 +3115,7 @@ func (m *model) setConnRequestLimitersLocked(cfg config.DeviceConfiguration) {
func (m *model) cleanPending(existingDevices map[protocol.DeviceID]config.DeviceConfiguration, existingFolders map[string]config.FolderConfiguration, ignoredDevices deviceIDSet, removedFolders map[string]struct{}) {
var removedPendingFolders []map[string]string
pendingFolders, err := m.db.PendingFolders()
pendingFolders, err := m.observed.PendingFolders()
if err != nil {
msg := "Could not iterate through pending folder entries for cleanup"
l.Warnf("%v: %v", msg, err)
@@ -3164,7 +3128,7 @@ func (m *model) cleanPending(existingDevices map[protocol.DeviceID]config.Device
// folders as well, assuming the folder is no longer of interest
// at all (but might become pending again).
l.Debugf("Discarding pending removed folder %v from all devices", folderID)
if err := m.db.RemovePendingFolder(folderID); err != nil {
if err := m.observed.RemovePendingFolder(folderID); err != nil {
msg := "Failed to remove pending folder entry"
l.Warnf("%v (%v): %v", msg, folderID, err)
m.evLogger.Log(events.Failure, msg)
@@ -3191,7 +3155,7 @@ func (m *model) cleanPending(existingDevices map[protocol.DeviceID]config.Device
}
continue
removeFolderForDevice:
if err := m.db.RemovePendingFolderForDevice(folderID, deviceID); err != nil {
if err := m.observed.RemovePendingFolderForDevice(folderID, deviceID); err != nil {
msg := "Failed to remove pending folder-device entry"
l.Warnf("%v (%v, %v): %v", msg, folderID, deviceID, err)
m.evLogger.Log(events.Failure, msg)
@@ -3210,7 +3174,7 @@ func (m *model) cleanPending(existingDevices map[protocol.DeviceID]config.Device
}
var removedPendingDevices []map[string]string
pendingDevices, err := m.db.PendingDevices()
pendingDevices, err := m.observed.PendingDevices()
if err != nil {
msg := "Could not iterate through pending device entries for cleanup"
l.Warnf("%v: %v", msg, err)
@@ -3228,7 +3192,7 @@ func (m *model) cleanPending(existingDevices map[protocol.DeviceID]config.Device
}
continue
removeDevice:
if err := m.db.RemovePendingDevice(deviceID); err != nil {
if err := m.observed.RemovePendingDevice(deviceID); err != nil {
msg := "Failed to remove pending device entry"
l.Warnf("%v: %v", msg, err)
m.evLogger.Log(events.Failure, msg)
@@ -3265,20 +3229,20 @@ func (m *model) checkFolderRunningRLocked(folder string) error {
// PendingDevices lists unknown devices that tried to connect.
func (m *model) PendingDevices() (map[protocol.DeviceID]db.ObservedDevice, error) {
return m.db.PendingDevices()
return m.observed.PendingDevices()
}
// PendingFolders lists folders that we don't yet share with the offering devices. It
// returns the entries grouped by folder and filters for a given device unless the
// argument is specified as EmptyDeviceID.
func (m *model) PendingFolders(device protocol.DeviceID) (map[string]db.PendingFolder, error) {
return m.db.PendingFoldersForDevice(device)
return m.observed.PendingFoldersForDevice(device)
}
// DismissPendingDevices removes the record of a specific pending device.
func (m *model) DismissPendingDevice(device protocol.DeviceID) error {
l.Debugf("Discarding pending device %v", device)
err := m.db.RemovePendingDevice(device)
err := m.observed.RemovePendingDevice(device)
if err != nil {
return err
}
@@ -3298,7 +3262,7 @@ func (m *model) DismissPendingFolder(device protocol.DeviceID, folder string) er
var removedPendingFolders []map[string]string
if device == protocol.EmptyDeviceID {
l.Debugf("Discarding pending removed folder %s from all devices", folder)
err := m.db.RemovePendingFolder(folder)
err := m.observed.RemovePendingFolder(folder)
if err != nil {
return err
}
@@ -3307,7 +3271,7 @@ func (m *model) DismissPendingFolder(device protocol.DeviceID, folder string) er
}
} else {
l.Debugf("Discarding pending folder %s from device %v", folder, device)
err := m.db.RemovePendingFolderForDevice(folder, device)
err := m.observed.RemovePendingFolderForDevice(folder, device)
if err != nil {
return err
}
@@ -3436,7 +3400,7 @@ type storedEncryptionToken struct {
}
func readEncryptionToken(cfg config.FolderConfiguration) ([]byte, error) {
fd, err := cfg.Filesystem(nil).Open(encryptionTokenPath(cfg))
fd, err := cfg.Filesystem().Open(encryptionTokenPath(cfg))
if err != nil {
return nil, err
}
@@ -3450,7 +3414,7 @@ func readEncryptionToken(cfg config.FolderConfiguration) ([]byte, error) {
func writeEncryptionToken(token []byte, cfg config.FolderConfiguration) error {
tokenName := encryptionTokenPath(cfg)
fd, err := cfg.Filesystem(nil).OpenFile(tokenName, fs.OptReadWrite|fs.OptCreate, 0o666)
fd, err := cfg.Filesystem().OpenFile(tokenName, fs.OptReadWrite|fs.OptCreate, 0o666)
if err != nil {
return err
}
lib/model/model_test.go
+132 -218
View File
@@ -13,6 +13,7 @@ import (
"errors"
"fmt"
"io"
"iter"
mrand "math/rand"
"os"
"path/filepath"
@@ -24,10 +25,10 @@ import (
"testing"
"time"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
@@ -77,7 +78,7 @@ func addFolderDevicesToClusterConfig(cc *protocol.ClusterConfig, remote protocol
func TestRequest(t *testing.T) {
wrapper, fcfg, cancel := newDefaultCfgWrapper()
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
defer cancel()
m := setupModel(t, wrapper)
defer cleanupModel(m)
@@ -165,7 +166,7 @@ func BenchmarkIndex_100(b *testing.B) {
func benchmarkIndex(b *testing.B, nfiles int) {
m, _, fcfg, wcfgCancel := setupModelWithConnection(b)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
files := genFiles(nfiles)
must(b, m.Index(device1Conn, &protocol.Index{Folder: fcfg.ID, Files: files}))
@@ -192,7 +193,7 @@ func BenchmarkIndexUpdate_10000_1(b *testing.B) {
func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
m, _, fcfg, wcfgCancel := setupModelWithConnection(b)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
files := genFiles(nfiles)
ufiles := genFiles(nufiles)
@@ -235,7 +236,7 @@ func BenchmarkRequestOut(b *testing.B) {
func BenchmarkRequestInSingleFile(b *testing.B) {
w, cancel := newConfigWrapper(defaultCfg)
defer cancel()
ffs := w.FolderList()[0].Filesystem(nil)
ffs := w.FolderList()[0].Filesystem()
m := setupModel(b, w)
defer cleanupModel(m)
@@ -1195,7 +1196,7 @@ func TestAutoAcceptPrefersLabel(t *testing.T) {
func TestAutoAcceptFallsBackToID(t *testing.T) {
// Prefers label, falls back to ID.
m, cancel := newState(t, defaultAutoAcceptCfg)
ffs := defaultFolderConfig.Filesystem(nil)
ffs := defaultFolderConfig.Filesystem()
id := srand.String(8)
label := srand.String(8)
if err := ffs.MkdirAll(label, 0o777); err != nil {
@@ -1488,7 +1489,7 @@ func changeIgnores(t *testing.T, m *testModel, expected []string) {
func TestIgnores(t *testing.T) {
w, cancel := newConfigWrapper(defaultCfg)
defer cancel()
ffs := w.FolderList()[0].Filesystem(nil)
ffs := w.FolderList()[0].Filesystem()
m := setupModel(t, w)
defer cleanupModel(m)
@@ -1523,7 +1524,7 @@ func TestIgnores(t *testing.T) {
ID: "fresh", Path: "XXX",
FilesystemType: config.FilesystemTypeFake,
}
ignores := ignore.New(fcfg.Filesystem(nil), ignore.WithCache(m.cfg.Options().CacheIgnoredFiles))
ignores := ignore.New(fcfg.Filesystem(), ignore.WithCache(m.cfg.Options().CacheIgnoredFiles))
m.mut.Lock()
m.folderCfgs[fcfg.ID] = fcfg
m.folderIgnores[fcfg.ID] = ignores
@@ -1555,7 +1556,7 @@ func TestIgnores(t *testing.T) {
func TestEmptyIgnores(t *testing.T) {
w, cancel := newConfigWrapper(defaultCfg)
defer cancel()
ffs := w.FolderList()[0].Filesystem(nil)
ffs := w.FolderList()[0].Filesystem()
m := setupModel(t, w)
defer cleanupModel(m)
@@ -1628,12 +1629,11 @@ func TestROScanRecovery(t *testing.T) {
defer cancel()
m := newModel(t, cfg, myID, nil)
set := newFileSet(t, "default", m.db)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
m.sdb.Update("default", protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile", Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}}},
})
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
// Remove marker to generate an error
ffs.Remove(fcfg.MarkerName)
@@ -1675,12 +1675,11 @@ func TestRWScanRecovery(t *testing.T) {
defer cancel()
m := newModel(t, cfg, myID, nil)
set := newFileSet(t, "default", m.db)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
m.sdb.Update("default", protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile", Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}}},
})
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
// Generate error
if err := ffs.Remove(config.DefaultMarkerName); err != nil {
@@ -1706,8 +1705,9 @@ func TestRWScanRecovery(t *testing.T) {
func TestGlobalDirectoryTree(t *testing.T) {
m, conn, fcfg, wCancel := setupModelWithConnection(t)
defer wCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
var seq int64
b := func(isfile bool, path ...string) protocol.FileInfo {
typ := protocol.FileInfoTypeDirectory
var blocks []protocol.BlockInfo
@@ -1716,12 +1716,14 @@ func TestGlobalDirectoryTree(t *testing.T) {
typ = protocol.FileInfoTypeFile
blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
}
seq++
return protocol.FileInfo{
Name: filepath.Join(path...),
Type: typ,
ModifiedS: 0x666,
Blocks: blocks,
Size: 0xa,
Sequence: seq,
}
}
f := func(name string) *TreeEntry {
@@ -1813,13 +1815,13 @@ func TestGlobalDirectoryTree(t *testing.T) {
result, _ := m.GlobalDirectoryTree("default", "", -1, false)
if mm(result) != mm(expectedResult) {
t.Errorf("Does not match:\n%s\n============\n%s", mm(result), mm(expectedResult))
t.Fatalf("Does not match:\n%s\n============\n%s", mm(result), mm(expectedResult))
}
result, _ = m.GlobalDirectoryTree("default", "another", -1, false)
if mm(result) != mm(findByName(expectedResult, "another").Children) {
t.Errorf("Does not match:\n%s\n============\n%s", mm(result), mm(findByName(expectedResult, "another").Children))
t.Fatalf("Does not match:\n%s\n============\n%s", mm(result), mm(findByName(expectedResult, "another").Children))
}
result, _ = m.GlobalDirectoryTree("default", "", 0, false)
@@ -1831,7 +1833,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n============\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n============\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "", 1, false)
@@ -1852,7 +1854,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "", -1, true)
@@ -1882,7 +1884,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "", 1, true)
@@ -1901,7 +1903,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "another", 0, false)
@@ -1911,7 +1913,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "some/directory", 0, false)
@@ -1920,7 +1922,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "some/directory", 1, false)
@@ -1931,7 +1933,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "some/directory", 2, false)
@@ -1944,7 +1946,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
result, _ = m.GlobalDirectoryTree("default", "another", -1, true)
@@ -1957,7 +1959,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
// No prefix matching!
@@ -1965,7 +1967,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
currentResult = []*TreeEntry{}
if mm(result) != mm(currentResult) {
t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
t.Fatalf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
}
}
@@ -2010,7 +2012,7 @@ func BenchmarkTree_100_10(b *testing.B) {
func benchmarkTree(b *testing.B, n1, n2 int) {
m, _, fcfg, wcfgCancel := setupModelWithConnection(b)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
m.ScanFolder(fcfg.ID)
files := genDeepFiles(n1, n2)
@@ -2027,7 +2029,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
func TestIssue3028(t *testing.T) {
w, cancel := newConfigWrapper(defaultCfg)
defer cancel()
ffs := w.FolderList()[0].Filesystem(nil)
ffs := w.FolderList()[0].Filesystem()
m := setupModel(t, w)
defer cleanupModel(m)
@@ -2039,8 +2041,8 @@ func TestIssue3028(t *testing.T) {
// Scan, and get a count of how many files are there now
m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"})
locorigfiles := localSize(t, m, "default").Files
globorigfiles := globalSize(t, m, "default").Files
locorigfiles := mustV(m.LocalSize("default", protocol.LocalDeviceID)).Files
globorigfiles := mustV(m.GlobalSize("default")).Files
// Delete
@@ -2051,8 +2053,8 @@ func TestIssue3028(t *testing.T) {
// deleted files increases by two
m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"})
loc := localSize(t, m, "default")
glob := globalSize(t, m, "default")
loc := mustV(m.LocalSize("default", protocol.LocalDeviceID))
glob := mustV(m.GlobalSize("default"))
if loc.Files != locorigfiles-2 {
t.Errorf("Incorrect local accounting; got %d current files, expected %d", loc.Files, locorigfiles-2)
@@ -2127,24 +2129,22 @@ func TestIssue4357(t *testing.T) {
func TestIndexesForUnknownDevicesDropped(t *testing.T) {
m := newModel(t, defaultCfgWrapper, myID, nil)
files := newFileSet(t, "default", m.db)
files.Drop(device1)
files.Update(device1, genFiles(1))
files.Drop(device2)
files.Update(device2, genFiles(1))
m.sdb.DropAllFiles("default", device1)
m.sdb.Update("default", device1, genFiles(1))
m.sdb.DropAllFiles("default", device2)
m.sdb.Update("default", device2, genFiles(1))
if len(files.ListDevices()) != 2 {
if devs, err := m.sdb.ListDevicesForFolder("default"); err != nil || len(devs) != 2 {
t.Log(devs, err)
t.Error("expected two devices")
}
m.newFolder(defaultFolderConfig, false)
defer cleanupModel(m)
// Remote sequence is cached, hence need to recreated.
files = newFileSet(t, "default", m.db)
if l := len(files.ListDevices()); l != 1 {
t.Errorf("Expected one device got %v", l)
if devs, err := m.sdb.ListDevicesForFolder("default"); err != nil || len(devs) != 1 {
t.Log(devs, err)
t.Error("expected one device")
}
}
@@ -2270,7 +2270,7 @@ func TestIssue3829(t *testing.T) {
func TestIssue4573(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
testFs := fcfg.Filesystem(nil)
testFs := fcfg.Filesystem()
defer os.RemoveAll(testFs.URI())
must(t, testFs.MkdirAll("inaccessible", 0o755))
@@ -2300,7 +2300,7 @@ func TestIssue4573(t *testing.T) {
func TestInternalScan(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
testFs := fcfg.Filesystem(nil)
testFs := fcfg.Filesystem()
defer os.RemoveAll(testFs.URI())
testCases := map[string]func(protocol.FileInfo) bool{
@@ -2372,12 +2372,11 @@ func TestCustomMarkerName(t *testing.T) {
})
defer cancel()
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
m := newModel(t, cfg, myID, nil)
set := newFileSet(t, "default", m.db)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
m.sdb.Update("default", protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile"},
})
@@ -2401,7 +2400,7 @@ func TestCustomMarkerName(t *testing.T) {
func TestRemoveDirWithContent(t *testing.T) {
m, conn, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
tfs.MkdirAll("dirwith", 0o755)
@@ -2463,7 +2462,7 @@ func TestIssue4475(t *testing.T) {
m, conn, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModel(m)
testFs := fcfg.Filesystem(nil)
testFs := fcfg.Filesystem()
// Scenario: Dir is deleted locally and before syncing/index exchange
// happens, a file is create in that dir on the remote.
@@ -2525,7 +2524,7 @@ func TestVersionRestore(t *testing.T) {
fcfg := newFolderConfiguration(defaultCfgWrapper, "default", "default", config.FilesystemTypeFake, srand.String(32))
fcfg.Versioning.Type = "simple"
fcfg.FSWatcherEnabled = false
filesystem := fcfg.Filesystem(nil)
filesystem := fcfg.Filesystem()
rawConfig := config.Configuration{
Version: config.CurrentVersion,
@@ -2759,7 +2758,7 @@ func TestIssue4094(t *testing.T) {
t.Fatalf("failed setting ignores: %v", err)
}
if _, err := fcfg.Filesystem(nil).Lstat(".stignore"); err != nil {
if _, err := fcfg.Filesystem().Lstat(".stignore"); err != nil {
t.Fatalf("failed stating .stignore: %v", err)
}
}
@@ -2788,7 +2787,7 @@ func TestIssue4903(t *testing.T) {
t.Fatalf("expected path missing error, got: %v, debug: %s", err, fcfg.CheckPath())
}
if _, err := fcfg.Filesystem(nil).Lstat("."); !fs.IsNotExist(err) {
if _, err := fcfg.Filesystem().Lstat("."); !fs.IsNotExist(err) {
t.Fatalf("Expected missing path error, got: %v", err)
}
}
@@ -2798,7 +2797,7 @@ func TestIssue5002(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
fd, err := ffs.Create("foo")
must(t, err)
@@ -2827,7 +2826,7 @@ func TestIssue5002(t *testing.T) {
func TestParentOfUnignored(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
must(t, ffs.Mkdir("bar", 0o755))
must(t, ffs.Mkdir("baz", 0o755))
@@ -2906,7 +2905,7 @@ func TestFolderRestartZombies(t *testing.T) {
func TestRequestLimit(t *testing.T) {
wrapper, fcfg, cancel := newDefaultCfgWrapper()
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
file := "tmpfile"
fd, err := ffs.Create(file)
@@ -2966,7 +2965,7 @@ func TestConnCloseOnRestart(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
m := setupModel(t, w)
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
br := &testutil.BlockingRW{}
nw := &testutil.NoopRW{}
@@ -3009,7 +3008,7 @@ func TestModTimeWindow(t *testing.T) {
defer wCancel()
tfs := modtimeTruncatingFS{
trunc: 0,
Filesystem: fcfg.Filesystem(nil),
Filesystem: fcfg.Filesystem(),
}
// fcfg.RawModTimeWindowS = 2
setFolder(t, w, fcfg)
@@ -3069,7 +3068,7 @@ func TestModTimeWindow(t *testing.T) {
func TestDevicePause(t *testing.T) {
m, _, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
sub := m.evLogger.Subscribe(events.DevicePaused)
defer sub.Unsubscribe()
@@ -3099,7 +3098,7 @@ func TestDevicePause(t *testing.T) {
func TestDeviceWasSeen(t *testing.T) {
m, _, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
m.deviceWasSeen(device1)
@@ -3194,7 +3193,7 @@ func TestRenameSequenceOrder(t *testing.T) {
numFiles := 20
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
for i := 0; i < numFiles; i++ {
v := fmt.Sprintf("%d", i)
writeFile(t, ffs, v, []byte(v))
@@ -3202,14 +3201,7 @@ func TestRenameSequenceOrder(t *testing.T) {
m.ScanFolders()
count := 0
snap := dbSnapshot(t, m, "default")
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
count++
return true
})
snap.Release()
count := countIterator[protocol.FileInfo](t)(m.LocalFiles("default", protocol.LocalDeviceID))
if count != numFiles {
t.Errorf("Unexpected count: %d != %d", count, numFiles)
}
@@ -3229,14 +3221,11 @@ func TestRenameSequenceOrder(t *testing.T) {
// Scan
m.ScanFolders()
// Verify sequence of a appearing is followed by c disappearing.
snap = dbSnapshot(t, m, "default")
defer snap.Release()
var firstExpectedSequence int64
var secondExpectedSequence int64
failed := false
snap.WithHaveSequence(0, func(i protocol.FileInfo) bool {
it, errFn := m.LocalFilesSequenced("default", protocol.LocalDeviceID, 0)
for i := range it {
t.Log(i)
if i.FileName() == "17" {
firstExpectedSequence = i.SequenceNo() + 1
@@ -3250,8 +3239,10 @@ func TestRenameSequenceOrder(t *testing.T) {
if i.FileName() == "16" {
failed = i.SequenceNo() != secondExpectedSequence || failed
}
return true
})
}
if err := errFn(); err != nil {
t.Fatal(err)
}
if failed {
t.Fail()
}
@@ -3263,19 +3254,12 @@ func TestRenameSameFile(t *testing.T) {
m := setupModel(t, wcfg)
defer cleanupModel(m)
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
writeFile(t, ffs, "file", []byte("file"))
m.ScanFolders()
count := 0
snap := dbSnapshot(t, m, "default")
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
count++
return true
})
snap.Release()
count := countIterator[protocol.FileInfo](t)(m.LocalFiles("default", protocol.LocalDeviceID))
if count != 1 {
t.Errorf("Unexpected count: %d != %d", count, 1)
}
@@ -3288,12 +3272,10 @@ func TestRenameSameFile(t *testing.T) {
m.ScanFolders()
snap = dbSnapshot(t, m, "default")
defer snap.Release()
prevSeq := int64(0)
seen := false
snap.WithHaveSequence(0, func(i protocol.FileInfo) bool {
it, errFn := m.LocalFilesSequenced("default", protocol.LocalDeviceID, 0)
for i := range it {
if i.SequenceNo() <= prevSeq {
t.Fatalf("non-increasing sequences: %d <= %d", i.SequenceNo(), prevSeq)
}
@@ -3304,84 +3286,9 @@ func TestRenameSameFile(t *testing.T) {
seen = true
}
prevSeq = i.SequenceNo()
return true
})
}
func TestRenameEmptyFile(t *testing.T) {
wcfg, fcfg, wcfgCancel := newDefaultCfgWrapper()
defer wcfgCancel()
m := setupModel(t, wcfg)
defer cleanupModel(m)
ffs := fcfg.Filesystem(nil)
writeFile(t, ffs, "file", []byte("data"))
writeFile(t, ffs, "empty", nil)
m.ScanFolders()
snap := dbSnapshot(t, m, "default")
defer snap.Release()
empty, eok := snap.Get(protocol.LocalDeviceID, "empty")
if !eok {
t.Fatal("failed to find empty file")
}
file, fok := snap.Get(protocol.LocalDeviceID, "file")
if !fok {
t.Fatal("failed to find non-empty file")
}
count := 0
snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileInfo) bool {
count++
return true
})
if count != 0 {
t.Fatalf("Found %d entries for empty file, expected 0", count)
}
count = 0
snap.WithBlocksHash(file.BlocksHash, func(_ protocol.FileInfo) bool {
count++
return true
})
if count != 1 {
t.Fatalf("Found %d entries for non-empty file, expected 1", count)
}
must(t, ffs.Rename("file", "new-file"))
must(t, ffs.Rename("empty", "new-empty"))
// Scan
m.ScanFolders()
snap = dbSnapshot(t, m, "default")
defer snap.Release()
count = 0
snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileInfo) bool {
count++
return true
})
if count != 0 {
t.Fatalf("Found %d entries for empty file, expected 0", count)
}
count = 0
snap.WithBlocksHash(file.BlocksHash, func(i protocol.FileInfo) bool {
count++
if i.FileName() != "new-file" {
t.Fatalf("unexpected file name %s, expected new-file", i.FileName())
}
return true
})
if count != 1 {
t.Fatalf("Found %d entries for non-empty file, expected 1", count)
if err := errFn(); err != nil {
t.Fatal(err)
}
}
@@ -3391,7 +3298,7 @@ func TestBlockListMap(t *testing.T) {
m := setupModel(t, wcfg)
defer cleanupModel(m)
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
writeFile(t, ffs, "one", []byte("content"))
writeFile(t, ffs, "two", []byte("content"))
writeFile(t, ffs, "three", []byte("content"))
@@ -3400,23 +3307,25 @@ func TestBlockListMap(t *testing.T) {
m.ScanFolders()
snap := dbSnapshot(t, m, "default")
defer snap.Release()
fi, ok := snap.Get(protocol.LocalDeviceID, "one")
fi, ok, err := m.model.CurrentFolderFile("default", "one")
if err != nil {
t.Fatal(err)
}
if !ok {
t.Error("failed to find existing file")
}
var paths []string
snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileInfo) bool {
paths = append(paths, fi.FileName())
return true
})
snap.Release()
for fi, err := range itererr.Zip(m.model.AllForBlocksHash(fcfg.ID, fi.BlocksHash)) {
if err != nil {
t.Fatal(err)
}
paths = append(paths, fi.Name)
}
expected := []string{"one", "two", "three", "four", "five"}
if !equalStringsInAnyOrder(paths, expected) {
t.Errorf("expected %q got %q", expected, paths)
t.Fatalf("expected %q got %q", expected, paths)
}
// Fudge the files around
@@ -3437,19 +3346,18 @@ func TestBlockListMap(t *testing.T) {
m.ScanFolders()
// Check we're left with 2 of the 5
snap = dbSnapshot(t, m, "default")
defer snap.Release()
paths = paths[:0]
snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileInfo) bool {
paths = append(paths, fi.FileName())
return true
})
snap.Release()
for fi, err := range itererr.Zip(m.model.AllForBlocksHash(fcfg.ID, fi.BlocksHash)) {
if err != nil {
t.Fatal(err)
}
paths = append(paths, fi.Name)
}
expected = []string{"new-three", "five"}
if !equalStringsInAnyOrder(paths, expected) {
t.Errorf("expected %q got %q", expected, paths)
t.Fatalf("expected %q got %q", expected, paths)
}
}
@@ -3459,16 +3367,17 @@ func TestScanRenameCaseOnly(t *testing.T) {
m := setupModel(t, wcfg)
defer cleanupModel(m)
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
name := "foo"
writeFile(t, ffs, name, []byte("contents"))
m.ScanFolders()
snap := dbSnapshot(t, m, fcfg.ID)
defer snap.Release()
found := false
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
for i, err := range itererr.Zip(m.LocalFiles(fcfg.ID, protocol.LocalDeviceID)) {
if err != nil {
t.Fatal(err)
}
if found {
t.Fatal("got more than one file")
}
@@ -3476,21 +3385,20 @@ func TestScanRenameCaseOnly(t *testing.T) {
t.Fatalf("got file %v, expected %v", i.FileName(), name)
}
found = true
return true
})
snap.Release()
}
upper := strings.ToUpper(name)
must(t, ffs.Rename(name, upper))
m.ScanFolders()
snap = dbSnapshot(t, m, fcfg.ID)
defer snap.Release()
found = false
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
for i, err := range itererr.Zip(m.LocalFiles(fcfg.ID, protocol.LocalDeviceID)) {
if err != nil {
t.Fatal(err)
}
if i.FileName() == name {
if i.IsDeleted() {
return true
continue
}
t.Fatal("renamed file not deleted")
}
@@ -3501,8 +3409,7 @@ func TestScanRenameCaseOnly(t *testing.T) {
t.Fatal("got more than the expected files")
}
found = true
return true
})
}
}
func TestClusterConfigOnFolderAdd(t *testing.T) {
@@ -3577,7 +3484,7 @@ func TestAddFolderCompletion(t *testing.T) {
func TestScanDeletedROChangedOnSR(t *testing.T) {
m, conn, fcfg, wCancel := setupModelWithConnection(t)
ffs := fcfg.Filesystem(nil)
ffs := fcfg.Filesystem()
defer wCancel()
defer cleanupModelAndRemoveDir(m, ffs.URI())
fcfg.Type = config.FolderTypeReceiveOnly
@@ -3599,7 +3506,7 @@ func TestScanDeletedROChangedOnSR(t *testing.T) {
must(t, ffs.Remove(name))
m.ScanFolders()
if receiveOnlyChangedSize(t, m, fcfg.ID).Deleted != 1 {
if mustV(m.ReceiveOnlySize(fcfg.ID)).Deleted != 1 {
t.Fatal("expected one receive only changed deleted item")
}
@@ -3607,10 +3514,10 @@ func TestScanDeletedROChangedOnSR(t *testing.T) {
setFolder(t, m.cfg, fcfg)
m.ScanFolders()
if receiveOnlyChangedSize(t, m, fcfg.ID).Deleted != 0 {
if mustV(m.ReceiveOnlySize(fcfg.ID)).Deleted != 0 {
t.Fatal("expected no receive only changed deleted item")
}
if localSize(t, m, fcfg.ID).Deleted != 1 {
if mustV(m.LocalSize(fcfg.ID, protocol.LocalDeviceID)).Deleted != 1 {
t.Fatal("expected one local deleted item")
}
}
@@ -3682,7 +3589,7 @@ func testConfigChangeTriggersClusterConfigs(t *testing.T, expectFirst, expectSec
func TestIssue6961(t *testing.T) {
wcfg, fcfg, wcfgCancel := newDefaultCfgWrapper()
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
waiter, err := wcfg.Modify(func(cfg *config.Configuration) {
cfg.SetDevice(newDeviceConfiguration(cfg.Defaults.Device, device2, "device2"))
fcfg.Type = config.FolderTypeReceiveOnly
@@ -3693,11 +3600,6 @@ func TestIssue6961(t *testing.T) {
waiter.Wait()
// Always recalc/repair when opening a fileset.
m := newModel(t, wcfg, myID, nil)
m.db.Close()
m.db, err = db.NewLowlevel(backend.OpenMemory(), m.evLogger, db.WithRecheckInterval(time.Millisecond))
if err != nil {
t.Fatal(err)
}
m.ServeBackground()
defer cleanupModelAndRemoveDir(m, tfs.URI())
conn1 := addFakeConn(m, device1, fcfg.ID)
@@ -3752,11 +3654,9 @@ func TestIssue6961(t *testing.T) {
func TestCompletionEmptyGlobal(t *testing.T) {
m, conn, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
files := []protocol.FileInfo{{Name: "foo", Version: protocol.Vector{}.Update(myID.Short()), Sequence: 1}}
m.mut.Lock()
m.folderFiles[fcfg.ID].Update(protocol.LocalDeviceID, files)
m.mut.Unlock()
m.sdb.Update(fcfg.ID, protocol.LocalDeviceID, files)
files[0].Deleted = true
files[0].Version = files[0].Version.Update(device1.Short())
must(t, m.IndexUpdate(conn, &protocol.IndexUpdate{Folder: fcfg.ID, Files: files}))
@@ -3953,7 +3853,7 @@ func TestCCFolderNotRunning(t *testing.T) {
// Create the folder, but don't start it.
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
m := newModel(t, w, myID, nil)
defer cleanupModelAndRemoveDir(m, tfs.URI())
@@ -3990,7 +3890,7 @@ func TestPendingFolder(t *testing.T) {
Time: time.Now().Truncate(time.Second),
Label: pfolder,
}
if err := m.db.AddOrUpdatePendingFolder(pfolder, of, device2); err != nil {
if err := m.observed.AddOrUpdatePendingFolder(pfolder, of, device2); err != nil {
t.Fatal(err)
}
deviceFolders, err := m.PendingFolders(protocol.EmptyDeviceID)
@@ -4009,7 +3909,7 @@ func TestPendingFolder(t *testing.T) {
t.Fatal(err)
}
setDevice(t, w, config.DeviceConfiguration{DeviceID: device3})
if err := m.db.AddOrUpdatePendingFolder(pfolder, of, device3); err != nil {
if err := m.observed.AddOrUpdatePendingFolder(pfolder, of, device3); err != nil {
t.Fatal(err)
}
deviceFolders, err = m.PendingFolders(device2)
@@ -4060,7 +3960,7 @@ func TestDeletedNotLocallyChangedReceiveEncrypted(t *testing.T) {
func deletedNotLocallyChanged(t *testing.T, ft config.FolderType) {
w, fcfg, wCancel := newDefaultCfgWrapper()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
fcfg.Type = ft
setFolder(t, w, fcfg)
defer wCancel()
@@ -4141,3 +4041,17 @@ type modtimeTruncatingFileInfo struct {
func (fi modtimeTruncatingFileInfo) ModTime() time.Time {
return fi.FileInfo.ModTime().Truncate(fi.trunc)
}
func countIterator[T any](t *testing.T) func(it iter.Seq[T], errFn func() error) int {
return func(it iter.Seq[T], errFn func() error) int {
t.Helper()
count := 0
for range it {
count++
}
if err := errFn(); err != nil {
t.Fatal(err)
}
return count
}
}
lib/model/progressemitter.go
-5
View File
@@ -76,7 +76,6 @@ func (t *ProgressEmitter) Serve(ctx context.Context) error {
return nil
case <-t.timer.C:
t.mut.Lock()
l.Debugln("progress emitter: timer - looking after", len(t.registry))
newLastUpdated := lastUpdate
newCount = t.lenRegistryLocked()
@@ -94,8 +93,6 @@ func (t *ProgressEmitter) Serve(ctx context.Context) error {
lastCount = newCount
t.sendDownloadProgressEventLocked()
progressUpdates = t.computeProgressUpdates()
} else {
l.Debugln("progress emitter: nothing new")
}
if newCount != 0 {
@@ -247,7 +244,6 @@ func (t *ProgressEmitter) Register(s *sharedPullerState) {
t.mut.Lock()
defer t.mut.Unlock()
if t.disabled {
l.Debugln("progress emitter: disabled, skip registering")
return
}
l.Debugln("progress emitter: registering", s.folder, s.file.Name)
@@ -266,7 +262,6 @@ func (t *ProgressEmitter) Deregister(s *sharedPullerState) {
defer t.mut.Unlock()
if t.disabled {
l.Debugln("progress emitter: disabled, skip deregistering")
return
}
lib/model/queue.go
-51
View File
@@ -7,10 +7,8 @@
package model
import (
"sort"
"time"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -127,13 +125,6 @@ func (q *jobQueue) Jobs(page, perpage int) ([]string, []string, int) {
return progress, queued, (page - 1) * perpage
}
func (q *jobQueue) Shuffle() {
q.mut.Lock()
defer q.mut.Unlock()
rand.Shuffle(q.queued)
}
func (q *jobQueue) Reset() {
q.mut.Lock()
defer q.mut.Unlock()
@@ -152,45 +143,3 @@ func (q *jobQueue) lenProgress() int {
defer q.mut.Unlock()
return len(q.progress)
}
func (q *jobQueue) SortSmallestFirst() {
q.mut.Lock()
defer q.mut.Unlock()
sort.Sort(smallestFirst(q.queued))
}
func (q *jobQueue) SortLargestFirst() {
q.mut.Lock()
defer q.mut.Unlock()
sort.Sort(sort.Reverse(smallestFirst(q.queued)))
}
func (q *jobQueue) SortOldestFirst() {
q.mut.Lock()
defer q.mut.Unlock()
sort.Sort(oldestFirst(q.queued))
}
func (q *jobQueue) SortNewestFirst() {
q.mut.Lock()
defer q.mut.Unlock()
sort.Sort(sort.Reverse(oldestFirst(q.queued)))
}
// The usual sort.Interface boilerplate
type smallestFirst []jobQueueEntry
func (q smallestFirst) Len() int { return len(q) }
func (q smallestFirst) Less(a, b int) bool { return q[a].size < q[b].size }
func (q smallestFirst) Swap(a, b int) { q[a], q[b] = q[b], q[a] }
type oldestFirst []jobQueueEntry
func (q oldestFirst) Len() int { return len(q) }
func (q oldestFirst) Less(a, b int) bool { return q[a].modified < q[b].modified }
func (q oldestFirst) Swap(a, b int) { q[a], q[b] = q[b], q[a] }
lib/model/queue_test.go
-89
View File
@@ -163,95 +163,6 @@ func TestBringToFront(t *testing.T) {
}
}
func TestShuffle(t *testing.T) {
q := newJobQueue()
q.Push("f1", 0, time.Time{})
q.Push("f2", 0, time.Time{})
q.Push("f3", 0, time.Time{})
q.Push("f4", 0, time.Time{})
// This test will fail once in eight million times (1 / (4!)^5) :)
for i := 0; i < 5; i++ {
q.Shuffle()
_, queued, _ := q.Jobs(1, 100)
if l := len(queued); l != 4 {
t.Fatalf("Weird length %d returned from jobs(1, 100)", l)
}
t.Logf("%v", queued)
if _, equal := messagediff.PrettyDiff([]string{"f1", "f2", "f3", "f4"}, queued); !equal {
// The queue was shuffled
return
}
}
t.Error("Queue was not shuffled after five attempts.")
}
func TestSortBySize(t *testing.T) {
q := newJobQueue()
q.Push("f1", 20, time.Time{})
q.Push("f2", 40, time.Time{})
q.Push("f3", 30, time.Time{})
q.Push("f4", 10, time.Time{})
q.SortSmallestFirst()
_, actual, _ := q.Jobs(1, 100)
if l := len(actual); l != 4 {
t.Fatalf("Weird length %d returned from jobs(1, 100)", l)
}
expected := []string{"f4", "f1", "f3", "f2"}
if diff, equal := messagediff.PrettyDiff(expected, actual); !equal {
t.Errorf("SortSmallestFirst() diff:\n%s", diff)
}
q.SortLargestFirst()
_, actual, _ = q.Jobs(1, 100)
if l := len(actual); l != 4 {
t.Fatalf("Weird length %d returned from jobs(1, 100)", l)
}
expected = []string{"f2", "f3", "f1", "f4"}
if diff, equal := messagediff.PrettyDiff(expected, actual); !equal {
t.Errorf("SortLargestFirst() diff:\n%s", diff)
}
}
func TestSortByAge(t *testing.T) {
q := newJobQueue()
q.Push("f1", 0, time.Unix(20, 0))
q.Push("f2", 0, time.Unix(40, 0))
q.Push("f3", 0, time.Unix(30, 0))
q.Push("f4", 0, time.Unix(10, 0))
q.SortOldestFirst()
_, actual, _ := q.Jobs(1, 100)
if l := len(actual); l != 4 {
t.Fatalf("Weird length %d returned from jobs(1, 100)", l)
}
expected := []string{"f4", "f1", "f3", "f2"}
if diff, equal := messagediff.PrettyDiff(expected, actual); !equal {
t.Errorf("SortOldestFirst() diff:\n%s", diff)
}
q.SortNewestFirst()
_, actual, _ = q.Jobs(1, 100)
if l := len(actual); l != 4 {
t.Fatalf("Weird length %d returned from jobs(1, 100)", l)
}
expected = []string{"f2", "f3", "f1", "f4"}
if diff, equal := messagediff.PrettyDiff(expected, actual); !equal {
t.Errorf("SortNewestFirst() diff:\n%s", diff)
}
}
func BenchmarkJobQueueBump(b *testing.B) {
files := genFiles(10000)
lib/model/requests_test.go
+25 -30
View File
@@ -32,7 +32,7 @@ func TestRequestSimple(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
// We listen for incoming index updates and trigger when we see one for
@@ -80,7 +80,7 @@ func TestSymlinkTraversalRead(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
// We listen for incoming index updates and trigger when we see one for
// the expected test file.
@@ -123,7 +123,7 @@ func TestSymlinkTraversalWrite(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
// We listen for incoming index updates and trigger when we see one for
// the expected names.
@@ -182,7 +182,7 @@ func TestRequestCreateTmpSymlink(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
// We listen for incoming index updates and trigger when we see one for
// the expected test file.
@@ -229,7 +229,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
w, wCancel := newConfigWrapper(defaultCfgWrapper.RawCopy())
defer wCancel()
fcfg := w.FolderList()[0]
fss := fcfg.Filesystem(nil)
fss := fcfg.Filesystem()
fcfg.Type = ft
setFolder(t, w, fcfg)
m := setupModel(t, w)
@@ -358,7 +358,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
func TestIssue4841(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
received := make(chan []protocol.FileInfo)
fc.setIndexFn(func(_ context.Context, _ string, fs []protocol.FileInfo) error {
@@ -407,7 +407,7 @@ func TestIssue4841(t *testing.T) {
func TestRescanIfHaveInvalidContent(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
payload := []byte("hello")
@@ -465,9 +465,11 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
}
func TestParentDeletion(t *testing.T) {
t.Skip("flaky")
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
testFs := fcfg.Filesystem(nil)
testFs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, testFs.URI())
parent := "foo"
@@ -546,7 +548,7 @@ func TestRequestSymlinkWindows(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
received := make(chan []protocol.FileInfo)
fc.setIndexFn(func(_ context.Context, folder string, fs []protocol.FileInfo) error {
@@ -623,7 +625,7 @@ func TestRequestRemoteRenameChanged(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModel(m)
received := make(chan []protocol.FileInfo)
@@ -756,7 +758,7 @@ func TestRequestRemoteRenameChanged(t *testing.T) {
func TestRequestRemoteRenameConflict(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModel(m)
recv := make(chan int)
@@ -846,7 +848,7 @@ func TestRequestRemoteRenameConflict(t *testing.T) {
func TestRequestDeleteChanged(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
done := make(chan struct{})
@@ -960,7 +962,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
m := setupModel(t, w)
fss := fcfg.Filesystem(nil)
fss := fcfg.Filesystem()
defer cleanupModel(m)
folderIgnoresAlwaysReload(t, m, fcfg)
@@ -1054,7 +1056,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) {
func TestRequestLastFileProgress(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
done := make(chan struct{})
@@ -1089,7 +1091,7 @@ func TestRequestIndexSenderPause(t *testing.T) {
m, fc, fcfg, wcfgCancel := setupModelWithConnection(t)
defer wcfgCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
indexChan := make(chan []protocol.FileInfo)
@@ -1202,7 +1204,7 @@ func TestRequestIndexSenderPause(t *testing.T) {
func TestRequestIndexSenderClusterConfigBeforeStart(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
dir1 := "foo"
dir2 := "bar"
@@ -1217,7 +1219,7 @@ func TestRequestIndexSenderClusterConfigBeforeStart(t *testing.T) {
// Add connection (sends incoming cluster config) before starting the new model
m = &testModel{
model: NewModel(m.cfg, m.id, m.db, m.protectedFiles, m.evLogger, protocol.NewKeyGenerator()).(*model),
model: NewModel(m.cfg, m.id, m.sdb, m.protectedFiles, m.evLogger, protocol.NewKeyGenerator()).(*model),
evCancel: m.evCancel,
stopped: make(chan struct{}),
}
@@ -1269,7 +1271,7 @@ func TestRequestReceiveEncrypted(t *testing.T) {
w, fcfg, wCancel := newDefaultCfgWrapper()
defer wCancel()
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
fcfg.Type = config.FolderTypeReceiveEncrypted
setFolder(t, w, fcfg)
@@ -1281,10 +1283,7 @@ func TestRequestReceiveEncrypted(t *testing.T) {
files := genFiles(2)
files[1].LocalFlags = protocol.FlagLocalReceiveOnly
m.mut.RLock()
fset := m.folderFiles[fcfg.ID]
m.mut.RUnlock()
fset.Update(protocol.LocalDeviceID, files)
m.sdb.Update(fcfg.ID, protocol.LocalDeviceID, files)
indexChan := make(chan []protocol.FileInfo, 10)
done := make(chan struct{})
@@ -1376,7 +1375,7 @@ func TestRequestGlobalInvalidToValid(t *testing.T) {
must(t, err)
waiter.Wait()
conn := addFakeConn(m, device2, fcfg.ID)
tfs := fcfg.Filesystem(nil)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
indexChan := make(chan []protocol.FileInfo, 1)
@@ -1402,7 +1401,7 @@ func TestRequestGlobalInvalidToValid(t *testing.T) {
file.SetIgnored()
m.IndexUpdate(conn, &protocol.IndexUpdate{Folder: fcfg.ID, Files: []protocol.FileInfo{prepareFileInfoForIndex(file)}})
// Wait for the ignored file to be received and possible pulled
// Wait for the ignored file to be received and possibly pulled
timeout := time.After(10 * time.Second)
globalUpdated := false
for {
@@ -1422,13 +1421,9 @@ func TestRequestGlobalInvalidToValid(t *testing.T) {
}
globalUpdated = true
}
snap, err := m.DBSnapshot(fcfg.ID)
if err != nil {
if s, err := m.NeedSize(fcfg.ID, protocol.LocalDeviceID); err != nil {
t.Fatal(err)
}
need := snap.NeedSize(protocol.LocalDeviceID)
snap.Release()
if need.Files == 0 {
} else if s.Files == 0 {
break
}
}
lib/model/testos_test.go
+4 -8
View File
@@ -6,10 +6,6 @@
package model
import (
"github.com/syncthing/syncthing/lib/fs"
)
// fatal is the required common interface between *testing.B and *testing.T
type fatal interface {
Fatal(...interface{})
@@ -23,9 +19,9 @@ func must(f fatal, err error) {
}
}
func mustRemove(f fatal, err error) {
f.Helper()
if err != nil && !fs.IsNotExist(err) {
f.Fatal(err)
func mustV[T any](v T, err error) T {
if err != nil {
panic(err)
}
return v
}
lib/model/testutils_test.go
+14 -75
View File
@@ -12,9 +12,8 @@ import (
"testing"
"time"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
@@ -149,11 +148,14 @@ type testModel struct {
func newModel(t testing.TB, cfg config.Wrapper, id protocol.DeviceID, protectedFiles []string) *testModel {
t.Helper()
evLogger := events.NewLogger()
ldb, err := db.NewLowlevel(backend.OpenMemory(), evLogger)
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
m := NewModel(cfg, id, ldb, protectedFiles, evLogger, protocol.NewKeyGenerator()).(*model)
t.Cleanup(func() {
mdb.Close()
})
m := NewModel(cfg, id, mdb, protectedFiles, evLogger, protocol.NewKeyGenerator()).(*model)
ctx, cancel := context.WithCancel(context.Background())
go evLogger.Serve(ctx)
return &testModel{
@@ -174,12 +176,6 @@ func (m *testModel) ServeBackground() {
<-m.started
}
func (m *testModel) testAvailability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
av, err := m.model.Availability(folder, file, block)
must(m.t, err)
return av
}
func (m *testModel) testCurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
f, ok, err := m.model.CurrentFolderFile(folder, file)
must(m.t, err)
@@ -198,7 +194,7 @@ func cleanupModel(m *testModel) {
<-m.stopped
}
m.evCancel()
m.db.Close()
m.sdb.Close()
os.Remove(m.cfg.ConfigPath())
}
@@ -240,52 +236,6 @@ func (*alwaysChanged) Changed() bool {
return true
}
func localSize(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.LocalSize()
}
func globalSize(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.GlobalSize()
}
func receiveOnlyChangedSize(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.ReceiveOnlyChangedSize()
}
func needSizeLocal(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.NeedSize(protocol.LocalDeviceID)
}
func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot {
t.Helper()
snap, err := m.DBSnapshot(folder)
if err != nil {
t.Fatal(err)
}
return snap
}
func fsetSnapshot(t *testing.T, fset *db.FileSet) *db.Snapshot {
t.Helper()
snap, err := fset.Snapshot()
if err != nil {
t.Fatal(err)
}
return snap
}
// Reach in and update the ignore matcher to one that always does
// reloads when asked to, instead of checking file mtimes. This is
// because we will be changing the files on disk often enough that the
@@ -293,10 +243,9 @@ func fsetSnapshot(t *testing.T, fset *db.FileSet) *db.Snapshot {
func folderIgnoresAlwaysReload(t testing.TB, m *testModel, fcfg config.FolderConfiguration) {
t.Helper()
m.removeFolder(fcfg)
fset := newFileSet(t, fcfg.ID, m.db)
ignores := ignore.New(fcfg.Filesystem(nil), ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
ignores := ignore.New(fcfg.Filesystem(), ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
m.mut.Lock()
m.addAndStartFolderLockedWithIgnores(fcfg, fset, ignores)
m.addAndStartFolderLockedWithIgnores(fcfg, ignores)
m.mut.Unlock()
}
@@ -319,12 +268,11 @@ func basicClusterConfig(local, remote protocol.DeviceID, folders ...string) *pro
}
func localIndexUpdate(m *testModel, folder string, fs []protocol.FileInfo) {
m.mut.RLock()
fset := m.folderFiles[folder]
m.mut.RUnlock()
fset.Update(protocol.LocalDeviceID, fs)
seq := fset.Sequence(protocol.LocalDeviceID)
m.sdb.Update(folder, protocol.LocalDeviceID, fs)
seq, err := m.sdb.GetDeviceSequence(folder, protocol.LocalDeviceID)
if err != nil {
panic(err)
}
filenames := make([]string, len(fs))
for i, file := range fs {
filenames[i] = file.Name
@@ -345,15 +293,6 @@ func newDeviceConfiguration(defaultCfg config.DeviceConfiguration, id protocol.D
return cfg
}
func newFileSet(t testing.TB, folder string, ldb *db.Lowlevel) *db.FileSet {
t.Helper()
fset, err := db.NewFileSet(folder, ldb)
if err != nil {
t.Fatal(err)
}
return fset
}
func replace(t testing.TB, w config.Wrapper, to config.Configuration) {
t.Helper()
waiter, err := w.Modify(func(cfg *config.Configuration) {
lib/protocol/bep_fileinfo.go
+7 -5
View File
@@ -19,10 +19,12 @@ import (
// FileInfo.LocalFlags flags
const (
FlagLocalUnsupported = 1 << 0 // The kind is unsupported, e.g. symlinks on Windows
FlagLocalIgnored = 1 << 1 // Matches local ignore patterns
FlagLocalMustRescan = 1 << 2 // Doesn't match content on disk, must be rechecked fully
FlagLocalReceiveOnly = 1 << 3 // Change detected on receive only folder
FlagLocalUnsupported = 1 << 0 // 1: The kind is unsupported, e.g. symlinks on Windows
FlagLocalIgnored = 1 << 1 // 2: Matches local ignore patterns
FlagLocalMustRescan = 1 << 2 // 4: Doesn't match content on disk, must be rechecked fully
FlagLocalReceiveOnly = 1 << 3 // 8: Change detected on receive only folder
FlagLocalGlobal = 1 << 4 // 16: This is the global file version
FlagLocalNeeded = 1 << 5 // 32: We need this file
// Flags that should result in the Invalid bit on outgoing updates
LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly
@@ -32,7 +34,7 @@ const (
// disk.
LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly
LocalAllFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly
LocalAllFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalGlobal | FlagLocalNeeded
)
// BlockSizes is the list of valid block sizes, from min to max
lib/protocol/protocol_test.go
-24
View File
@@ -10,10 +10,8 @@ import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"io"
"os"
"sync"
"testing"
"time"
@@ -280,28 +278,6 @@ func TestUnmarshalFDPUv16v17(t *testing.T) {
}
}
func testMarshal(t *testing.T, prefix string, m1, m2 proto.Message) bool {
buf, err := proto.Marshal(m1)
if err != nil {
t.Fatal(err)
}
err = proto.Unmarshal(buf, m2)
if err != nil {
t.Fatal(err)
}
bs1, _ := json.MarshalIndent(m1, "", " ")
bs2, _ := json.MarshalIndent(m2, "", " ")
if !bytes.Equal(bs1, bs2) {
os.WriteFile(prefix+"-1.txt", bs1, 0o644)
os.WriteFile(prefix+"-2.txt", bs2, 0o644)
return false
}
return true
}
func TestWriteCompressed(t *testing.T) {
for _, random := range []bool{false, true} {
buf := new(bytes.Buffer)
lib/protocol/vector.go
+41
View File
@@ -7,6 +7,11 @@
package protocol
import (
"encoding/binary"
"encoding/hex"
"fmt"
"strconv"
"strings"
"time"
"github.com/syncthing/syncthing/internal/gen/bep"
@@ -20,6 +25,17 @@ type Vector struct {
Counters []Counter
}
func (v *Vector) String() string {
var buf strings.Builder
for i, c := range v.Counters {
if i > 0 {
buf.WriteRune(',')
}
fmt.Fprintf(&buf, "%x:%d", c.ID, c.Value)
}
return buf.String()
}
func (v *Vector) ToWire() *bep.Vector {
counters := make([]*bep.Counter, len(v.Counters))
for i, c := range v.Counters {
@@ -42,6 +58,31 @@ func VectorFromWire(w *bep.Vector) Vector {
return v
}
func VectorFromString(s string) (Vector, error) {
pairs := strings.Split(s, ",")
var v Vector
v.Counters = make([]Counter, len(pairs))
for i, pair := range pairs {
idStr, valStr, ok := strings.Cut(pair, ":")
if !ok {
return Vector{}, fmt.Errorf("bad pair %q", pair)
}
idslice, err := hex.DecodeString(idStr)
if err != nil {
return Vector{}, fmt.Errorf("bad id in pair %q", pair)
}
var idbs [8]byte
copy(idbs[8-len(idslice):], idslice)
id := binary.BigEndian.Uint64(idbs[:])
val, err := strconv.ParseUint(valStr, 10, 64)
if err != nil {
return Vector{}, fmt.Errorf("bad val in pair %q", pair)
}
v.Counters[i] = Counter{ID: ShortID(id), Value: val}
}
return v, nil
}
// Counter represents a single counter in the version vector.
type Counter struct {
ID ShortID
lib/relay/protocol/packets_xdr.go
+19 -28
View File
@@ -31,7 +31,7 @@ struct header {
*/
func (header) XDRSize() int {
func (o header) XDRSize() int {
return 4 + 4 + 4
}
@@ -60,7 +60,6 @@ func (o *header) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *header) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.magic = u.UnmarshalUint32()
o.messageType = int32(u.UnmarshalUint32())
@@ -79,27 +78,26 @@ struct Ping {
*/
func (Ping) XDRSize() int {
func (o Ping) XDRSize() int {
return 0
}
func (Ping) MarshalXDR() ([]byte, error) {
func (o Ping) MarshalXDR() ([]byte, error) {
return nil, nil
}
func (Ping) MustMarshalXDR() []byte {
func (o Ping) MustMarshalXDR() []byte {
return nil
}
func (Ping) MarshalXDRInto(_ *xdr.Marshaller) error {
func (o Ping) MarshalXDRInto(m *xdr.Marshaller) error {
return nil
}
func (*Ping) UnmarshalXDR(_ []byte) error {
func (o *Ping) UnmarshalXDR(bs []byte) error {
return nil
}
func (*Ping) UnmarshalXDRFrom(_ *xdr.Unmarshaller) error {
func (o *Ping) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
return nil
}
@@ -114,27 +112,26 @@ struct Pong {
*/
func (Pong) XDRSize() int {
func (o Pong) XDRSize() int {
return 0
}
func (Pong) MarshalXDR() ([]byte, error) {
func (o Pong) MarshalXDR() ([]byte, error) {
return nil, nil
}
func (Pong) MustMarshalXDR() []byte {
func (o Pong) MustMarshalXDR() []byte {
return nil
}
func (Pong) MarshalXDRInto(_ *xdr.Marshaller) error {
func (o Pong) MarshalXDRInto(m *xdr.Marshaller) error {
return nil
}
func (*Pong) UnmarshalXDR(_ []byte) error {
func (o *Pong) UnmarshalXDR(bs []byte) error {
return nil
}
func (*Pong) UnmarshalXDRFrom(_ *xdr.Unmarshaller) error {
func (o *Pong) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
return nil
}
@@ -149,27 +146,26 @@ struct RelayFull {
*/
func (RelayFull) XDRSize() int {
func (o RelayFull) XDRSize() int {
return 0
}
func (RelayFull) MarshalXDR() ([]byte, error) {
func (o RelayFull) MarshalXDR() ([]byte, error) {
return nil, nil
}
func (RelayFull) MustMarshalXDR() []byte {
func (o RelayFull) MustMarshalXDR() []byte {
return nil
}
func (RelayFull) MarshalXDRInto(_ *xdr.Marshaller) error {
func (o RelayFull) MarshalXDRInto(m *xdr.Marshaller) error {
return nil
}
func (*RelayFull) UnmarshalXDR(_ []byte) error {
func (o *RelayFull) UnmarshalXDR(bs []byte) error {
return nil
}
func (*RelayFull) UnmarshalXDRFrom(_ *xdr.Unmarshaller) error {
func (o *RelayFull) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
return nil
}
@@ -219,7 +215,6 @@ func (o *JoinRelayRequest) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *JoinRelayRequest) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.Token = u.UnmarshalString()
return u.Error
@@ -274,7 +269,6 @@ func (o *JoinSessionRequest) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *JoinSessionRequest) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.Key = u.UnmarshalBytesMax(32)
return u.Error
@@ -331,7 +325,6 @@ func (o *Response) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *Response) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.Code = int32(u.UnmarshalUint32())
o.Message = u.UnmarshalString()
@@ -387,7 +380,6 @@ func (o *ConnectRequest) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *ConnectRequest) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.ID = u.UnmarshalBytesMax(32)
return u.Error
@@ -470,7 +462,6 @@ func (o *SessionInvitation) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *SessionInvitation) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.From = u.UnmarshalBytesMax(32)
o.Key = u.UnmarshalBytesMax(32)
lib/stats/debug.go
-13
View File
@@ -1,13 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package stats
import (
"github.com/syncthing/syncthing/lib/logger"
)
var l = logger.DefaultLogger.NewFacility("stats", "Persistent device and folder statistics")
lib/stats/device.go
+8 -16
View File
@@ -9,9 +9,7 @@ package stats
import (
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/internal/db"
)
const (
@@ -25,19 +23,17 @@ type DeviceStatistics struct {
}
type DeviceStatisticsReference struct {
ns *db.NamespacedKV
device protocol.DeviceID
kv *db.Typed
}
func NewDeviceStatisticsReference(dba backend.Backend, device protocol.DeviceID) *DeviceStatisticsReference {
func NewDeviceStatisticsReference(kv *db.Typed) *DeviceStatisticsReference {
return &DeviceStatisticsReference{
ns: db.NewDeviceStatisticsNamespace(dba, device.String()),
device: device,
kv: kv,
}
}
func (s *DeviceStatisticsReference) GetLastSeen() (time.Time, error) {
t, ok, err := s.ns.Time(lastSeenKey)
t, ok, err := s.kv.Time(lastSeenKey)
if err != nil {
return time.Time{}, err
} else if !ok {
@@ -45,29 +41,25 @@ func (s *DeviceStatisticsReference) GetLastSeen() (time.Time, error) {
// time.Time{} from s.ns
return time.Unix(0, 0), nil
}
l.Debugln("stats.DeviceStatisticsReference.GetLastSeen:", s.device, t)
return t, nil
}
func (s *DeviceStatisticsReference) GetLastConnectionDuration() (time.Duration, error) {
d, ok, err := s.ns.Int64(connDurationKey)
d, ok, err := s.kv.Int64(connDurationKey)
if err != nil {
return 0, err
} else if !ok {
return 0, nil
}
l.Debugln("stats.DeviceStatisticsReference.GetLastConnectionDuration:", s.device, d)
return time.Duration(d), nil
}
func (s *DeviceStatisticsReference) WasSeen() error {
l.Debugln("stats.DeviceStatisticsReference.WasSeen:", s.device)
return s.ns.PutTime(lastSeenKey, time.Now().Truncate(time.Second))
return s.kv.PutTime(lastSeenKey, time.Now().Truncate(time.Second))
}
func (s *DeviceStatisticsReference) LastConnectionDuration(d time.Duration) error {
l.Debugln("stats.DeviceStatisticsReference.LastConnectionDuration:", s.device, d)
return s.ns.PutInt64(connDurationKey, d.Nanoseconds())
return s.kv.PutInt64(connDurationKey, d.Nanoseconds())
}
func (s *DeviceStatisticsReference) GetStatistics() (DeviceStatistics, error) {
lib/stats/folder.go
+12 -15
View File
@@ -9,7 +9,7 @@ package stats
import (
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/internal/db"
)
type FolderStatistics struct {
@@ -18,8 +18,7 @@ type FolderStatistics struct {
}
type FolderStatisticsReference struct {
ns *db.NamespacedKV
folder string
kv *db.Typed
}
type LastFile struct {
@@ -28,27 +27,26 @@ type LastFile struct {
Deleted bool `json:"deleted"`
}
func NewFolderStatisticsReference(ldb *db.Lowlevel, folder string) *FolderStatisticsReference {
func NewFolderStatisticsReference(kv *db.Typed) *FolderStatisticsReference {
return &FolderStatisticsReference{
ns: db.NewFolderStatisticsNamespace(ldb, folder),
folder: folder,
kv: kv,
}
}
func (s *FolderStatisticsReference) GetLastFile() (LastFile, error) {
at, ok, err := s.ns.Time("lastFileAt")
at, ok, err := s.kv.Time("lastFileAt")
if err != nil {
return LastFile{}, err
} else if !ok {
return LastFile{}, nil
}
file, ok, err := s.ns.String("lastFileName")
file, ok, err := s.kv.String("lastFileName")
if err != nil {
return LastFile{}, err
} else if !ok {
return LastFile{}, nil
}
deleted, _, err := s.ns.Bool("lastFileDeleted")
deleted, _, err := s.kv.Bool("lastFileDeleted")
if err != nil {
return LastFile{}, err
}
@@ -60,25 +58,24 @@ func (s *FolderStatisticsReference) GetLastFile() (LastFile, error) {
}
func (s *FolderStatisticsReference) ReceivedFile(file string, deleted bool) error {
l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder, file)
if err := s.ns.PutTime("lastFileAt", time.Now().Truncate(time.Second)); err != nil {
if err := s.kv.PutTime("lastFileAt", time.Now().Truncate(time.Second)); err != nil {
return err
}
if err := s.ns.PutString("lastFileName", file); err != nil {
if err := s.kv.PutString("lastFileName", file); err != nil {
return err
}
if err := s.ns.PutBool("lastFileDeleted", deleted); err != nil {
if err := s.kv.PutBool("lastFileDeleted", deleted); err != nil {
return err
}
return nil
}
func (s *FolderStatisticsReference) ScanCompleted() error {
return s.ns.PutTime("lastScan", time.Now().Truncate(time.Second))
return s.kv.PutTime("lastScan", time.Now().Truncate(time.Second))
}
func (s *FolderStatisticsReference) GetLastScanTime() (time.Time, error) {
lastScan, ok, err := s.ns.Time("lastScan")
lastScan, ok, err := s.kv.Time("lastScan")
if err != nil {
return time.Time{}, err
} else if !ok {
lib/stats/stats_test.go
+10 -5
View File
@@ -13,15 +13,20 @@ import (
"testing"
"time"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/db/sqlite"
)
func TestDeviceStat(t *testing.T) {
db := backend.OpenLevelDBMemory()
defer db.Close()
sdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
sdb.Close()
})
sr := NewDeviceStatisticsReference(db, protocol.LocalDeviceID)
sr := NewDeviceStatisticsReference(db.NewTyped(sdb, "devstatref"))
if err := sr.WasSeen(); err != nil {
t.Fatal(err)
}
lib/syncthing/globalmigrations.go
+4 -103
View File
@@ -7,13 +7,8 @@
package syncthing
import (
"encoding/binary"
"io"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
const (
@@ -21,8 +16,8 @@ const (
globalMigrationDBKey = "globalMigrationVersion"
)
func globalMigration(ll *db.Lowlevel, cfg config.Wrapper) error {
miscDB := db.NewMiscDataNamespace(ll)
func globalMigration(kv db.KV, cfg config.Wrapper) error {
miscDB := db.NewMiscDB(kv)
prevVersion, _, err := miscDB.Int64(globalMigrationDBKey)
if err != nil {
return err
@@ -32,101 +27,7 @@ func globalMigration(ll *db.Lowlevel, cfg config.Wrapper) error {
return nil
}
if prevVersion < 1 {
if err := encryptionTrailerSizeMigration(ll, cfg); err != nil {
return err
}
}
// currently no migrations
return miscDB.PutInt64(globalMigrationDBKey, globalMigrationVersion)
}
func encryptionTrailerSizeMigration(ll *db.Lowlevel, cfg config.Wrapper) error {
encFolders := cfg.Folders()
for folderID, folderCfg := range cfg.Folders() {
if folderCfg.Type != config.FolderTypeReceiveEncrypted {
delete(encFolders, folderID)
}
}
if len(encFolders) == 0 {
return nil
}
l.Infoln("Running global migration to fix encryption file sizes")
// Trigger index re-transfer with fixed up sizes
db.DropDeltaIndexIDs(ll)
for folderID, folderCfg := range encFolders {
fset, err := db.NewFileSet(folderID, ll)
if err != nil {
return err
}
snap, err := fset.Snapshot()
if err != nil {
return err
}
batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
// As we can't touch the version, we need to first invalidate the
// files, and then re-add the modified valid files
invalidFiles := make([]protocol.FileInfo, len(files))
for i, f := range files {
f.SetUnsupported()
invalidFiles[i] = f
}
fset.Update(protocol.LocalDeviceID, invalidFiles)
fset.Update(protocol.LocalDeviceID, files)
return nil
})
filesystem := folderCfg.Filesystem(fset)
var innerErr error
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
size, err := sizeOfEncryptedTrailer(filesystem, fi.Name)
if err != nil {
// Best effort: If we fail to read a file, it will show as
// locally changed on next scan.
return true
}
fi.EncryptionTrailerSize = size
batch.Append(fi)
err = batch.FlushIfFull()
if err != nil {
innerErr = err
return false
}
return true
})
snap.Release()
if innerErr != nil {
return innerErr
}
err = batch.Flush()
if err != nil {
return err
}
}
return nil
}
// sizeOfEncryptedTrailer returns the size of the encrypted trailer on disk.
// This amount of bytes should be subtracted from the file size to get the
// original file size.
func sizeOfEncryptedTrailer(fs fs.Filesystem, name string) (int, error) {
f, err := fs.Open(name)
if err != nil {
return 0, err
}
defer f.Close()
if _, err := f.Seek(-4, io.SeekEnd); err != nil {
return 0, err
}
var buf [4]byte
if _, err := io.ReadFull(f, buf[:]); err != nil {
return 0, err
}
// The stored size is the size of the encrypted data.
size := int(binary.BigEndian.Uint32(buf[:]))
// We add the size of the length word itself as well.
return size + 4, nil
}
lib/syncthing/internals.go
+32 -5
View File
@@ -8,9 +8,10 @@ package syncthing
import (
"context"
"iter"
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stats"
@@ -23,6 +24,8 @@ type Internals struct {
model model.Model
}
type Counts = db.Counts
func newInternals(model model.Model) *Internals {
return &Internals{
model: model,
@@ -77,14 +80,38 @@ func (m *Internals) PendingFolders(deviceID protocol.DeviceID) (map[string]db.Pe
return m.model.PendingFolders(deviceID)
}
func (m *Internals) DBSnapshot(folderID string) (*db.Snapshot, error) {
return m.model.DBSnapshot(folderID)
}
func (m *Internals) ScanFolderSubdirs(folderID string, paths []string) error {
return m.model.ScanFolderSubdirs(folderID, paths)
}
func (m *Internals) GlobalSize(folder string) (Counts, error) {
counts, err := m.model.GlobalSize(folder)
if err != nil {
return Counts{}, err
}
return counts, nil
}
func (m *Internals) LocalSize(folder string) (Counts, error) {
counts, err := m.model.LocalSize(folder, protocol.LocalDeviceID)
if err != nil {
return Counts{}, err
}
return counts, nil
}
func (m *Internals) NeedSize(folder string, device protocol.DeviceID) (Counts, error) {
counts, err := m.model.NeedSize(folder, device)
if err != nil {
return Counts{}, err
}
return counts, nil
}
func (m *Internals) AllGlobalFiles(folder string) (iter.Seq[db.FileMetadata], func() error) {
return m.model.AllGlobalFiles(folder)
}
func (m *Internals) FolderProgressBytesCompleted(folder string) int64 {
return m.model.FolderProgressBytesCompleted(folder)
}
lib/syncthing/syncthing.go
+33 -34
View File
@@ -22,13 +22,12 @@ import (
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/api"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/connections/registry"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/locations"
@@ -52,21 +51,19 @@ const (
)
type Options struct {
AuditWriter io.Writer
NoUpgrade bool
ProfilerAddr string
ResetDeltaIdxs bool
Verbose bool
// null duration means use default value
DBRecheckInterval time.Duration
DBIndirectGCInterval time.Duration
AuditWriter io.Writer
NoUpgrade bool
ProfilerAddr string
ResetDeltaIdxs bool
Verbose bool
DBMaintenanceInterval time.Duration
}
type App struct {
myID protocol.DeviceID
mainService *suture.Supervisor
cfg config.Wrapper
ll *db.Lowlevel
sdb db.DB
evLogger events.Logger
cert tls.Certificate
opts Options
@@ -80,14 +77,10 @@ type App struct {
Internals *Internals
}
func New(cfg config.Wrapper, dbBackend backend.Backend, evLogger events.Logger, cert tls.Certificate, opts Options) (*App, error) {
ll, err := db.NewLowlevel(dbBackend, evLogger, db.WithRecheckInterval(opts.DBRecheckInterval), db.WithIndirectGCInterval(opts.DBIndirectGCInterval))
if err != nil {
return nil, err
}
func New(cfg config.Wrapper, sdb db.DB, evLogger events.Logger, cert tls.Certificate, opts Options) (*App, error) {
a := &App{
cfg: cfg,
ll: ll,
sdb: sdb,
evLogger: evLogger,
opts: opts,
cert: cert,
@@ -124,7 +117,7 @@ func (a *App) Start() error {
func (a *App) startup() error {
a.mainService.Add(ur.NewFailureHandler(a.cfg, a.evLogger))
a.mainService.Add(a.ll)
a.mainService.Add(a.sdb.Service(a.opts.DBMaintenanceInterval))
if a.opts.AuditWriter != nil {
a.mainService.Add(newAuditService(a.opts.AuditWriter, a.evLogger))
@@ -180,14 +173,12 @@ func (a *App) startup() error {
perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond)
l.Infof("Hashing performance is %.02f MB/s", perf)
if err := db.UpdateSchema(a.ll); err != nil {
l.Warnln("Database schema:", err)
return err
}
if a.opts.ResetDeltaIdxs {
l.Infoln("Reinitializing delta index IDs")
db.DropDeltaIndexIDs(a.ll)
if err := a.sdb.DropAllIndexIDs(); err != nil {
l.Warnln("Drop index IDs:", err)
return err
}
}
protectedFiles := []string{
@@ -198,17 +189,22 @@ func (a *App) startup() error {
}
// Remove database entries for folders that no longer exist in the config
folders := a.cfg.Folders()
for _, folder := range a.ll.ListFolders() {
if _, ok := folders[folder]; !ok {
cfgFolders := a.cfg.Folders()
dbFolders, err := a.sdb.ListFolders()
if err != nil {
l.Warnln("Listing folders:", err)
return err
}
for _, folder := range dbFolders {
if _, ok := cfgFolders[folder]; !ok {
l.Infof("Cleaning metadata for dropped folder %q", folder)
db.DropFolder(a.ll, folder)
a.sdb.DropFolder(folder)
}
}
// Grab the previously running version string from the database.
miscDB := db.NewMiscDataNamespace(a.ll)
miscDB := db.NewMiscDB(a.sdb)
prevVersion, _, err := miscDB.String("prevVersion")
if err != nil {
l.Warnln("Database:", err)
@@ -229,7 +225,10 @@ func (a *App) startup() error {
if a.cfg.Options().SendFullIndexOnUpgrade {
// Drop delta indexes in case we've changed random stuff we
// shouldn't have. We will resend our index on next connect.
db.DropDeltaIndexIDs(a.ll)
if err := a.sdb.DropAllIndexIDs(); err != nil {
l.Warnln("Drop index IDs:", err)
return err
}
}
}
@@ -238,13 +237,13 @@ func (a *App) startup() error {
miscDB.PutString("prevVersion", build.Version)
}
if err := globalMigration(a.ll, a.cfg); err != nil {
if err := globalMigration(a.sdb, a.cfg); err != nil {
l.Warnln("Global migration:", err)
return err
}
keyGen := protocol.NewKeyGenerator()
m := model.NewModel(a.cfg, a.myID, a.ll, protectedFiles, a.evLogger, keyGen)
m := model.NewModel(a.cfg, a.myID, a.sdb, protectedFiles, a.evLogger, keyGen)
a.Internals = newInternals(m)
a.mainService.Add(m)
@@ -333,7 +332,7 @@ func (a *App) wait(errChan <-chan error) {
done := make(chan struct{})
go func() {
a.ll.Close()
a.sdb.Close()
close(done)
}()
select {
@@ -399,7 +398,7 @@ func (a *App) stopWithErr(stopReason svcutil.ExitStatus, err error) svcutil.Exit
return a.exitStatus
}
func (a *App) setupGUI(m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, errors, systemLog logger.Recorder, miscDB *db.NamespacedKV) error {
func (a *App) setupGUI(m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, errors, systemLog logger.Recorder, miscDB *db.Typed) error {
guiCfg := a.cfg.GUI()
if !guiCfg.Enabled {
lib/syncthing/syncthing_test.go
+12 -6
View File
@@ -8,11 +8,12 @@ package syncthing
import (
"os"
"strings"
"testing"
"time"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/svcutil"
@@ -71,8 +72,14 @@ func TestStartupFail(t *testing.T) {
}, protocol.LocalDeviceID, events.NoopLogger)
defer os.Remove(cfg.ConfigPath())
db := backend.OpenMemory()
app, err := New(cfg, db, events.NoopLogger, cert, Options{})
sdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
sdb.Close()
})
app, err := New(cfg, sdb, events.NoopLogger, cert, Options{})
if err != nil {
t.Fatal(err)
}
@@ -102,10 +109,9 @@ func TestStartupFail(t *testing.T) {
t.Errorf(`Got different errors "%v" from Start and "%v" from Error`, startErr, err)
}
if trans, err := db.NewReadTransaction(); err == nil {
if _, err := sdb.ListFolders(); err == nil {
t.Error("Expected error due to db being closed, got nil")
trans.Release()
} else if !backend.IsClosed(err) {
} else if !strings.Contains(err.Error(), "closed") {
t.Error("Expected error due to db being closed, got", err)
}
}
lib/syncthing/utils.go
+129 -3
View File
@@ -12,9 +12,16 @@ import (
"fmt"
"io"
"os"
"sync"
"time"
"github.com/syncthing/syncthing/internal/db"
newdb "github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/db/olddb"
"github.com/syncthing/syncthing/internal/db/olddb/backend"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/locations"
@@ -150,6 +157,125 @@ func copyFile(src, dst string) error {
return nil
}
func OpenDBBackend(path string, tuning config.Tuning) (backend.Backend, error) {
return backend.Open(path, backend.Tuning(tuning))
// Opens a database
func OpenDatabase(path string) (newdb.DB, error) {
sql, err := sqlite.Open(path)
if err != nil {
return nil, err
}
sdb := newdb.MetricsWrap(sql)
return sdb, nil
}
// Attempts migration of the old (LevelDB-based) database type to the new (SQLite-based) type
func TryMigrateDatabase() error {
oldDBDir := locations.Get(locations.LegacyDatabase)
if _, err := os.Lstat(oldDBDir); err != nil {
// No old database
return nil
}
be, err := backend.OpenLevelDBRO(oldDBDir)
if err != nil {
// Apparently, not a valid old database
return nil
}
sdb, err := sqlite.OpenForMigration(locations.Get(locations.Database))
if err != nil {
return err
}
miscDB := db.NewMiscDB(sdb)
if when, ok, err := miscDB.Time("migrated-from-leveldb-at"); err == nil && ok {
l.Warnf("Old-style database present but already migrated at %v; please manually move or remove %s.", when, oldDBDir)
return nil
}
l.Infoln("Migrating old-style database to SQLite; this may take a while...")
t0 := time.Now()
ll, err := olddb.NewLowlevel(be)
if err != nil {
return err
}
totFiles, totBlocks := 0, 0
for _, folder := range ll.ListFolders() {
// Start a writer routine
fis := make(chan protocol.FileInfo, 50)
var writeErr error
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
var batch []protocol.FileInfo
files, blocks := 0, 0
t0 := time.Now()
t1 := time.Now()
for fi := range fis {
batch = append(batch, fi)
files++
blocks += len(fi.Blocks)
if len(batch) == 1000 {
writeErr = sdb.Update(folder, protocol.LocalDeviceID, batch)
if writeErr != nil {
return
}
batch = batch[:0]
if time.Since(t1) > 10*time.Second {
d := time.Since(t0) + 1
t1 = time.Now()
l.Infof("Migrating folder %s... (%d files and %dk blocks in %v, %.01f files/s)", folder, files, blocks/1000, d.Truncate(time.Second), float64(files)/d.Seconds())
}
}
}
if len(batch) > 0 {
writeErr = sdb.Update(folder, protocol.LocalDeviceID, batch)
}
d := time.Since(t0) + 1
l.Infof("Migrated folder %s; %d files and %dk blocks in %v, %.01f files/s", folder, files, blocks/1000, d.Truncate(time.Second), float64(files)/d.Seconds())
totFiles += files
totBlocks += blocks
}()
// Iterate the existing files
fs, err := olddb.NewFileSet(folder, ll)
if err != nil {
return err
}
snap, err := fs.Snapshot()
if err != nil {
return err
}
_ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool {
fis <- fi
return true
})
close(fis)
snap.Release()
// Wait for writes to complete
wg.Wait()
if writeErr != nil {
return writeErr
}
}
l.Infoln("Migrating virtual mtimes...")
if err := ll.IterateMtimes(sdb.PutMtime); err != nil {
l.Warnln("Failed to migrate mtimes:", err)
}
_ = miscDB.PutTime("migrated-from-leveldb-at", time.Now())
_ = miscDB.PutString("migrated-from-leveldb-by", build.LongVersion)
be.Close()
sdb.Close()
_ = os.Rename(oldDBDir, oldDBDir+"-migrated")
l.Infof("Migration complete, %d files and %dk blocks in %s", totFiles, totBlocks/1000, time.Since(t0).Truncate(time.Second))
return nil
}
lib/ur/usage_report.go
+4 -6
View File
@@ -22,10 +22,10 @@ import (
"time"
"github.com/shirou/gopsutil/v4/process"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
@@ -41,7 +41,7 @@ const Version = 3
var StartTime = time.Now().Truncate(time.Second)
type Model interface {
DBSnapshot(folder string) (*db.Snapshot, error)
GlobalSize(folder string) (db.Counts, error)
UsageReportingStats(report *contract.Report, version int, preview bool)
}
@@ -83,12 +83,10 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
var totFiles, maxFiles int
var totBytes, maxBytes int64
for folderID := range s.cfg.Folders() {
snap, err := s.model.DBSnapshot(folderID)
global, err := s.model.GlobalSize(folderID)
if err != nil {
continue
return nil, err
}
global := snap.GlobalSize()
snap.Release()
totFiles += int(global.Files)
totBytes += global.Bytes
if int(global.Files) > maxFiles {
lib/versioner/external.go
+1 -1
View File
@@ -41,7 +41,7 @@ func newExternal(cfg config.FolderConfiguration) Versioner {
s := external{
command: command,
filesystem: cfg.Filesystem(nil),
filesystem: cfg.Filesystem(),
}
l.Debugf("instantiated %#v", s)
lib/versioner/simple.go
+1 -1
View File
@@ -41,7 +41,7 @@ func newSimple(cfg config.FolderConfiguration) Versioner {
s := simple{
keep: keep,
cleanoutDays: cleanoutDays,
folderFs: cfg.Filesystem(nil),
folderFs: cfg.Filesystem(),
versionsFs: versionerFsFromFolderCfg(cfg),
copyRangeMethod: cfg.CopyRangeMethod.ToFS(),
}
lib/versioner/simple_test.go
+2 -2
View File
@@ -64,7 +64,7 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
},
},
}
fs := cfg.Filesystem(nil)
fs := cfg.Filesystem()
v := newSimple(cfg)
@@ -116,7 +116,7 @@ func TestPathTildes(t *testing.T) {
},
},
}
fs := cfg.Filesystem(nil)
fs := cfg.Filesystem()
v := newSimple(cfg)
const testPath = "test"
lib/versioner/staggered.go
+1 -1
View File
@@ -44,7 +44,7 @@ func newStaggered(cfg config.FolderConfiguration) Versioner {
versionsFs := versionerFsFromFolderCfg(cfg)
s := &staggered{
folderFs: cfg.Filesystem(nil),
folderFs: cfg.Filesystem(),
versionsFs: versionsFs,
interval: [4]interval{
{30, 60 * 60}, // first hour -> 30 sec between versions
lib/versioner/trashcan.go
+1 -1
View File
@@ -33,7 +33,7 @@ func newTrashcan(cfg config.FolderConfiguration) Versioner {
// On error we default to 0, "do not clean out the trash can"
s := &trashcan{
folderFs: cfg.Filesystem(nil),
folderFs: cfg.Filesystem(),
versionsFs: versionerFsFromFolderCfg(cfg),
cleanoutDays: cleanoutDays,
copyRangeMethod: cfg.CopyRangeMethod.ToFS(),
lib/versioner/trashcan_test.go
+3 -3
View File
@@ -34,7 +34,7 @@ func TestTrashcanArchiveRestoreSwitcharoo(t *testing.T) {
FSPath: tmpDir2,
},
}
folderFs := cfg.Filesystem(nil)
folderFs := cfg.Filesystem()
versionsFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir2)
@@ -113,7 +113,7 @@ func TestTrashcanRestoreDeletedFile(t *testing.T) {
},
}
folderFs := cfg.Filesystem(nil)
folderFs := cfg.Filesystem()
versionsFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir2)
@@ -209,7 +209,7 @@ func TestTrashcanCleanOut(t *testing.T) {
},
}
fs := cfg.Filesystem(nil)
fs := cfg.Filesystem()
v := newTrashcan(cfg)
lib/versioner/util.go
+1 -1
View File
@@ -259,7 +259,7 @@ func restoreFile(method fs.CopyRangeMethod, src, dst fs.Filesystem, filePath str
}
func versionerFsFromFolderCfg(cfg config.FolderConfiguration) (versionsFs fs.Filesystem) {
folderFs := cfg.Filesystem(nil)
folderFs := cfg.Filesystem()
if cfg.Versioning.FSPath == "" {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), DefaultPath))
} else if cfg.Versioning.FSType == config.FilesystemTypeBasic {