felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

all: Refactor preparing configuration (#7127)

Browse Source
This commit is contained in:
Simon Frei
2020-11-20 14:21:54 +01:00
committed by GitHub
parent 641b7aee38
commit 24af89c8e2
19 changed files with 207 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/config/commit_test.go
+1 -1
View File
@@ -44,7 +44,7 @@ func (validationError) String() string {
func TestReplaceCommit(t *testing.T) {
t.Skip("broken, fails randomly, #3834")
w := wrap("/dev/null", Configuration{Version: 0})
w := wrap("/dev/null", Configuration{Version: 0}, device1)
if w.RawCopy().Version != 0 {
t.Fatal("Config incorrect")
}
lib/config/config.go
+87 -111
View File
@@ -25,7 +25,6 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/util"
)
@@ -105,8 +104,6 @@ func New(myID protocol.DeviceID) Configuration {
cfg.Options.UnackedNotificationIDs = []string{"authenticationUserAndPassword"}
util.SetDefaults(&cfg)
util.SetDefaults(&cfg.Options)
util.SetDefaults(&cfg.GUI)
// Can't happen.
if err := cfg.prepare(myID); err != nil {
@@ -152,8 +149,6 @@ func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, int, error) {
var cfg xmlConfiguration
util.SetDefaults(&cfg)
util.SetDefaults(&cfg.Options)
util.SetDefaults(&cfg.GUI)
if err := xml.NewDecoder(r).Decode(&cfg); err != nil {
return Configuration{}, 0, err
@@ -171,8 +166,6 @@ func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
var cfg Configuration
util.SetDefaults(&cfg)
util.SetDefaults(&cfg.Options)
util.SetDefaults(&cfg.GUI)
bs, err := ioutil.ReadAll(r)
if err != nil {
@@ -230,38 +223,61 @@ func (cfg *Configuration) WriteXML(w io.Writer) error {
}
func (cfg *Configuration) prepare(myID protocol.DeviceID) error {
var myName string
cfg.ensureMyDevice(myID)
// Ensure this device is present in the config
for _, device := range cfg.Devices {
if device.DeviceID == myID {
goto found
}
}
myName, _ = os.Hostname()
cfg.Devices = append(cfg.Devices, DeviceConfiguration{
DeviceID: myID,
Name: myName,
})
found:
if err := cfg.clean(); err != nil {
existingDevices, err := cfg.prepareFoldersAndDevices(myID)
if err != nil {
return err
}
// Ensure that we are part of the devices
for i := range cfg.Folders {
cfg.Folders[i].Devices = ensureDevicePresent(cfg.Folders[i].Devices, myID)
}
cfg.GUI.prepare()
guiPWIsSet := cfg.GUI.User != "" && cfg.GUI.Password != ""
cfg.Options.prepare(guiPWIsSet)
ignoredDevices := cfg.prepareIgnoredDevices(existingDevices)
cfg.preparePendingDevices(existingDevices, ignoredDevices)
cfg.removeDeprecatedProtocols()
util.FillNilExceptDeprecated(cfg)
// TestIssue1750 relies on migrations happening after preparing options.
cfg.applyMigrations()
return nil
}
func (cfg *Configuration) clean() error {
util.FillNilSlices(&cfg.Options)
func (cfg *Configuration) ensureMyDevice(myID protocol.DeviceID) {
// Ensure this device is present in the config
for _, device := range cfg.Devices {
if device.DeviceID == myID {
return
}
}
myName, _ := os.Hostname()
cfg.Devices = append(cfg.Devices, DeviceConfiguration{
DeviceID: myID,
Name: myName,
})
}
func (cfg *Configuration) prepareFoldersAndDevices(myID protocol.DeviceID) (map[protocol.DeviceID]bool, error) {
existingDevices := cfg.prepareDeviceList()
sharedFolders, err := cfg.prepareFolders(myID, existingDevices)
if err != nil {
return nil, err
}
cfg.prepareDevices(sharedFolders)
return existingDevices, nil
}
func (cfg *Configuration) prepareDeviceList() map[protocol.DeviceID]bool {
// Ensure that the device list is
// - free from duplicates
// - no devices with empty ID
@@ -272,89 +288,62 @@ func (cfg *Configuration) clean() error {
return cfg.Devices[a].DeviceID.Compare(cfg.Devices[b].DeviceID) == -1
})
// Build a list of available devices
existingDevices := make(map[protocol.DeviceID]bool, len(cfg.Devices))
for _, device := range cfg.Devices {
existingDevices[device.DeviceID] = true
}
return existingDevices
}
func (cfg *Configuration) prepareFolders(myID protocol.DeviceID, existingDevices map[protocol.DeviceID]bool) (map[protocol.DeviceID][]string, error) {
// Prepare folders and check for duplicates. Duplicates are bad and
// dangerous, can't currently be resolved in the GUI, and shouldn't
// happen when configured by the GUI. We return with an error in that
// situation.
existingFolders := make(map[string]*FolderConfiguration)
sharedFolders := make(map[protocol.DeviceID][]string, len(cfg.Devices))
existingFolders := make(map[string]*FolderConfiguration, len(cfg.Folders))
for i := range cfg.Folders {
folder := &cfg.Folders[i]
folder.prepare()
if folder.ID == "" {
return errFolderIDEmpty
return nil, errFolderIDEmpty
}
if folder.Path == "" {
return fmt.Errorf("folder %q: %w", folder.ID, errFolderPathEmpty)
return nil, fmt.Errorf("folder %q: %w", folder.ID, errFolderPathEmpty)
}
if _, ok := existingFolders[folder.ID]; ok {
return fmt.Errorf("folder %q: %w", folder.ID, errFolderIDDuplicate)
return nil, fmt.Errorf("folder %q: %w", folder.ID, errFolderIDDuplicate)
}
folder.prepare(myID, existingDevices)
existingFolders[folder.ID] = folder
for _, dev := range folder.Devices {
sharedFolders[dev.DeviceID] = append(sharedFolders[dev.DeviceID], folder.ID)
}
}
cfg.Options.RawListenAddresses = util.UniqueTrimmedStrings(cfg.Options.RawListenAddresses)
cfg.Options.RawGlobalAnnServers = util.UniqueTrimmedStrings(cfg.Options.RawGlobalAnnServers)
if cfg.Version > 0 && cfg.Version < OldestHandledVersion {
l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version)
}
// Upgrade configuration versions as appropriate
migrationsMut.Lock()
migrations.apply(cfg)
migrationsMut.Unlock()
// Build a list of available devices
existingDevices := make(map[protocol.DeviceID]bool)
for _, device := range cfg.Devices {
existingDevices[device.DeviceID] = true
}
// Ensure that the folder list is sorted by ID
sort.Slice(cfg.Folders, func(a, b int) bool {
return cfg.Folders[a].ID < cfg.Folders[b].ID
})
return sharedFolders, nil
}
// Ensure that in all folder configs
// - any loose devices are not present in the wrong places
// - there are no duplicate devices
// - the versioning configuration parameter map is not nil
sharedFolders := make(map[protocol.DeviceID][]string, len(cfg.Devices))
for i := range cfg.Folders {
cfg.Folders[i].Devices = ensureExistingDevices(cfg.Folders[i].Devices, existingDevices)
cfg.Folders[i].Devices = ensureNoDuplicateFolderDevices(cfg.Folders[i].Devices)
if cfg.Folders[i].Versioning.Params == nil {
cfg.Folders[i].Versioning.Params = map[string]string{}
}
sort.Slice(cfg.Folders[i].Devices, func(a, b int) bool {
return cfg.Folders[i].Devices[a].DeviceID.Compare(cfg.Folders[i].Devices[b].DeviceID) == -1
})
for _, dev := range cfg.Folders[i].Devices {
sharedFolders[dev.DeviceID] = append(sharedFolders[dev.DeviceID], cfg.Folders[i].ID)
}
}
func (cfg *Configuration) prepareDevices(sharedFolders map[protocol.DeviceID][]string) {
for i := range cfg.Devices {
cfg.Devices[i].prepare(sharedFolders[cfg.Devices[i].DeviceID])
}
}
// Very short reconnection intervals are annoying
if cfg.Options.ReconnectIntervalS < 5 {
cfg.Options.ReconnectIntervalS = 5
}
if cfg.GUI.APIKey == "" {
cfg.GUI.APIKey = rand.String(32)
}
func (cfg *Configuration) prepareIgnoredDevices(existingDevices map[protocol.DeviceID]bool) map[protocol.DeviceID]bool {
// The list of ignored devices should not contain any devices that have
// been manually added to the config.
var newIgnoredDevices []ObservedDevice
ignoredDevices := make(map[protocol.DeviceID]bool)
newIgnoredDevices := cfg.IgnoredDevices[:0]
ignoredDevices := make(map[protocol.DeviceID]bool, len(cfg.IgnoredDevices))
for _, dev := range cfg.IgnoredDevices {
if !existingDevices[dev.ID] {
ignoredDevices[dev.ID] = true
@@ -362,7 +351,10 @@ func (cfg *Configuration) clean() error {
}
}
cfg.IgnoredDevices = newIgnoredDevices
return ignoredDevices
}
func (cfg *Configuration) preparePendingDevices(existingDevices, ignoredDevices map[protocol.DeviceID]bool) {
// The list of pending devices should not contain devices that were added manually, nor should it contain
// ignored devices.
@@ -371,7 +363,7 @@ func (cfg *Configuration) clean() error {
return cfg.PendingDevices[i].Time.Before(cfg.PendingDevices[j].Time)
})
var newPendingDevices []ObservedDevice
newPendingDevices := cfg.PendingDevices[:0]
nextPendingDevice:
for _, pendingDevice := range cfg.PendingDevices {
if !existingDevices[pendingDevice.ID] && !ignoredDevices[pendingDevice.ID] {
@@ -385,7 +377,9 @@ nextPendingDevice:
}
}
cfg.PendingDevices = newPendingDevices
}
func (cfg *Configuration) removeDeprecatedProtocols() {
// Deprecated protocols are removed from the list of listeners and
// device addresses. So far just kcp*.
for _, prefix := range []string{"kcp"} {
@@ -395,35 +389,17 @@ nextPendingDevice:
dev.Addresses = filterURLSchemePrefix(dev.Addresses, prefix)
}
}
}
// Initialize any empty slices
if cfg.Folders == nil {
cfg.Folders = []FolderConfiguration{}
}
if cfg.IgnoredDevices == nil {
cfg.IgnoredDevices = []ObservedDevice{}
}
if cfg.PendingDevices == nil {
cfg.PendingDevices = []ObservedDevice{}
}
if cfg.Options.AlwaysLocalNets == nil {
cfg.Options.AlwaysLocalNets = []string{}
}
if cfg.Options.UnackedNotificationIDs == nil {
cfg.Options.UnackedNotificationIDs = []string{}
} else if cfg.GUI.User != "" && cfg.GUI.Password != "" {
for i, key := range cfg.Options.UnackedNotificationIDs {
if key == "authenticationUserAndPassword" {
cfg.Options.UnackedNotificationIDs = append(cfg.Options.UnackedNotificationIDs[:i], cfg.Options.UnackedNotificationIDs[i+1:]...)
break
}
}
}
if cfg.Options.FeatureFlags == nil {
cfg.Options.FeatureFlags = []string{}
func (cfg *Configuration) applyMigrations() {
if cfg.Version > 0 && cfg.Version < OldestHandledVersion {
l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version)
}
return nil
// Upgrade configuration versions as appropriate
migrationsMut.Lock()
migrations.apply(cfg)
migrationsMut.Unlock()
}
// DeviceMap returns a map of device ID to device configuration for the given configuration.
lib/config/config_test.go
+14 -9
View File
@@ -526,7 +526,7 @@ func TestNewSaveLoad(t *testing.T) {
}
intCfg := New(device1)
cfg := wrap(path, intCfg)
cfg := wrap(path, intCfg, device1)
if exists(path) {
t.Error(path, "exists")
@@ -646,7 +646,7 @@ func TestPullOrder(t *testing.T) {
if err != nil {
t.Fatal(err)
}
wrapper = wrap("testdata/pullorder.xml", cfg)
wrapper = wrap("testdata/pullorder.xml", cfg, device1)
folders = wrapper.Folders()
for _, tc := range expected {
@@ -941,7 +941,8 @@ func TestIssue4219(t *testing.T) {
]
}`))
cfg, err := ReadJSON(r, protocol.LocalDeviceID)
myID := protocol.LocalDeviceID
cfg, err := ReadJSON(r, myID)
if err != nil {
t.Fatal(err)
}
@@ -959,7 +960,7 @@ func TestIssue4219(t *testing.T) {
t.Errorf("There should be three ignored folders, not %d", ignoredFolders)
}
w := wrap("/tmp/cfg", cfg)
w := wrap("/tmp/cfg", cfg, myID)
if !w.IgnoredFolder(device2, "t1") {
t.Error("Folder device2 t1 should be ignored")
}
@@ -1119,12 +1120,16 @@ func TestRemoveDeviceWithEmptyID(t *testing.T) {
},
}
cfg.clean()
cfg.prepare(device1)
if len(cfg.Devices) != 0 {
if len(cfg.Devices) != 1 {
t.Error("Expected one device")
} else if cfg.Devices[0].DeviceID != device1 {
t.Error("Expected device with empty ID to be removed from config:", cfg.Devices)
}
if len(cfg.Folders[0].Devices) != 0 {
if len(cfg.Folders[0].Devices) != 1 {
t.Error("Expected one device in folder")
} else if cfg.Folders[0].Devices[0].DeviceID != device1 {
t.Error("Expected device with empty ID to be removed from folder")
}
}
@@ -1175,8 +1180,8 @@ func load(path string, myID protocol.DeviceID) (Wrapper, error) {
return cfg, err
}
func wrap(path string, cfg Configuration) Wrapper {
return Wrap(path, cfg, events.NoopLogger)
func wrap(path string, cfg Configuration, myID protocol.DeviceID) Wrapper {
return Wrap(path, cfg, myID, events.NoopLogger)
}
func TestInternalVersioningConfiguration(t *testing.T) {
lib/config/folderconfiguration.go
+15 -5
View File
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"runtime"
"sort"
"strings"
"time"
@@ -43,7 +44,7 @@ func NewFolderConfiguration(myID protocol.DeviceID, id, label string, fsType fs.
util.SetDefaults(&f)
f.prepare()
f.prepare(myID, nil)
return f
}
@@ -182,7 +183,19 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
return deviceIDs
}
func (f *FolderConfiguration) prepare() {
func (f *FolderConfiguration) prepare(myID protocol.DeviceID, existingDevices map[protocol.DeviceID]bool) {
// Ensure that
// - any loose devices are not present in the wrong places
// - there are no duplicate devices
// - we are part of the devices
f.Devices = ensureExistingDevices(f.Devices, existingDevices)
f.Devices = ensureNoDuplicateFolderDevices(f.Devices)
f.Devices = ensureDevicePresent(f.Devices, myID)
sort.Slice(f.Devices, func(a, b int) bool {
return f.Devices[a].DeviceID.Compare(f.Devices[b].DeviceID) == -1
})
if f.RescanIntervalS > MaxRescanIntervalS {
f.RescanIntervalS = MaxRescanIntervalS
} else if f.RescanIntervalS < 0 {
@@ -194,9 +207,6 @@ func (f *FolderConfiguration) prepare() {
f.FSWatcherDelayS = 10
}
if f.Versioning.Params == nil {
f.Versioning.Params = make(map[string]string)
}
if f.Versioning.CleanupIntervalS > MaxRescanIntervalS {
f.Versioning.CleanupIntervalS = MaxRescanIntervalS
} else if f.Versioning.CleanupIntervalS < 0 {
lib/config/guiconfiguration.go
+8
View File
@@ -11,6 +11,8 @@ import (
"os"
"strconv"
"strings"
"github.com/syncthing/syncthing/lib/rand"
)
func (c GUIConfiguration) IsAuthEnabled() bool {
@@ -126,6 +128,12 @@ func (c GUIConfiguration) IsValidAPIKey(apiKey string) bool {
}
}
func (c *GUIConfiguration) prepare() {
if c.APIKey == "" {
c.APIKey = rand.String(32)
}
}
func (c GUIConfiguration) Copy() GUIConfiguration {
return c
}
lib/config/optionsconfiguration.go
+21
View File
@@ -28,6 +28,27 @@ func (opts OptionsConfiguration) Copy() OptionsConfiguration {
return optsCopy
}
func (opts *OptionsConfiguration) prepare(guiPWIsSet bool) {
util.FillNilSlices(opts)
opts.RawListenAddresses = util.UniqueTrimmedStrings(opts.RawListenAddresses)
opts.RawGlobalAnnServers = util.UniqueTrimmedStrings(opts.RawGlobalAnnServers)
// Very short reconnection intervals are annoying
if opts.ReconnectIntervalS < 5 {
opts.ReconnectIntervalS = 5
}
if guiPWIsSet && len(opts.UnackedNotificationIDs) > 0 {
for i, key := range opts.UnackedNotificationIDs {
if key == "authenticationUserAndPassword" {
opts.UnackedNotificationIDs = append(opts.UnackedNotificationIDs[:i], opts.UnackedNotificationIDs[i+1:]...)
break
}
}
}
}
// RequiresRestartOnly returns a copy with only the attributes that require
// restart on change.
func (opts OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration {
lib/config/wrapper.go
+10 -3
View File
@@ -55,6 +55,7 @@ func (noopWaiter) Wait() {}
// notifications of changes to registered Handlers
type Wrapper interface {
ConfigPath() string
MyID() protocol.DeviceID
RawCopy() Configuration
Replace(cfg Configuration) (Waiter, error)
@@ -97,6 +98,7 @@ type wrapper struct {
cfg Configuration
path string
evLogger events.Logger
myID protocol.DeviceID
waiter Waiter // Latest ongoing config change
subs []Committer
@@ -107,11 +109,12 @@ type wrapper struct {
// Wrap wraps an existing Configuration structure and ties it to a file on
// disk.
func Wrap(path string, cfg Configuration, evLogger events.Logger) Wrapper {
func Wrap(path string, cfg Configuration, myID protocol.DeviceID, evLogger events.Logger) Wrapper {
w := &wrapper{
cfg: cfg,
path: path,
evLogger: evLogger,
myID: myID,
waiter: noopWaiter{}, // Noop until first config change
mut: sync.NewMutex(),
}
@@ -132,13 +135,17 @@ func Load(path string, myID protocol.DeviceID, evLogger events.Logger) (Wrapper,
return nil, 0, err
}
return Wrap(path, cfg, evLogger), originalVersion, nil
return Wrap(path, cfg, myID, evLogger), originalVersion, nil
}
func (w *wrapper) ConfigPath() string {
return w.path
}
func (w *wrapper) MyID() protocol.DeviceID {
return w.myID
}
// Subscribe registers the given handler to be called on any future
// configuration changes.
func (w *wrapper) Subscribe(c Committer) {
@@ -184,7 +191,7 @@ func (w *wrapper) Replace(cfg Configuration) (Waiter, error) {
func (w *wrapper) replaceLocked(to Configuration) (Waiter, error) {
from := w.cfg
if err := to.clean(); err != nil {
if err := to.prepare(w.myID); err != nil {
return noopWaiter{}, err
}