felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

all: Add copy-on-write filesystem support (fixes #4271) (#6746)

Browse Source
This commit is contained in:
Audrius Butkevicius
2020-06-18 08:15:47 +02:00
committed by GitHub
parent 273cc9cef8
commit 4812fd3ec1
26 changed files with 858 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/versioner/external.go
+4 -3
View File
@@ -14,6 +14,7 @@ import (
"strings"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
"github.com/kballard/go-shellquote"
@@ -29,8 +30,8 @@ type external struct {
filesystem fs.Filesystem
}
func newExternal(filesystem fs.Filesystem, params map[string]string) Versioner {
command := params["command"]
func newExternal(cfg config.FolderConfiguration) Versioner {
command := cfg.Versioning.Params["command"]
if runtime.GOOS == "windows" {
command = strings.Replace(command, `\`, `\\`, -1)
@@ -38,7 +39,7 @@ func newExternal(filesystem fs.Filesystem, params map[string]string) Versioner {
s := external{
command: command,
filesystem: filesystem,
filesystem: cfg.Filesystem(),
}
l.Debugf("instantiated %#v", s)
lib/versioner/simple.go
+13 -10
View File
@@ -10,6 +10,7 @@ import (
"strconv"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
)
@@ -19,21 +20,23 @@ func init() {
}
type simple struct {
keep int
folderFs fs.Filesystem
versionsFs fs.Filesystem
keep int
folderFs fs.Filesystem
versionsFs fs.Filesystem
copyRangeMethod fs.CopyRangeMethod
}
func newSimple(folderFs fs.Filesystem, params map[string]string) Versioner {
keep, err := strconv.Atoi(params["keep"])
func newSimple(cfg config.FolderConfiguration) Versioner {
var keep, err = strconv.Atoi(cfg.Versioning.Params["keep"])
if err != nil {
keep = 5 // A reasonable default
}
s := simple{
keep: keep,
folderFs: folderFs,
versionsFs: fsFromParams(folderFs, params),
keep: keep,
folderFs: cfg.Filesystem(),
versionsFs: versionerFsFromFolderCfg(cfg),
copyRangeMethod: cfg.CopyRangeMethod,
}
l.Debugf("instantiated %#v", s)
@@ -43,7 +46,7 @@ func newSimple(folderFs fs.Filesystem, params map[string]string) Versioner {
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v simple) Archive(filePath string) error {
err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename)
err := archiveFile(v.copyRangeMethod, v.folderFs, v.versionsFs, filePath, TagFilename)
if err != nil {
return err
}
@@ -68,5 +71,5 @@ func (v simple) GetVersions() (map[string][]FileVersion, error) {
}
func (v simple) Restore(filepath string, versionTime time.Time) error {
return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
return restoreFile(v.copyRangeMethod, v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
}
lib/versioner/simple_test.go
+12 -2
View File
@@ -7,6 +7,7 @@
package versioner
import (
"github.com/syncthing/syncthing/lib/config"
"io/ioutil"
"math"
"path/filepath"
@@ -59,9 +60,18 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
t.Error(err)
}
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
cfg := config.FolderConfiguration{
FilesystemType: fs.FilesystemTypeBasic,
Path: dir,
Versioning: config.VersioningConfiguration{
Params: map[string]string{
"keep": "2",
},
},
}
fs := cfg.Filesystem()
v := newSimple(fs, map[string]string{"keep": "2"})
v := newSimple(cfg)
path := "test"
lib/versioner/staggered.go
+15 -11
View File
@@ -15,6 +15,7 @@ import (
"github.com/thejerf/suture"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
@@ -32,16 +33,18 @@ type interval struct {
type staggered struct {
suture.Service
cleanInterval int64
folderFs fs.Filesystem
versionsFs fs.Filesystem
interval [4]interval
mutex sync.Mutex
cleanInterval int64
folderFs fs.Filesystem
versionsFs fs.Filesystem
interval [4]interval
copyRangeMethod fs.CopyRangeMethod
mutex sync.Mutex
testCleanDone chan struct{}
}
func newStaggered(folderFs fs.Filesystem, params map[string]string) Versioner {
func newStaggered(cfg config.FolderConfiguration) Versioner {
params := cfg.Versioning.Params
maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
if err != nil {
maxAge = 31536000 // Default: ~1 year
@@ -53,11 +56,11 @@ func newStaggered(folderFs fs.Filesystem, params map[string]string) Versioner {
// Backwards compatibility
params["fsPath"] = params["versionsPath"]
versionsFs := fsFromParams(folderFs, params)
versionsFs := versionerFsFromFolderCfg(cfg)
s := &staggered{
cleanInterval: cleanInterval,
folderFs: folderFs,
folderFs: cfg.Filesystem(),
versionsFs: versionsFs,
interval: [4]interval{
{30, 60 * 60}, // first hour -> 30 sec between versions
@@ -65,7 +68,8 @@ func newStaggered(folderFs fs.Filesystem, params map[string]string) Versioner {
{24 * 60 * 60, 30 * 24 * 60 * 60}, // next 30 days -> 1 day between versions
{7 * 24 * 60 * 60, maxAge}, // next year -> 1 week between versions
},
mutex: sync.NewMutex(),
copyRangeMethod: cfg.CopyRangeMethod,
mutex: sync.NewMutex(),
}
s.Service = util.AsService(s.serve, s.String())
@@ -216,7 +220,7 @@ func (v *staggered) Archive(filePath string) error {
v.mutex.Lock()
defer v.mutex.Unlock()
if err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename); err != nil {
if err := archiveFile(v.copyRangeMethod, v.folderFs, v.versionsFs, filePath, TagFilename); err != nil {
return err
}
@@ -230,7 +234,7 @@ func (v *staggered) GetVersions() (map[string][]FileVersion, error) {
}
func (v *staggered) Restore(filepath string, versionTime time.Time) error {
return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
return restoreFile(v.copyRangeMethod, v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
}
func (v *staggered) String() string {
lib/versioner/staggered_test.go
+12 -3
View File
@@ -7,6 +7,7 @@
package versioner
import (
"github.com/syncthing/syncthing/lib/config"
"sort"
"strconv"
"testing"
@@ -96,9 +97,17 @@ func TestStaggeredVersioningVersionCount(t *testing.T) {
}
sort.Strings(delete)
v := newStaggered(fs.NewFilesystem(fs.FilesystemTypeFake, "testdata"), map[string]string{
"maxAge": strconv.Itoa(365 * 86400),
}).(*staggered)
cfg := config.FolderConfiguration{
FilesystemType: fs.FilesystemTypeBasic,
Path: "testdata",
Versioning: config.VersioningConfiguration{
Params: map[string]string{
"maxAge": strconv.Itoa(365 * 86400),
},
},
}
v := newStaggered(cfg).(*staggered)
rem := v.toRemove(versionsWithMtime, now)
sort.Strings(rem)
lib/versioner/trashcan.go
+13 -10
View File
@@ -14,6 +14,7 @@ import (
"github.com/thejerf/suture"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/util"
)
@@ -25,19 +26,21 @@ func init() {
type trashcan struct {
suture.Service
folderFs fs.Filesystem
versionsFs fs.Filesystem
cleanoutDays int
folderFs fs.Filesystem
versionsFs fs.Filesystem
cleanoutDays int
copyRangeMethod fs.CopyRangeMethod
}
func newTrashcan(folderFs fs.Filesystem, params map[string]string) Versioner {
cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
func newTrashcan(cfg config.FolderConfiguration) Versioner {
cleanoutDays, _ := strconv.Atoi(cfg.Versioning.Params["cleanoutDays"])
// On error we default to 0, "do not clean out the trash can"
s := &trashcan{
folderFs: folderFs,
versionsFs: fsFromParams(folderFs, params),
cleanoutDays: cleanoutDays,
folderFs: cfg.Filesystem(),
versionsFs: versionerFsFromFolderCfg(cfg),
cleanoutDays: cleanoutDays,
copyRangeMethod: cfg.CopyRangeMethod,
}
s.Service = util.AsService(s.serve, s.String())
@@ -48,7 +51,7 @@ func newTrashcan(folderFs fs.Filesystem, params map[string]string) Versioner {
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (t *trashcan) Archive(filePath string) error {
return archiveFile(t.folderFs, t.versionsFs, filePath, func(name, tag string) string {
return archiveFile(t.copyRangeMethod, t.folderFs, t.versionsFs, filePath, func(name, tag string) string {
return name
})
}
@@ -144,7 +147,7 @@ func (t *trashcan) Restore(filepath string, versionTime time.Time) error {
return name
}
err := restoreFile(t.versionsFs, t.folderFs, filepath, versionTime, tagger)
err := restoreFile(t.copyRangeMethod, t.versionsFs, t.folderFs, filepath, versionTime, tagger)
if taggedName == "" {
return err
}
lib/versioner/trashcan_test.go
+25 -6
View File
@@ -7,6 +7,7 @@
package versioner
import (
"github.com/syncthing/syncthing/lib/config"
"io/ioutil"
"os"
"path/filepath"
@@ -53,7 +54,17 @@ func TestTrashcanCleanout(t *testing.T) {
}
}
versioner := newTrashcan(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"cleanoutDays": "7"}).(*trashcan)
cfg := config.FolderConfiguration{
FilesystemType: fs.FilesystemTypeBasic,
Path: "testdata",
Versioning: config.VersioningConfiguration{
Params: map[string]string{
"cleanoutDays": "7",
},
},
}
versioner := newTrashcan(cfg).(*trashcan)
if err := versioner.cleanoutArchive(); err != nil {
t.Fatal(err)
}
@@ -90,15 +101,23 @@ func TestTrashcanArchiveRestoreSwitcharoo(t *testing.T) {
t.Fatal(err)
}
folderFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir1)
cfg := config.FolderConfiguration{
FilesystemType: fs.FilesystemTypeBasic,
Path: tmpDir1,
Versioning: config.VersioningConfiguration{
Params: map[string]string{
"fsType": "basic",
"fsPath": tmpDir2,
},
},
}
folderFs := cfg.Filesystem()
versionsFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir2)
writeFile(t, folderFs, "file", "A")
versioner := newTrashcan(folderFs, map[string]string{
"fsType": "basic",
"fsPath": tmpDir2,
})
versioner := newTrashcan(cfg)
if err := versioner.Archive("file"); err != nil {
t.Fatal(err)
lib/versioner/util.go
+10 -6
View File
@@ -14,6 +14,8 @@ import (
"time"
"github.com/pkg/errors"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/util"
@@ -129,7 +131,7 @@ func retrieveVersions(fileSystem fs.Filesystem) (map[string][]FileVersion, error
type fileTagger func(string, string) string
func archiveFile(srcFs, dstFs fs.Filesystem, filePath string, tagger fileTagger) error {
func archiveFile(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, filePath string, tagger fileTagger) error {
filePath = osutil.NativeFilename(filePath)
info, err := srcFs.Lstat(filePath)
if fs.IsNotExist(err) {
@@ -170,7 +172,7 @@ func archiveFile(srcFs, dstFs fs.Filesystem, filePath string, tagger fileTagger)
ver := tagger(file, now.Format(TimeFormat))
dst := filepath.Join(inFolderPath, ver)
l.Debugln("archiving", filePath, "moving to", dst)
err = osutil.RenameOrCopy(srcFs, dstFs, filePath, dst)
err = osutil.RenameOrCopy(method, srcFs, dstFs, filePath, dst)
mtime := info.ModTime()
// If it's a trashcan versioner type thing, then it does not have version time in the name
@@ -184,7 +186,7 @@ func archiveFile(srcFs, dstFs fs.Filesystem, filePath string, tagger fileTagger)
return err
}
func restoreFile(src, dst fs.Filesystem, filePath string, versionTime time.Time, tagger fileTagger) error {
func restoreFile(method fs.CopyRangeMethod, src, dst fs.Filesystem, filePath string, versionTime time.Time, tagger fileTagger) error {
tag := versionTime.In(time.Local).Truncate(time.Second).Format(TimeFormat)
taggedFilePath := tagger(filePath, tag)
@@ -200,7 +202,7 @@ func restoreFile(src, dst fs.Filesystem, filePath string, versionTime time.Time,
return errors.Wrap(err, "removing existing symlink")
}
case info.IsRegular():
if err := archiveFile(dst, src, filePath, tagger); err != nil {
if err := archiveFile(method, dst, src, filePath, tagger); err != nil {
return errors.Wrap(err, "archiving existing file")
}
default:
@@ -247,12 +249,14 @@ func restoreFile(src, dst fs.Filesystem, filePath string, versionTime time.Time,
}
_ = dst.MkdirAll(filepath.Dir(filePath), 0755)
err := osutil.RenameOrCopy(src, dst, sourceFile, filePath)
err := osutil.RenameOrCopy(method, src, dst, sourceFile, filePath)
_ = dst.Chtimes(filePath, sourceMtime, sourceMtime)
return err
}
func fsFromParams(folderFs fs.Filesystem, params map[string]string) (versionsFs fs.Filesystem) {
func versionerFsFromFolderCfg(cfg config.FolderConfiguration) (versionsFs fs.Filesystem) {
params := cfg.Versioning.Params
folderFs := cfg.Filesystem()
if params["fsType"] == "" && params["fsPath"] == "" {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
lib/versioner/versioner.go
+4 -5
View File
@@ -14,7 +14,6 @@ import (
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
)
type Versioner interface {
@@ -29,7 +28,7 @@ type FileVersion struct {
Size int64 `json:"size"`
}
type factory func(filesystem fs.Filesystem, params map[string]string) Versioner
type factory func(cfg config.FolderConfiguration) Versioner
var factories = make(map[string]factory)
@@ -40,11 +39,11 @@ const (
timeGlob = "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]" // glob pattern matching TimeFormat
)
func New(fs fs.Filesystem, cfg config.VersioningConfiguration) (Versioner, error) {
fac, ok := factories[cfg.Type]
func New(cfg config.FolderConfiguration) (Versioner, error) {
fac, ok := factories[cfg.Versioning.Type]
if !ok {
return nil, fmt.Errorf("requested versioning type %q does not exist", cfg.Type)
}
return fac(fs, cfg.Params), nil
return fac(cfg), nil
}