felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib: Get rid of buggy filesystem wrapping (#8257)

Browse Source
This commit is contained in:
Simon Frei
2022-04-10 20:55:05 +02:00
committed by GitHub
parent 9b09bcc5f1
commit db72579f0e
26 changed files with 226 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/fs/basicfs.go
+2 -1
View File
@@ -27,12 +27,13 @@ var (
type OptionJunctionsAsDirs struct{}
func (o *OptionJunctionsAsDirs) apply(fs Filesystem) {
func (o *OptionJunctionsAsDirs) apply(fs Filesystem) Filesystem {
if basic, ok := fs.(*BasicFilesystem); !ok {
l.Warnln("WithJunctionsAsDirs must only be used with FilesystemTypeBasic")
} else {
basic.junctionsAsDirs = true
}
return fs
}
func (o *OptionJunctionsAsDirs) String() string {
lib/fs/casefs.go
+16 -10
View File
@@ -123,21 +123,27 @@ func (r *caseFilesystemRegistry) cleaner() {
var globalCaseFilesystemRegistry = caseFilesystemRegistry{fss: make(map[fskey]*caseFilesystem)}
// caseFilesystem is a BasicFilesystem with additional checks to make a
// potentially case insensitive underlying FS behave like it's case-sensitive.
type caseFilesystem struct {
Filesystem
realCaser
}
// NewCaseFilesystem ensures that the given, potentially case-insensitive filesystem
// OptionDetectCaseConflicts ensures that the potentially case-insensitive filesystem
// behaves like a case-sensitive filesystem. Meaning that it takes into account
// the real casing of a path and returns ErrCaseConflict if the given path differs
// from the real path. It is safe to use with any filesystem, i.e. also a
// case-sensitive one. However it will add some overhead and thus shouldn't be
// used if the filesystem is known to already behave case-sensitively.
func NewCaseFilesystem(fs Filesystem) Filesystem {
return wrapFilesystem(fs, globalCaseFilesystemRegistry.get)
type OptionDetectCaseConflicts struct{}
func (o *OptionDetectCaseConflicts) apply(fs Filesystem) Filesystem {
return globalCaseFilesystemRegistry.get(fs)
}
func (o *OptionDetectCaseConflicts) String() string {
return "detectCaseConflicts"
}
// caseFilesystem is a BasicFilesystem with additional checks to make a
// potentially case insensitive underlying FS behave like it's case-sensitive.
type caseFilesystem struct {
Filesystem
realCaser
}
func (f *caseFilesystem) Chmod(name string, mode FileMode) error {
lib/fs/casefs_test.go
+7 -3
View File
@@ -34,8 +34,12 @@ func TestRealCase(t *testing.T) {
})
}
func newCaseFilesystem(fsys Filesystem) *caseFilesystem {
return globalCaseFilesystemRegistry.get(fsys).(*caseFilesystem)
}
func testRealCase(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
testFs := newCaseFilesystem(fsys)
comps := []string{"Foo", "bar", "BAZ", "bAs"}
path := filepath.Join(comps...)
testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777)
@@ -86,7 +90,7 @@ func TestRealCaseSensitive(t *testing.T) {
}
func testRealCaseSensitive(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
testFs := newCaseFilesystem(fsys)
names := make([]string, 2)
names[0] = "foo"
@@ -139,7 +143,7 @@ func testCaseFSStat(t *testing.T, fsys Filesystem) {
sensitive = false
}
testFs := NewCaseFilesystem(fsys)
testFs := newCaseFilesystem(fsys)
_, err = testFs.Stat("FOO")
if sensitive {
if IsNotExist(err) {
lib/fs/filesystem.go
+31 -16
View File
@@ -202,10 +202,29 @@ var IsPathSeparator = os.IsPathSeparator
// representation of those must be part of the returned string.
type Option interface {
String() string
apply(Filesystem)
apply(Filesystem) Filesystem
}
func NewFilesystem(fsType FilesystemType, uri string, opts ...Option) Filesystem {
var caseOpt Option
var mtimeOpt Option
i := 0
for _, opt := range opts {
if caseOpt != nil && mtimeOpt != nil {
break
}
switch opt.(type) {
case *OptionDetectCaseConflicts:
caseOpt = opt
case *optionMtime:
mtimeOpt = opt
default:
opts[i] = opt
i++
}
}
opts = opts[:i]
var fs Filesystem
switch fsType {
case FilesystemTypeBasic:
@@ -221,6 +240,17 @@ func NewFilesystem(fsType FilesystemType, uri string, opts ...Option) Filesystem
}
}
// Case handling is the innermost, as any filesystem calls by wrappers should be case-resolved
if caseOpt != nil {
fs = caseOpt.apply(fs)
}
// mtime handling should happen inside walking, as filesystem calls while
// walking should be mtime-resolved too
if mtimeOpt != nil {
fs = mtimeOpt.apply(fs)
}
if l.ShouldDebug("walkfs") {
return NewWalkFilesystem(&logFilesystem{fs})
}
@@ -289,21 +319,6 @@ func Canonicalize(file string) (string, error) {
return file, nil
}
// wrapFilesystem should always be used when wrapping a Filesystem.
// It ensures proper wrapping order, which right now means:
// `logFilesystem` needs to be the outermost wrapper for caller lookup.
func wrapFilesystem(fs Filesystem, wrapFn func(Filesystem) Filesystem) Filesystem {
logFs, ok := fs.(*logFilesystem)
if ok {
fs = logFs.Filesystem
}
fs = wrapFn(fs)
if ok {
fs = &logFilesystem{fs}
}
return fs
}
// unwrapFilesystem removes "wrapping" filesystems to expose the filesystem of the requested wrapperType, if it exists.
func unwrapFilesystem(fs Filesystem, wrapperType filesystemWrapperType) (Filesystem, bool) {
var ok bool
lib/fs/mtimefs.go
+28 -33
View File
@@ -33,20 +33,34 @@ func WithCaseInsensitivity(v bool) MtimeFSOption {
}
}
// NewMtimeFS returns a filesystem with nanosecond mtime precision, regardless
// of what shenanigans the underlying filesystem gets up to.
func NewMtimeFS(fs Filesystem, db database, options ...MtimeFSOption) Filesystem {
return wrapFilesystem(fs, func(underlying Filesystem) Filesystem {
f := &mtimeFS{
Filesystem: underlying,
chtimes: underlying.Chtimes, // for mocking it out in the tests
db: db,
}
for _, opt := range options {
opt(f)
}
return f
})
type optionMtime struct {
db database
options []MtimeFSOption
}
// NewMtimeOption makes any filesystem provide nanosecond mtime precision,
// regardless of what shenanigans the underlying filesystem gets up to.
func NewMtimeOption(db database, options ...MtimeFSOption) Option {
return &optionMtime{
db: db,
options: options,
}
}
func (o *optionMtime) apply(fs Filesystem) Filesystem {
f := &mtimeFS{
Filesystem: fs,
chtimes: fs.Chtimes, // for mocking it out in the tests
db: o.db,
}
for _, opt := range o.options {
opt(f)
}
return f
}
func (_ *optionMtime) String() string {
return "mtime"
}
func (f *mtimeFS) Chtimes(name string, atime, mtime time.Time) error {
@@ -104,25 +118,6 @@ func (f *mtimeFS) Lstat(name string) (FileInfo, error) {
return info, nil
}
func (f *mtimeFS) Walk(root string, walkFn WalkFunc) error {
return f.Filesystem.Walk(root, func(path string, info FileInfo, err error) error {
if info != nil {
mtimeMapping, loadErr := f.load(path)
if loadErr != nil && err == nil {
// The iterator gets to deal with the error
err = loadErr
}
if mtimeMapping.Real == info.ModTime() {
info = mtimeFileInfo{
FileInfo: info,
mtime: mtimeMapping.Virtual,
}
}
}
return walkFn(path, info, err)
})
}
func (f *mtimeFS) Create(name string) (File, error) {
fd, err := f.Filesystem.Create(name)
if err != nil {
lib/fs/mtimefs_test.go
+16 -10
View File
@@ -26,7 +26,7 @@ func TestMtimeFS(t *testing.T) {
// a random time with nanosecond precision
testTime := time.Unix(1234567890, 123456789)
mtimefs := newMtimeFS(newBasicFilesystem("."), make(mapStore))
mtimefs := newMtimeFS(".", make(mapStore))
// Do one Chtimes call that will go through to the normal filesystem
mtimefs.chtimes = os.Chtimes
@@ -88,8 +88,8 @@ func TestMtimeFSWalk(t *testing.T) {
}
defer func() { _ = os.RemoveAll(dir) }()
underlying := NewFilesystem(FilesystemTypeBasic, dir)
mtimefs := newMtimeFS(underlying, make(mapStore))
mtimefs, walkFs := newMtimeFSWithWalk(dir, make(mapStore))
underlying := mtimefs.Filesystem
mtimefs.chtimes = failChtimes
if err := os.WriteFile(filepath.Join(dir, "file"), []byte("hello"), 0644); err != nil {
@@ -120,7 +120,7 @@ func TestMtimeFSWalk(t *testing.T) {
}
found := false
_ = mtimefs.Walk("", func(path string, info FileInfo, err error) error {
_ = walkFs.Walk("", func(path string, info FileInfo, err error) error {
if path == "file" {
found = true
if !info.ModTime().Equal(newTime) {
@@ -142,8 +142,8 @@ func TestMtimeFSOpen(t *testing.T) {
}
defer func() { _ = os.RemoveAll(dir) }()
underlying := NewFilesystem(FilesystemTypeBasic, dir)
mtimefs := newMtimeFS(underlying, make(mapStore))
mtimefs := newMtimeFS(dir, make(mapStore))
underlying := mtimefs.Filesystem
mtimefs.chtimes = failChtimes
if err := os.WriteFile(filepath.Join(dir, "file"), []byte("hello"), 0644); err != nil {
@@ -222,12 +222,12 @@ func TestMtimeFSInsensitive(t *testing.T) {
// The test should fail with a case sensitive mtimefs
t.Run("with case sensitive mtimefs", func(t *testing.T) {
theTest(t, newMtimeFS(newBasicFilesystem("."), make(mapStore)), false)
theTest(t, newMtimeFS(".", make(mapStore)), false)
})
// And succeed with a case insensitive one.
t.Run("with case insensitive mtimefs", func(t *testing.T) {
theTest(t, newMtimeFS(newBasicFilesystem("."), make(mapStore), WithCaseInsensitivity(true)), true)
theTest(t, newMtimeFS(".", make(mapStore), WithCaseInsensitivity(true)), true)
})
}
@@ -261,6 +261,12 @@ func evilChtimes(name string, mtime, atime time.Time) error {
return os.Chtimes(name, mtime.Add(300*time.Hour).Truncate(time.Hour), atime.Add(300*time.Hour).Truncate(time.Hour))
}
func newMtimeFS(fs Filesystem, db database, options ...MtimeFSOption) *mtimeFS {
return NewMtimeFS(fs, db, options...).(*mtimeFS)
func newMtimeFS(path string, db database, options ...MtimeFSOption) *mtimeFS {
mtimefs, _ := newMtimeFSWithWalk(path, db, options...)
return mtimefs
}
func newMtimeFSWithWalk(path string, db database, options ...MtimeFSOption) (*mtimeFS, *walkFilesystem) {
wfs := NewFilesystem(FilesystemTypeBasic, path, NewMtimeOption(db, options...)).(*walkFilesystem)
return wfs.Filesystem.(*mtimeFS), wfs
}