felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708)

Browse Source 
With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei
2020-07-28 11:15:11 +02:00
committed by Jakob Borg
co-authored by Jakob Borg
parent 21dd9d6b43
commit 932d8c69de
27 changed files with 1241 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/osutil/osutil.go
+4 -2
View File
@@ -131,8 +131,10 @@ func copyFileContents(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, src
}
func IsDeleted(ffs fs.Filesystem, name string) bool {
if _, err := ffs.Lstat(name); fs.IsNotExist(err) {
return true
if _, err := ffs.Lstat(name); err != nil {
if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
return true
}
}
switch TraversesSymlink(ffs, filepath.Dir(name)).(type) {
case *NotADirectoryError, *TraversesSymlinkError:
lib/osutil/osutil_test.go
+1 -1
View File
@@ -62,7 +62,7 @@ func TestIsDeleted(t *testing.T) {
}
}
for _, n := range []string{"Dir", "File", "Del"} {
if err := osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), strings.ToLower(n)), filepath.Join(testFs.URI(), "linkTo"+n)); err != nil {
if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, strings.ToLower(n), "linkTo"+n); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}
lib/osutil/symlink.go
-21
View File
@@ -1,21 +0,0 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"os"
)
// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
// hence the cumbersome name to make it obvious if this ever leaks. Its
// reason for existence is the Windows version, which allows creating
// symlinks when non-elevated.
func DebugSymlinkForTestsOnly(oldname, newname string) error {
return os.Symlink(oldname, newname)
}
lib/osutil/symlink_windows.go
-137
View File
@@ -1,137 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package osutil
import (
"os"
"path/filepath"
"syscall"
)
// DebugSymlinkForTestsOnly is os.Symlink taken from the 1.9.2 stdlib,
// hacked with the SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE flag to
// create symlinks when not elevated.
//
// This is not and should not be used in Syncthing code, hence the
// cumbersome name to make it obvious if this ever leaks. Nonetheless it's
// useful in tests.
func DebugSymlinkForTestsOnly(oldname, newname string) error {
// CreateSymbolicLink is not supported before Windows Vista
if syscall.LoadCreateSymbolicLink() != nil {
return &os.LinkError{"symlink", oldname, newname, syscall.EWINDOWS}
}
// '/' does not work in link's content
oldname = filepath.FromSlash(oldname)
// need the exact location of the oldname when it's relative to determine if it's a directory
destpath := oldname
if !filepath.IsAbs(oldname) {
destpath = filepath.Dir(newname) + `\` + oldname
}
fi, err := os.Lstat(destpath)
isdir := err == nil && fi.IsDir()
n, err := syscall.UTF16PtrFromString(fixLongPath(newname))
if err != nil {
return &os.LinkError{"symlink", oldname, newname, err}
}
o, err := syscall.UTF16PtrFromString(fixLongPath(oldname))
if err != nil {
return &os.LinkError{"symlink", oldname, newname, err}
}
var flags uint32
if isdir {
flags |= syscall.SYMBOLIC_LINK_FLAG_DIRECTORY
}
flags |= 0x02 // SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
err = syscall.CreateSymbolicLink(n, o, flags)
if err != nil {
return &os.LinkError{"symlink", oldname, newname, err}
}
return nil
}
// fixLongPath returns the extended-length (\\?\-prefixed) form of
// path when needed, in order to avoid the default 260 character file
// path limit imposed by Windows. If path is not easily converted to
// the extended-length form (for example, if path is a relative path
// or contains .. elements), or is short enough, fixLongPath returns
// path unmodified.
//
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
func fixLongPath(path string) string {
// Do nothing (and don't allocate) if the path is "short".
// Empirically (at least on the Windows Server 2013 builder),
// the kernel is arbitrarily okay with < 248 bytes. That
// matches what the docs above say:
// "When using an API to create a directory, the specified
// path cannot be so long that you cannot append an 8.3 file
// name (that is, the directory name cannot exceed MAX_PATH
// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
//
// The MSDN docs appear to say that a normal path that is 248 bytes long
// will work; empirically the path must be less than 248 bytes long.
if len(path) < 248 {
// Don't fix. (This is how Go 1.7 and earlier worked,
// not automatically generating the \\?\ form)
return path
}
// The extended form begins with \\?\, as in
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
// The extended form disables evaluation of . and .. path
// elements and disables the interpretation of / as equivalent
// to \. The conversion here rewrites / to \ and elides
// . elements as well as trailing or duplicate separators. For
// simplicity it avoids the conversion entirely for relative
// paths or paths containing .. elements. For now,
// \\server\share paths are not converted to
// \\?\UNC\server\share paths because the rules for doing so
// are less well-specified.
if len(path) >= 2 && path[:2] == `\\` {
// Don't canonicalize UNC paths.
return path
}
if !filepath.IsAbs(path) {
// Relative path
return path
}
const prefix = `\\?`
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
copy(pathbuf, prefix)
n := len(path)
r, w := 0, len(prefix)
for r < n {
switch {
case os.IsPathSeparator(path[r]):
// empty block
r++
case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
// /./
r++
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
// /../ is currently unhandled
return path
default:
pathbuf[w] = '\\'
w++
for ; r < n && !os.IsPathSeparator(path[r]); r++ {
pathbuf[w] = path[r]
w++
}
}
}
// A drive's root directory needs a trailing \
if w == len(`\\?\c:`) {
pathbuf[w] = '\\'
w++
}
return string(pathbuf[:w])
}
lib/osutil/traversessymlink_test.go
+7 -7
View File
@@ -24,9 +24,9 @@ func TestTraversesSymlink(t *testing.T) {
}
defer os.RemoveAll(tmpDir)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
fs.MkdirAll("a/b/c", 0755)
if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(fs.URI(), "a", "b"), filepath.Join(fs.URI(), "a", "l")); err != nil {
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
testFs.MkdirAll("a/b/c", 0755)
if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}
@@ -34,7 +34,7 @@ func TestTraversesSymlink(t *testing.T) {
}
// a/l -> b, so a/l/c should resolve by normal stat
info, err := fs.Lstat("a/l/c")
info, err := testFs.Lstat("a/l/c")
if err != nil {
t.Fatal("unexpected error", err)
}
@@ -64,7 +64,7 @@ func TestTraversesSymlink(t *testing.T) {
}
for _, tc := range cases {
if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) {
if res := osutil.TraversesSymlink(testFs, tc.name); tc.traverses == (res == nil) {
t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses)
}
}
@@ -78,8 +78,8 @@ func TestIssue4875(t *testing.T) {
defer os.RemoveAll(tmpDir)
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
testFs.MkdirAll("a/b/c", 0755)
if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "a", "b"), filepath.Join(testFs.URI(), "a", "l")); err != nil {
testFs.MkdirAll(filepath.Join("a", "b", "c"), 0755)
if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}