felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708)

Browse Source 
With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei
2020-07-28 11:15:11 +02:00
committed by Jakob Borg
co-authored by Jakob Borg
parent 21dd9d6b43
commit 932d8c69de
27 changed files with 1241 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/scanner/walk.go
+4
View File
@@ -540,6 +540,10 @@ func (w *walker) handleError(ctx context.Context, context, path string, err erro
}
}
func (w *walker) String() string {
return fmt.Sprintf("walker/%s@%p", w.Folder, w)
}
// A byteCounter gets bytes added to it via Update() and then provides the
// Total() and one minute moving average Rate() in bytes per second.
type byteCounter struct {
lib/scanner/walk_test.go
+28 -25
View File
@@ -26,7 +26,6 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/text/unicode/norm"
@@ -40,17 +39,19 @@ type testfile struct {
type testfileList []testfile
var testFs fs.Filesystem
var testdata = testfileList{
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
{"dir1", 128, ""},
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
{"dir2", 128, ""},
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
}
var (
testFs fs.Filesystem
testFsType = fs.FilesystemTypeBasic
testdata = testfileList{
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
{"dir1", 128, ""},
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
{"dir2", 128, ""},
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
}
)
func init() {
// This test runs the risk of entering infinite recursion if it fails.
@@ -270,7 +271,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
defer os.RemoveAll("_symlinks")
os.Symlink("../testdata", "_symlinks/link")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
fs := fs.NewFilesystem(testFsType, "_symlinks")
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil, 0)
@@ -298,15 +299,15 @@ func TestWalkSymlinkWindows(t *testing.T) {
os.RemoveAll(name)
os.Mkdir(name, 0755)
defer os.RemoveAll(name)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, name)
if err := osutil.DebugSymlinkForTestsOnly("../testdata", "_symlinks/link"); err != nil {
testFs := fs.NewFilesystem(testFsType, name)
if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, "../testdata", "link"); err != nil {
// Probably we require permissions we don't have.
t.Skip(err)
}
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil, 0)
files := walkDir(testFs, path, nil, nil, 0)
// Verify that we got zero symlinks
if len(files) != 0 {
@@ -322,10 +323,12 @@ func TestWalkRootSymlink(t *testing.T) {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
testFs := fs.NewFilesystem(testFsType, tmp)
link := filepath.Join(tmp, "link")
link := "link"
dest, _ := filepath.Abs("testdata/dir1")
if err := osutil.DebugSymlinkForTestsOnly(dest, link); err != nil {
destFs := fs.NewFilesystem(testFsType, dest)
if err := fs.DebugSymlinkForTestsOnly(destFs, testFs, ".", "link"); err != nil {
if runtime.GOOS == "windows" {
// Probably we require permissions we don't have.
t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
@@ -335,15 +338,15 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan root with symlink at FS root
files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0)
files := walkDir(fs.NewFilesystem(testFsType, filepath.Join(testFs.URI(), link)), ".", nil, nil, 0)
// Verify that we got two files
if len(files) != 2 {
t.Errorf("expected two files, not %d", len(files))
t.Fatalf("expected two files, not %d", len(files))
}
// Scan symlink below FS root
files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), "link", nil, nil, 0)
files = walkDir(testFs, "link", nil, nil, 0)
// Verify that we got the one symlink, except on windows
if runtime.GOOS == "windows" {
@@ -355,7 +358,7 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan path below symlink
files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
files = walkDir(fs.NewFilesystem(testFsType, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
// Verify that we get nothing
if len(files) != 0 {
@@ -554,7 +557,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(testFsType, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@@ -652,7 +655,7 @@ func TestIssue4799(t *testing.T) {
}
defer os.RemoveAll(tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
fs := fs.NewFilesystem(testFsType, tmp)
fd, err := fs.Create("foo")
if err != nil {
@@ -714,7 +717,7 @@ func TestIssue4841(t *testing.T) {
}
defer os.RemoveAll(tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
fs := fs.NewFilesystem(testFsType, tmp)
fd, err := fs.Create("foo")
if err != nil {