With this change we emulate a case sensitive filesystem on top of insensitive filesystems. This means we correctly pick up case-only renames and throw a case conflict error when there would be multiple files differing only in case. This safety check has a small performance hit (about 20% more filesystem operations when scanning for changes). The new advanced folder option `caseSensitiveFS` can be used to disable the safety checks, retaining the previous behavior on systems known to be fully case sensitive. Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei
committed by
Jakob Borg
Jakob Borg
co-authored by
Jakob Borg
Jakob Borg
parent
21dd9d6b43
commit
932d8c69de
@@ -0,0 +1,279 @@
|
||||
// Copyright (C) 2020 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRealCase(t *testing.T) {
|
||||
// Verify realCase lookups on various underlying filesystems.
|
||||
|
||||
t.Run("fake-sensitive", func(t *testing.T) {
|
||||
testRealCase(t, newFakeFilesystem(t.Name()))
|
||||
})
|
||||
t.Run("fake-insensitive", func(t *testing.T) {
|
||||
testRealCase(t, newFakeFilesystem(t.Name()+"?insens=true"))
|
||||
})
|
||||
t.Run("actual", func(t *testing.T) {
|
||||
fsys, tmpDir := setup(t)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
testRealCase(t, fsys)
|
||||
})
|
||||
}
|
||||
|
||||
func testRealCase(t *testing.T, fsys Filesystem) {
|
||||
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
|
||||
comps := []string{"Foo", "bar", "BAZ", "bAs"}
|
||||
path := filepath.Join(comps...)
|
||||
testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777)
|
||||
fd, err := testFs.Create(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fd.Close()
|
||||
|
||||
for i, tc := range []struct {
|
||||
in string
|
||||
len int
|
||||
}{
|
||||
{path, 4},
|
||||
{strings.ToLower(path), 4},
|
||||
{strings.ToUpper(path), 4},
|
||||
{"foo", 1},
|
||||
{"FOO", 1},
|
||||
{"foO", 1},
|
||||
{filepath.Join("Foo", "bar"), 2},
|
||||
{filepath.Join("Foo", "bAr"), 2},
|
||||
{filepath.Join("FoO", "bar"), 2},
|
||||
{filepath.Join("foo", "bar", "BAZ"), 3},
|
||||
{filepath.Join("Foo", "bar", "bAz"), 3},
|
||||
{filepath.Join("foo", "bar", "BAZ"), 3}, // Repeat on purpose
|
||||
} {
|
||||
out, err := testFs.realCase(tc.in)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else if exp := filepath.Join(comps[:tc.len]...); out != exp {
|
||||
t.Errorf("tc %v: Expected %v, got %v", i, exp, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealCaseSensitive(t *testing.T) {
|
||||
// Verify that realCase returns the best on-disk case for case sensitive
|
||||
// systems. Test is skipped if the underlying fs is insensitive.
|
||||
|
||||
t.Run("fake-sensitive", func(t *testing.T) {
|
||||
testRealCaseSensitive(t, newFakeFilesystem(t.Name()))
|
||||
})
|
||||
t.Run("actual", func(t *testing.T) {
|
||||
fsys, tmpDir := setup(t)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
testRealCaseSensitive(t, fsys)
|
||||
})
|
||||
}
|
||||
|
||||
func testRealCaseSensitive(t *testing.T, fsys Filesystem) {
|
||||
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
|
||||
|
||||
names := make([]string, 2)
|
||||
names[0] = "foo"
|
||||
names[1] = strings.ToUpper(names[0])
|
||||
for _, n := range names {
|
||||
if err := testFs.MkdirAll(n, 0777); err != nil {
|
||||
if IsErrCaseConflict(err) {
|
||||
t.Skip("Filesystem is case-insensitive")
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, n := range names {
|
||||
if rn, err := testFs.realCase(n); err != nil {
|
||||
t.Error(err)
|
||||
} else if rn != n {
|
||||
t.Errorf("Got %v, expected %v", rn, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaseFSStat(t *testing.T) {
|
||||
// Verify that a Stat() lookup behaves in a case sensitive manner
|
||||
// regardless of the underlying fs.
|
||||
|
||||
t.Run("fake-sensitive", func(t *testing.T) {
|
||||
testCaseFSStat(t, newFakeFilesystem(t.Name()))
|
||||
})
|
||||
t.Run("fake-insensitive", func(t *testing.T) {
|
||||
testCaseFSStat(t, newFakeFilesystem(t.Name()+"?insens=true"))
|
||||
})
|
||||
t.Run("actual", func(t *testing.T) {
|
||||
fsys, tmpDir := setup(t)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
testCaseFSStat(t, fsys)
|
||||
})
|
||||
}
|
||||
|
||||
func testCaseFSStat(t *testing.T, fsys Filesystem) {
|
||||
fd, err := fsys.Create("foo")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fd.Close()
|
||||
|
||||
// Check if the underlying fs is sensitive or not
|
||||
sensitive := true
|
||||
if _, err = fsys.Stat("FOO"); err == nil {
|
||||
sensitive = false
|
||||
}
|
||||
|
||||
testFs := NewCaseFilesystem(fsys)
|
||||
_, err = testFs.Stat("FOO")
|
||||
if sensitive {
|
||||
if IsNotExist(err) {
|
||||
t.Log("pass: case sensitive underlying fs")
|
||||
} else {
|
||||
t.Error("expected NotExist, not", err, "for sensitive fs")
|
||||
}
|
||||
} else if IsErrCaseConflict(err) {
|
||||
t.Log("pass: case insensitive underlying fs")
|
||||
} else {
|
||||
t.Error("expected ErrCaseConflict, not", err, "for insensitive fs")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWalkCaseFakeFS10k(b *testing.B) {
|
||||
fsys, paths, err := fakefsForBenchmark(10_000, 0)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
slowsys, paths, err := fakefsForBenchmark(10_000, 100*time.Microsecond)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.Run("raw-fastfs", func(b *testing.B) {
|
||||
benchmarkWalkFakeFS(b, fsys, paths)
|
||||
b.ReportAllocs()
|
||||
})
|
||||
b.Run("case-fastfs", func(b *testing.B) {
|
||||
benchmarkWalkFakeFS(b, NewCaseFilesystem(fsys), paths)
|
||||
b.ReportAllocs()
|
||||
})
|
||||
b.Run("raw-slowfs", func(b *testing.B) {
|
||||
benchmarkWalkFakeFS(b, slowsys, paths)
|
||||
b.ReportAllocs()
|
||||
})
|
||||
b.Run("case-slowfs", func(b *testing.B) {
|
||||
benchmarkWalkFakeFS(b, NewCaseFilesystem(slowsys), paths)
|
||||
b.ReportAllocs()
|
||||
})
|
||||
}
|
||||
|
||||
func benchmarkWalkFakeFS(b *testing.B, fsys Filesystem, paths []string) {
|
||||
// Simulate a scanner pass over the filesystem. First walk it to
|
||||
// discover all names, then stat each name individually to check if it's
|
||||
// been deleted or not (pretending that they all existed in the
|
||||
// database).
|
||||
|
||||
var ms0 runtime.MemStats
|
||||
runtime.ReadMemStats(&ms0)
|
||||
t0 := time.Now()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
if err := doubleWalkFS(fsys, paths); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
t1 := time.Now()
|
||||
var ms1 runtime.MemStats
|
||||
runtime.ReadMemStats(&ms1)
|
||||
|
||||
// We add metrics per path entry
|
||||
b.ReportMetric(float64(t1.Sub(t0))/float64(b.N)/float64(len(paths)), "ns/entry")
|
||||
b.ReportMetric(float64(ms1.Mallocs-ms0.Mallocs)/float64(b.N)/float64(len(paths)), "allocs/entry")
|
||||
b.ReportMetric(float64(ms1.TotalAlloc-ms0.TotalAlloc)/float64(b.N)/float64(len(paths)), "B/entry")
|
||||
}
|
||||
|
||||
func TestStressCaseFS(t *testing.T) {
|
||||
// Exercise a bunch of paralell operations for stressing out race
|
||||
// conditions in the realnamer cache etc.
|
||||
|
||||
const limit = 10 * time.Second
|
||||
if testing.Short() {
|
||||
t.Skip("long test")
|
||||
}
|
||||
|
||||
fsys, paths, err := fakefsForBenchmark(10_000, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < runtime.NumCPU()/2+1; i++ {
|
||||
t.Run(fmt.Sprintf("walker-%d", i), func(t *testing.T) {
|
||||
// Walk the filesystem and stat everything
|
||||
t.Parallel()
|
||||
t0 := time.Now()
|
||||
for time.Since(t0) < limit {
|
||||
if err := doubleWalkFS(fsys, paths); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run(fmt.Sprintf("toucher-%d", i), func(t *testing.T) {
|
||||
// Touch all the things
|
||||
t.Parallel()
|
||||
t0 := time.Now()
|
||||
for time.Since(t0) < limit {
|
||||
for _, p := range paths {
|
||||
now := time.Now()
|
||||
if err := fsys.Chtimes(p, now, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func doubleWalkFS(fsys Filesystem, paths []string) error {
|
||||
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
if _, err := fsys.Lstat(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fakefsForBenchmark(nfiles int, latency time.Duration) (Filesystem, []string, error) {
|
||||
fsys := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("fakefsForBenchmark?files=%d&insens=true&latency=%s", nfiles, latency))
|
||||
|
||||
var paths []string
|
||||
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
|
||||
paths = append(paths, path)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(paths) < nfiles {
|
||||
return nil, nil, errors.New("didn't find enough stuff")
|
||||
}
|
||||
|
||||
return fsys, paths, nil
|
||||
}
|
||||
Reference in New Issue
Block a user