lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708)

With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>

lib/fs/basicfs_realcaser_unix.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package fs
+
+func newBasicRealCaser(fs Filesystem) realCaser {
+	return newDefaultRealCaser(fs)
+}

lib/fs/basicfs_realcaser_windows.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build windows
+
+package fs
+
+import (
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+type basicRealCaserWindows struct {
+	uri string
+}
+
+func newBasicRealCaser(fs Filesystem) realCaser {
+	return &basicRealCaserWindows{fs.URI()}
+}
+
+// RealCase returns the correct case for the given name, which is a relative
+// path below root, as it exists on disk.
+func (r *basicRealCaserWindows) realCase(name string) (string, error) {
+	if name == "." {
+		return ".", nil
+	}
+	path := r.uri
+	comps := strings.Split(name, string(PathSeparator))
+	var err error
+	for i, comp := range comps {
+		path = filepath.Join(path, comp)
+		comps[i], err = r.realCaseBase(path)
+		if err != nil {
+			return "", err
+		}
+	}
+	return filepath.Join(comps...), nil
+}
+
+func (*basicRealCaserWindows) realCaseBase(path string) (string, error) {
+	p, err := syscall.UTF16PtrFromString(fixLongPath(path))
+	if err != nil {
+		return "", err
+	}
+	var fd syscall.Win32finddata
+	h, err := syscall.FindFirstFile(p, &fd)
+	if err != nil {
+		return "", err
+	}
+	syscall.FindClose(h)
+	return syscall.UTF16ToString(fd.FileName[:]), nil
+}
+
+func (r *basicRealCaserWindows) dropCache() {}

lib/fs/casefs.go

@@ -0,0 +1,448 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Both values were chosen by magic.
+const (
+	caseCacheTimeout = time.Second
+	// When the number of names (all lengths of []string from DirNames)
+	// exceeds this, we drop the cache.
+	caseMaxCachedNames = 1 << 20
+)
+
+type ErrCaseConflict struct {
+	given, real string
+}
+
+func (e *ErrCaseConflict) Error() string {
+	return fmt.Sprintf(`given name "%v" differs from name in filesystem "%v"`, e.given, e.real)
+}
+
+func IsErrCaseConflict(err error) bool {
+	e := &ErrCaseConflict{}
+	return errors.As(err, &e)
+}
+
+type realCaser interface {
+	realCase(name string) (string, error)
+	dropCache()
+}
+
+type fskey struct {
+	fstype FilesystemType
+	uri    string
+}
+
+var (
+	caseFilesystems    = make(map[fskey]Filesystem)
+	caseFilesystemsMut sync.Mutex
+)
+
+// caseFilesystem is a BasicFilesystem with additional checks to make a
+// potentially case insensitive underlying FS behave like it's case-sensitive.
+type caseFilesystem struct {
+	Filesystem
+	realCaser
+}
+
+// NewCaseFilesystem ensures that the given, potentially case-insensitive filesystem
+// behaves like a case-sensitive filesystem. Meaning that it takes into account
+// the real casing of a path and returns ErrCaseConflict if the given path differs
+// from the real path. It is safe to use with any filesystem, i.e. also a
+// case-sensitive one. However it will add some overhead and thus shouldn't be
+// used if the filesystem is known to already behave case-sensitively.
+func NewCaseFilesystem(fs Filesystem) Filesystem {
+	caseFilesystemsMut.Lock()
+	defer caseFilesystemsMut.Unlock()
+	k := fskey{fs.Type(), fs.URI()}
+	if caseFs, ok := caseFilesystems[k]; ok {
+		return caseFs
+	}
+	caseFs := &caseFilesystem{
+		Filesystem: fs,
+	}
+	switch k.fstype {
+	case FilesystemTypeBasic:
+		caseFs.realCaser = newBasicRealCaser(fs)
+	default:
+		caseFs.realCaser = newDefaultRealCaser(fs)
+	}
+	caseFilesystems[k] = caseFs
+	return caseFs
+}
+
+func (f *caseFilesystem) Chmod(name string, mode FileMode) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Chmod(name, mode)
+}
+
+func (f *caseFilesystem) Lchown(name string, uid, gid int) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Lchown(name, uid, gid)
+}
+
+func (f *caseFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Chtimes(name, atime, mtime)
+}
+
+func (f *caseFilesystem) Mkdir(name string, perm FileMode) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.Mkdir(name, perm); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) MkdirAll(path string, perm FileMode) error {
+	if err := f.checkCase(path); err != nil {
+		return err
+	}
+	if err := f.Filesystem.MkdirAll(path, perm); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Lstat(name string) (FileInfo, error) {
+	var err error
+	if name, err = Canonicalize(name); err != nil {
+		return nil, err
+	}
+	stat, err := f.Filesystem.Lstat(name)
+	if err != nil {
+		return nil, err
+	}
+	if err = f.checkCaseExisting(name); err != nil {
+		return nil, err
+	}
+	return stat, nil
+}
+
+func (f *caseFilesystem) Remove(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.Remove(name); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) RemoveAll(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.RemoveAll(name); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Rename(oldpath, newpath string) error {
+	if err := f.checkCase(oldpath); err != nil {
+		return err
+	}
+	if err := f.Filesystem.Rename(oldpath, newpath); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Stat(name string) (FileInfo, error) {
+	var err error
+	if name, err = Canonicalize(name); err != nil {
+		return nil, err
+	}
+	stat, err := f.Filesystem.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+	if err = f.checkCaseExisting(name); err != nil {
+		return nil, err
+	}
+	return stat, nil
+}
+
+func (f *caseFilesystem) DirNames(name string) ([]string, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	return f.Filesystem.DirNames(name)
+}
+
+func (f *caseFilesystem) Open(name string) (File, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	return f.Filesystem.Open(name)
+}
+
+func (f *caseFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	file, err := f.Filesystem.OpenFile(name, flags, mode)
+	if err != nil {
+		return nil, err
+	}
+	f.dropCache()
+	return file, nil
+}
+
+func (f *caseFilesystem) ReadSymlink(name string) (string, error) {
+	if err := f.checkCase(name); err != nil {
+		return "", err
+	}
+	return f.Filesystem.ReadSymlink(name)
+}
+
+func (f *caseFilesystem) Create(name string) (File, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	file, err := f.Filesystem.Create(name)
+	if err != nil {
+		return nil, err
+	}
+	f.dropCache()
+	return file, nil
+}
+
+func (f *caseFilesystem) CreateSymlink(target, name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.CreateSymlink(target, name); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Walk(root string, walkFn WalkFunc) error {
+	// Walking the filesystem is likely (in Syncthing's case certainly) done
+	// to pick up external changes, for which caching is undesirable.
+	f.dropCache()
+	if err := f.checkCase(root); err != nil {
+		return err
+	}
+	return f.Filesystem.Walk(root, walkFn)
+}
+
+func (f *caseFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
+	if err := f.checkCase(path); err != nil {
+		return nil, nil, err
+	}
+	return f.Filesystem.Watch(path, ignore, ctx, ignorePerms)
+}
+
+func (f *caseFilesystem) Hide(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Hide(name)
+}
+
+func (f *caseFilesystem) Unhide(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Unhide(name)
+}
+
+func (f *caseFilesystem) checkCase(name string) error {
+	var err error
+	if name, err = Canonicalize(name); err != nil {
+		return err
+	}
+	// Stat is necessary for case sensitive FS, as it's then not a conflict
+	// if name is e.g. "foo" and on dir there is "Foo".
+	if _, err := f.Filesystem.Lstat(name); err != nil {
+		if IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	return f.checkCaseExisting(name)
+}
+
+// checkCaseExisting must only be called after successfully canonicalizing and
+// stating the file.
+func (f *caseFilesystem) checkCaseExisting(name string) error {
+	realName, err := f.realCase(name)
+	if IsNotExist(err) {
+		// It did exist just before -> cache is outdated, try again
+		f.dropCache()
+		realName, err = f.realCase(name)
+	}
+	if err != nil {
+		return err
+	}
+	if realName != name {
+		return &ErrCaseConflict{name, realName}
+	}
+	return nil
+}
+
+type defaultRealCaser struct {
+	fs        Filesystem
+	root      *caseNode
+	count     int
+	timer     *time.Timer
+	timerStop chan struct{}
+	mut       sync.RWMutex
+}
+
+func newDefaultRealCaser(fs Filesystem) *defaultRealCaser {
+	caser := &defaultRealCaser{
+		fs:    fs,
+		root:  &caseNode{name: "."},
+		timer: time.NewTimer(0),
+	}
+	<-caser.timer.C
+	return caser
+}
+
+func (r *defaultRealCaser) realCase(name string) (string, error) {
+	out := "."
+	if name == out {
+		return out, nil
+	}
+
+	r.mut.Lock()
+	defer func() {
+		if r.count > caseMaxCachedNames {
+			select {
+			case r.timerStop <- struct{}{}:
+			default:
+			}
+			r.dropCacheLocked()
+		}
+		r.mut.Unlock()
+	}()
+
+	node := r.root
+	for _, comp := range strings.Split(name, string(PathSeparator)) {
+		if node.dirNames == nil {
+			// Haven't called DirNames yet
+			var err error
+			node.dirNames, err = r.fs.DirNames(out)
+			if err != nil {
+				return "", err
+			}
+			node.dirNamesLower = make([]string, len(node.dirNames))
+			for i, n := range node.dirNames {
+				node.dirNamesLower[i] = UnicodeLowercase(n)
+			}
+			node.children = make(map[string]*caseNode)
+			node.results = make(map[string]*caseNode)
+			r.count += len(node.dirNames)
+		} else if child, ok := node.results[comp]; ok {
+			// Check if this exact name has been queried before to shortcut
+			node = child
+			out = filepath.Join(out, child.name)
+			continue
+		}
+		// Actually loop dirNames to search for a match
+		n, err := findCaseInsensitiveMatch(comp, node.dirNames, node.dirNamesLower)
+		if err != nil {
+			return "", err
+		}
+		child, ok := node.children[n]
+		if !ok {
+			child = &caseNode{name: n}
+		}
+		node.results[comp] = child
+		node.children[n] = child
+		node = child
+		out = filepath.Join(out, n)
+	}
+
+	return out, nil
+}
+
+func (r *defaultRealCaser) startCaseResetTimerLocked() {
+	r.timerStop = make(chan struct{})
+	r.timer.Reset(caseCacheTimeout)
+	go func() {
+		select {
+		case <-r.timer.C:
+			r.dropCache()
+		case <-r.timerStop:
+			if !r.timer.Stop() {
+				<-r.timer.C
+			}
+			r.mut.Lock()
+			r.timerStop = nil
+			r.mut.Unlock()
+		}
+	}()
+}
+
+func (r *defaultRealCaser) dropCache() {
+	r.mut.Lock()
+	r.dropCacheLocked()
+	r.mut.Unlock()
+}
+
+func (r *defaultRealCaser) dropCacheLocked() {
+	r.root = &caseNode{name: "."}
+	r.count = 0
+}
+
+// Both name and the key to children are "Real", case resolved names of the path
+// component this node represents (i.e. containing no path separator).
+// The key to results is also a path component, but as given to RealCase, not
+// case resolved.
+type caseNode struct {
+	name          string
+	dirNames      []string
+	dirNamesLower []string
+	children      map[string]*caseNode
+	results       map[string]*caseNode
+}
+
+func findCaseInsensitiveMatch(name string, names, namesLower []string) (string, error) {
+	lower := UnicodeLowercase(name)
+	candidate := ""
+	for i, n := range names {
+		if n == name {
+			return n, nil
+		}
+		if candidate == "" && namesLower[i] == lower {
+			candidate = n
+		}
+	}
+	if candidate == "" {
+		return "", ErrNotExist
+	}
+	return candidate, nil
+}

lib/fs/casefs_test.go

@@ -0,0 +1,279 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestRealCase(t *testing.T) {
+	// Verify realCase lookups on various underlying filesystems.
+
+	t.Run("fake-sensitive", func(t *testing.T) {
+		testRealCase(t, newFakeFilesystem(t.Name()))
+	})
+	t.Run("fake-insensitive", func(t *testing.T) {
+		testRealCase(t, newFakeFilesystem(t.Name()+"?insens=true"))
+	})
+	t.Run("actual", func(t *testing.T) {
+		fsys, tmpDir := setup(t)
+		defer os.RemoveAll(tmpDir)
+		testRealCase(t, fsys)
+	})
+}
+
+func testRealCase(t *testing.T, fsys Filesystem) {
+	testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
+	comps := []string{"Foo", "bar", "BAZ", "bAs"}
+	path := filepath.Join(comps...)
+	testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777)
+	fd, err := testFs.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fd.Close()
+
+	for i, tc := range []struct {
+		in  string
+		len int
+	}{
+		{path, 4},
+		{strings.ToLower(path), 4},
+		{strings.ToUpper(path), 4},
+		{"foo", 1},
+		{"FOO", 1},
+		{"foO", 1},
+		{filepath.Join("Foo", "bar"), 2},
+		{filepath.Join("Foo", "bAr"), 2},
+		{filepath.Join("FoO", "bar"), 2},
+		{filepath.Join("foo", "bar", "BAZ"), 3},
+		{filepath.Join("Foo", "bar", "bAz"), 3},
+		{filepath.Join("foo", "bar", "BAZ"), 3}, // Repeat on purpose
+	} {
+		out, err := testFs.realCase(tc.in)
+		if err != nil {
+			t.Error(err)
+		} else if exp := filepath.Join(comps[:tc.len]...); out != exp {
+			t.Errorf("tc %v: Expected %v, got %v", i, exp, out)
+		}
+	}
+}
+
+func TestRealCaseSensitive(t *testing.T) {
+	// Verify that realCase returns the best on-disk case for case sensitive
+	// systems. Test is skipped if the underlying fs is insensitive.
+
+	t.Run("fake-sensitive", func(t *testing.T) {
+		testRealCaseSensitive(t, newFakeFilesystem(t.Name()))
+	})
+	t.Run("actual", func(t *testing.T) {
+		fsys, tmpDir := setup(t)
+		defer os.RemoveAll(tmpDir)
+		testRealCaseSensitive(t, fsys)
+	})
+}
+
+func testRealCaseSensitive(t *testing.T, fsys Filesystem) {
+	testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
+
+	names := make([]string, 2)
+	names[0] = "foo"
+	names[1] = strings.ToUpper(names[0])
+	for _, n := range names {
+		if err := testFs.MkdirAll(n, 0777); err != nil {
+			if IsErrCaseConflict(err) {
+				t.Skip("Filesystem is case-insensitive")
+			}
+			t.Fatal(err)
+		}
+	}
+
+	for _, n := range names {
+		if rn, err := testFs.realCase(n); err != nil {
+			t.Error(err)
+		} else if rn != n {
+			t.Errorf("Got %v, expected %v", rn, n)
+		}
+	}
+}
+
+func TestCaseFSStat(t *testing.T) {
+	// Verify that a Stat() lookup behaves in a case sensitive manner
+	// regardless of the underlying fs.
+
+	t.Run("fake-sensitive", func(t *testing.T) {
+		testCaseFSStat(t, newFakeFilesystem(t.Name()))
+	})
+	t.Run("fake-insensitive", func(t *testing.T) {
+		testCaseFSStat(t, newFakeFilesystem(t.Name()+"?insens=true"))
+	})
+	t.Run("actual", func(t *testing.T) {
+		fsys, tmpDir := setup(t)
+		defer os.RemoveAll(tmpDir)
+		testCaseFSStat(t, fsys)
+	})
+}
+
+func testCaseFSStat(t *testing.T, fsys Filesystem) {
+	fd, err := fsys.Create("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fd.Close()
+
+	// Check if the underlying fs is sensitive or not
+	sensitive := true
+	if _, err = fsys.Stat("FOO"); err == nil {
+		sensitive = false
+	}
+
+	testFs := NewCaseFilesystem(fsys)
+	_, err = testFs.Stat("FOO")
+	if sensitive {
+		if IsNotExist(err) {
+			t.Log("pass: case sensitive underlying fs")
+		} else {
+			t.Error("expected NotExist, not", err, "for sensitive fs")
+		}
+	} else if IsErrCaseConflict(err) {
+		t.Log("pass: case insensitive underlying fs")
+	} else {
+		t.Error("expected ErrCaseConflict, not", err, "for insensitive fs")
+	}
+}
+
+func BenchmarkWalkCaseFakeFS10k(b *testing.B) {
+	fsys, paths, err := fakefsForBenchmark(10_000, 0)
+	if err != nil {
+		b.Fatal(err)
+	}
+	slowsys, paths, err := fakefsForBenchmark(10_000, 100*time.Microsecond)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.Run("raw-fastfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, fsys, paths)
+		b.ReportAllocs()
+	})
+	b.Run("case-fastfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, NewCaseFilesystem(fsys), paths)
+		b.ReportAllocs()
+	})
+	b.Run("raw-slowfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, slowsys, paths)
+		b.ReportAllocs()
+	})
+	b.Run("case-slowfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, NewCaseFilesystem(slowsys), paths)
+		b.ReportAllocs()
+	})
+}
+
+func benchmarkWalkFakeFS(b *testing.B, fsys Filesystem, paths []string) {
+	// Simulate a scanner pass over the filesystem. First walk it to
+	// discover all names, then stat each name individually to check if it's
+	// been deleted or not (pretending that they all existed in the
+	// database).
+
+	var ms0 runtime.MemStats
+	runtime.ReadMemStats(&ms0)
+	t0 := time.Now()
+
+	for i := 0; i < b.N; i++ {
+		if err := doubleWalkFS(fsys, paths); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	t1 := time.Now()
+	var ms1 runtime.MemStats
+	runtime.ReadMemStats(&ms1)
+
+	// We add metrics per path entry
+	b.ReportMetric(float64(t1.Sub(t0))/float64(b.N)/float64(len(paths)), "ns/entry")
+	b.ReportMetric(float64(ms1.Mallocs-ms0.Mallocs)/float64(b.N)/float64(len(paths)), "allocs/entry")
+	b.ReportMetric(float64(ms1.TotalAlloc-ms0.TotalAlloc)/float64(b.N)/float64(len(paths)), "B/entry")
+}
+
+func TestStressCaseFS(t *testing.T) {
+	// Exercise a bunch of paralell operations for stressing out race
+	// conditions in the realnamer cache etc.
+
+	const limit = 10 * time.Second
+	if testing.Short() {
+		t.Skip("long test")
+	}
+
+	fsys, paths, err := fakefsForBenchmark(10_000, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < runtime.NumCPU()/2+1; i++ {
+		t.Run(fmt.Sprintf("walker-%d", i), func(t *testing.T) {
+			// Walk the filesystem and stat everything
+			t.Parallel()
+			t0 := time.Now()
+			for time.Since(t0) < limit {
+				if err := doubleWalkFS(fsys, paths); err != nil {
+					t.Fatal(err)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("toucher-%d", i), func(t *testing.T) {
+			// Touch all the things
+			t.Parallel()
+			t0 := time.Now()
+			for time.Since(t0) < limit {
+				for _, p := range paths {
+					now := time.Now()
+					if err := fsys.Chtimes(p, now, now); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+		})
+	}
+}
+
+func doubleWalkFS(fsys Filesystem, paths []string) error {
+	if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
+		return err
+	}); err != nil {
+		return err
+	}
+
+	for _, p := range paths {
+		if _, err := fsys.Lstat(p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func fakefsForBenchmark(nfiles int, latency time.Duration) (Filesystem, []string, error) {
+	fsys := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("fakefsForBenchmark?files=%d&insens=true&latency=%s", nfiles, latency))
+
+	var paths []string
+	if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
+		paths = append(paths, path)
+		return err
+	}); err != nil {
+		return nil, nil, err
+	}
+	if len(paths) < nfiles {
+		return nil, nil, errors.New("didn't find enough stuff")
+	}
+
+	return fsys, paths, nil
+}

lib/fs/debug_symlink_unix.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package fs
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
+// hence the cumbersome name to make it obvious if this ever leaks. Its
+// reason for existence is the Windows version, which allows creating
+// symlinks when non-elevated.
+func DebugSymlinkForTestsOnly(oldFs, newFs Filesystem, oldname, newname string) error {
+	if caseFs, ok := unwrapFilesystem(newFs).(*caseFilesystem); ok {
+		if err := caseFs.checkCase(newname); err != nil {
+			return err
+		}
+		caseFs.dropCache()
+	}
+	if err := os.Symlink(filepath.Join(oldFs.URI(), oldname), filepath.Join(newFs.URI(), newname)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// unwrapFilesystem removes "wrapping" filesystems to expose the underlying filesystem.
+func unwrapFilesystem(fs Filesystem) Filesystem {
+	for {
+		switch sfs := fs.(type) {
+		case *logFilesystem:
+			fs = sfs.Filesystem
+		case *walkFilesystem:
+			fs = sfs.Filesystem
+		case *MtimeFS:
+			fs = sfs.Filesystem
+		default:
+			return sfs
+		}
+	}
+}

lib/fs/debug_symlink_windows.go

@@ -0,0 +1,140 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fs
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// DebugSymlinkForTestsOnly is os.Symlink taken from the 1.9.2 stdlib,
+// hacked with the SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE flag to
+// create symlinks when not elevated.
+//
+// This is not and should not be used in Syncthing code, hence the
+// cumbersome name to make it obvious if this ever leaks. Nonetheless it's
+// useful in tests.
+func DebugSymlinkForTestsOnly(oldFs, newFS Filesystem, oldname, newname string) error {
+	oldname = filepath.Join(oldFs.URI(), oldname)
+	newname = filepath.Join(newFS.URI(), newname)
+
+	// CreateSymbolicLink is not supported before Windows Vista
+	if syscall.LoadCreateSymbolicLink() != nil {
+		return &os.LinkError{"symlink", oldname, newname, syscall.EWINDOWS}
+	}
+
+	// '/' does not work in link's content
+	oldname = filepath.FromSlash(oldname)
+
+	// need the exact location of the oldname when it's relative to determine if it's a directory
+	destpath := oldname
+	if !filepath.IsAbs(oldname) {
+		destpath = filepath.Dir(newname) + `\` + oldname
+	}
+
+	fi, err := os.Lstat(destpath)
+	isdir := err == nil && fi.IsDir()
+
+	n, err := syscall.UTF16PtrFromString(fixLongPath(newname))
+	if err != nil {
+		return &os.LinkError{"symlink", oldname, newname, err}
+	}
+	o, err := syscall.UTF16PtrFromString(fixLongPath(oldname))
+	if err != nil {
+		return &os.LinkError{"symlink", oldname, newname, err}
+	}
+
+	var flags uint32
+	if isdir {
+		flags |= syscall.SYMBOLIC_LINK_FLAG_DIRECTORY
+	}
+	flags |= 0x02 // SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
+	err = syscall.CreateSymbolicLink(n, o, flags)
+	if err != nil {
+		return &os.LinkError{"symlink", oldname, newname, err}
+	}
+	return nil
+}
+
+// fixLongPath returns the extended-length (\\?\-prefixed) form of
+// path when needed, in order to avoid the default 260 character file
+// path limit imposed by Windows. If path is not easily converted to
+// the extended-length form (for example, if path is a relative path
+// or contains .. elements), or is short enough, fixLongPath returns
+// path unmodified.
+//
+// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+func fixLongPath(path string) string {
+	// Do nothing (and don't allocate) if the path is "short".
+	// Empirically (at least on the Windows Server 2013 builder),
+	// the kernel is arbitrarily okay with < 248 bytes. That
+	// matches what the docs above say:
+	// "When using an API to create a directory, the specified
+	// path cannot be so long that you cannot append an 8.3 file
+	// name (that is, the directory name cannot exceed MAX_PATH
+	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
+	//
+	// The MSDN docs appear to say that a normal path that is 248 bytes long
+	// will work; empirically the path must be less than 248 bytes long.
+	if len(path) < 248 {
+		// Don't fix. (This is how Go 1.7 and earlier worked,
+		// not automatically generating the \\?\ form)
+		return path
+	}
+
+	// The extended form begins with \\?\, as in
+	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
+	// The extended form disables evaluation of . and .. path
+	// elements and disables the interpretation of / as equivalent
+	// to \. The conversion here rewrites / to \ and elides
+	// . elements as well as trailing or duplicate separators. For
+	// simplicity it avoids the conversion entirely for relative
+	// paths or paths containing .. elements. For now,
+	// \\server\share paths are not converted to
+	// \\?\UNC\server\share paths because the rules for doing so
+	// are less well-specified.
+	if len(path) >= 2 && path[:2] == `\\` {
+		// Don't canonicalize UNC paths.
+		return path
+	}
+	if !filepath.IsAbs(path) {
+		// Relative path
+		return path
+	}
+
+	const prefix = `\\?`
+
+	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
+	copy(pathbuf, prefix)
+	n := len(path)
+	r, w := 0, len(prefix)
+	for r < n {
+		switch {
+		case os.IsPathSeparator(path[r]):
+			// empty block
+			r++
+		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
+			// /./
+			r++
+		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
+			// /../ is currently unhandled
+			return path
+		default:
+			pathbuf[w] = '\\'
+			w++
+			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
+				pathbuf[w] = path[r]
+				w++
+			}
+		}
+	}
+	// A drive's root directory needs a trailing \
+	if w == len(`\\?\c:`) {
+		pathbuf[w] = '\\'
+		w++
+	}
+	return string(pathbuf[:w])
+}

lib/fs/fakefs.go

@@ -48,14 +48,17 @@ const randomBlockShift = 14 // 128k
 //     sizeavg=n  to set the average size of random files, in bytes (default 1<<20)
 //     seed=n     to set the initial random seed (default 0)
 //     insens=b   "true" makes filesystem case-insensitive Windows- or OSX-style (default false)
+//     latency=d  to set the amount of time each "disk" operation takes, where d is time.ParseDuration format
 //
 // - Two fakefs:s pointing at the same root path see the same files.
 //
 type fakefs struct {
+	uri         string
 	mut         sync.Mutex
 	root        *fakeEntry
 	insens      bool
 	withContent bool
+	latency     time.Duration
 }
 
 var (
@@ -63,23 +66,25 @@ var (
 	fakefsFs  = make(map[string]*fakefs)
 )
 
-func newFakeFilesystem(root string) *fakefs {
+func newFakeFilesystem(rootURI string) *fakefs {
 	fakefsMut.Lock()
 	defer fakefsMut.Unlock()
 
+	root := rootURI
 	var params url.Values
-	uri, err := url.Parse(root)
+	uri, err := url.Parse(rootURI)
 	if err == nil {
 		root = uri.Path
 		params = uri.Query()
 	}
 
-	if fs, ok := fakefsFs[root]; ok {
+	if fs, ok := fakefsFs[rootURI]; ok {
 		// Already have an fs at this path
 		return fs
 	}
 
 	fs := &fakefs{
+		uri: "fake://" + rootURI,
 		root: &fakeEntry{
 			name:      "/",
 			entryType: fakeEntryTypeDir,
@@ -129,6 +134,10 @@ func newFakeFilesystem(root string) *fakefs {
 	// Also create a default folder marker for good measure
 	fs.Mkdir(".stfolder", 0700)
 
+	// We only set the latency after doing the operations required to create
+	// the filesystem initially.
+	fs.latency, _ = time.ParseDuration(params.Get("latency"))
+
 	fakefsFs[root] = fs
 	return fs
 }
@@ -185,6 +194,7 @@ func (fs *fakefs) entryForName(name string) *fakeEntry {
 func (fs *fakefs) Chmod(name string, mode FileMode) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 	entry := fs.entryForName(name)
 	if entry == nil {
 		return os.ErrNotExist
@@ -196,6 +206,7 @@ func (fs *fakefs) Chmod(name string, mode FileMode) error {
 func (fs *fakefs) Lchown(name string, uid, gid int) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 	entry := fs.entryForName(name)
 	if entry == nil {
 		return os.ErrNotExist
@@ -208,6 +219,7 @@ func (fs *fakefs) Lchown(name string, uid, gid int) error {
 func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 	entry := fs.entryForName(name)
 	if entry == nil {
 		return os.ErrNotExist
@@ -219,6 +231,7 @@ func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
 func (fs *fakefs) create(name string) (*fakeEntry, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	if entry := fs.entryForName(name); entry != nil {
 		if entry.entryType == fakeEntryTypeDir {
@@ -284,6 +297,7 @@ func (fs *fakefs) CreateSymlink(target, name string) error {
 func (fs *fakefs) DirNames(name string) ([]string, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil {
@@ -301,6 +315,7 @@ func (fs *fakefs) DirNames(name string) ([]string, error) {
 func (fs *fakefs) Lstat(name string) (FileInfo, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil {
@@ -318,6 +333,7 @@ func (fs *fakefs) Lstat(name string) (FileInfo, error) {
 func (fs *fakefs) Mkdir(name string, perm FileMode) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	dir := filepath.Dir(name)
 	base := filepath.Base(name)
@@ -348,6 +364,10 @@ func (fs *fakefs) Mkdir(name string, perm FileMode) error {
 }
 
 func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
+	fs.mut.Lock()
+	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
+
 	name = filepath.ToSlash(name)
 	name = strings.Trim(name, "/")
 	comps := strings.Split(name, "/")
@@ -382,6 +402,7 @@ func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
 func (fs *fakefs) Open(name string) (File, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil || entry.entryType != fakeEntryTypeFile {
@@ -401,6 +422,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
 
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	dir := filepath.Dir(name)
 	base := filepath.Base(name)
@@ -438,6 +460,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
 func (fs *fakefs) ReadSymlink(name string) (string, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil {
@@ -451,6 +474,7 @@ func (fs *fakefs) ReadSymlink(name string) (string, error) {
 func (fs *fakefs) Remove(name string) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	if fs.insens {
 		name = UnicodeLowercase(name)
@@ -472,6 +496,7 @@ func (fs *fakefs) Remove(name string) error {
 func (fs *fakefs) RemoveAll(name string) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	if fs.insens {
 		name = UnicodeLowercase(name)
@@ -491,6 +516,7 @@ func (fs *fakefs) RemoveAll(name string) error {
 func (fs *fakefs) Rename(oldname, newname string) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	oldKey := filepath.Base(oldname)
 	newKey := filepath.Base(newname)
@@ -578,7 +604,7 @@ func (fs *fakefs) Type() FilesystemType {
 }
 
 func (fs *fakefs) URI() string {
-	return "fake://" + fs.root.name
+	return fs.uri
 }
 
 func (fs *fakefs) SameFile(fi1, fi2 FileInfo) bool {