felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

all: Grand test refactor (fixes #8779, fixes #8799)

Browse Source 
This fixes various test issues with Go 1.20.

- Most tests rewritten to use fakefs where possible
- Some tests that were already skipped, or dubious (invasive,
  unmaintainable, unclear what they even tested) have been removed
- Some actual code rewritten to better support testing in fakefs

Co-authored-by: Eric P <eric@kastelo.net>
This commit is contained in:
Jakob Borg
2023-05-09 10:01:57 +00:00
co-authored by Eric P
parent ddce692f72
commit 1103a27337
54 changed files with 1133 additions and 1594 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/scanner/.gitignore
-1
View File
@@ -1 +0,0 @@
_random.data
lib/scanner/testdata/.stignore
Vendored
-5
View File
@@ -1,5 +0,0 @@
#include excludes
bfile
dir1/cfile
/dir2/dir21
lib/scanner/testdata/afile
Vendored
-1
View File
@@ -1 +0,0 @@
foo
lib/scanner/testdata/bfile
Vendored
-1
View File
@@ -1 +0,0 @@
bar
lib/scanner/testdata/dir1/cfile
Vendored
-1
View File
@@ -1 +0,0 @@
baz
lib/scanner/testdata/dir1/dfile
Vendored
-1
View File
@@ -1 +0,0 @@
quux
lib/scanner/testdata/dir2/cfile
Vendored
-1
View File
@@ -1 +0,0 @@
baz
lib/scanner/testdata/dir2/dfile
Vendored
-1
View File
@@ -1 +0,0 @@
quux
lib/scanner/testdata/dir2/dir21/dir22/dir23/efile
Vendored
View File
lib/scanner/testdata/dir2/dir21/dir22/efile/efile
Vendored
View File
lib/scanner/testdata/dir2/dir21/dira/efile
Vendored
View File
lib/scanner/testdata/dir2/dir21/dira/ffile
Vendored
View File
lib/scanner/testdata/dir2/dir21/efile/ign/efile
Vendored
View File
lib/scanner/testdata/dir3/cfile
Vendored
-1
View File
@@ -1 +0,0 @@
baz
lib/scanner/testdata/dir3/dfile
Vendored
-1
View File
@@ -1 +0,0 @@
quux
lib/scanner/testdata/excludes
Vendored
-2
View File
@@ -1,2 +0,0 @@
dir2/dfile
#include further-excludes
lib/scanner/testdata/further-excludes
Vendored
-1
View File
@@ -1 +0,0 @@
dir3
lib/scanner/walk_test.go
+77 -126
View File
@@ -9,7 +9,6 @@ package scanner
import (
"bytes"
"context"
"crypto/rand"
"errors"
"fmt"
"io"
@@ -26,6 +25,7 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/text/unicode/norm"
)
@@ -38,34 +38,58 @@ type testfile struct {
type testfileList []testfile
const (
testFsType = fs.FilesystemTypeBasic
testFsLocation = "testdata"
)
var (
testFs fs.Filesystem
testdata = testfileList{
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
{"dir1", 128, ""},
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
{"dir2", 128, ""},
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
}
)
var testdata = testfileList{
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
{"dir1", 128, ""},
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
{"dir2", 128, ""},
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
}
func init() {
// This test runs the risk of entering infinite recursion if it fails.
// Limit the stack size to 10 megs to crash early in that case instead of
// potentially taking down the box...
rdebug.SetMaxStack(10 * 1 << 20)
}
testFs = fs.NewFilesystem(testFsType, testFsLocation)
func newTestFs(opts ...fs.Option) fs.Filesystem {
// This mirrors some test data we used to have in a physical `testdata`
// directory here.
tfs := fs.NewFilesystem(fs.FilesystemTypeFake, rand.String(16)+"?content=true&nostfolder=true", opts...)
tfs.Mkdir("dir1", 0o755)
tfs.Mkdir("dir2", 0o755)
tfs.Mkdir("dir3", 0o755)
tfs.MkdirAll("dir2/dir21/dir22/dir23", 0o755)
tfs.MkdirAll("dir2/dir21/dir22/efile", 0o755)
tfs.MkdirAll("dir2/dir21/dira", 0o755)
tfs.MkdirAll("dir2/dir21/efile/ign", 0o755)
fs.WriteFile(tfs, "dir1/cfile", []byte("baz\n"), 0o644)
fs.WriteFile(tfs, "dir1/dfile", []byte("quux\n"), 0o644)
fs.WriteFile(tfs, "dir2/cfile", []byte("baz\n"), 0o644)
fs.WriteFile(tfs, "dir2/dfile", []byte("quux\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/dir22/dir23/efile", []byte("\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/dir22/efile/efile", []byte("\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/dir22/efile/ign/efile", []byte("\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/dira/efile", []byte("\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/dira/ffile", []byte("\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/efile/ign/efile", []byte("\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/cfile", []byte("foo\n"), 0o644)
fs.WriteFile(tfs, "dir2/dir21/dfile", []byte("quux\n"), 0o644)
fs.WriteFile(tfs, "dir3/cfile", []byte("foo\n"), 0o644)
fs.WriteFile(tfs, "dir3/dfile", []byte("quux\n"), 0o644)
fs.WriteFile(tfs, "afile", []byte("foo\n"), 0o644)
fs.WriteFile(tfs, "bfile", []byte("bar\n"), 0o644)
fs.WriteFile(tfs, ".stignore", []byte("#include excludes\n\nbfile\ndir1/cfile\n/dir2/dir21\n"), 0o644)
fs.WriteFile(tfs, "excludes", []byte("dir2/dfile\n#include further-excludes\n"), 0o644)
fs.WriteFile(tfs, "further-excludes", []byte("dir3\n"), 0o644)
return tfs
}
func TestWalkSub(t *testing.T) {
testFs := newTestFs()
ignores := ignore.New(testFs)
err := ignores.Load(".stignore")
if err != nil {
@@ -100,6 +124,7 @@ func TestWalkSub(t *testing.T) {
}
func TestWalk(t *testing.T) {
testFs := newTestFs()
ignores := ignore.New(testFs)
err := ignores.Load(".stignore")
if err != nil {
@@ -124,6 +149,7 @@ func TestWalk(t *testing.T) {
if diff, equal := messagediff.PrettyDiff(testdata, files); !equal {
t.Errorf("Walk returned unexpected data. Diff:\n%s", diff)
t.Error(testdata[4], files[4])
}
}
@@ -183,8 +209,7 @@ func TestNormalization(t *testing.T) {
return
}
os.RemoveAll("testdata/normalization")
defer os.RemoveAll("testdata/normalization")
testFs := newTestFs()
tests := []string{
"0-A", // ASCII A -- accepted
@@ -197,18 +222,11 @@ func TestNormalization(t *testing.T) {
}
numInvalid := 2
if build.IsWindows {
// On Windows, in case 5 the character gets replaced with a
// replacement character \xEF\xBF\xBD at the point it's written to disk,
// which means it suddenly becomes valid (sort of).
numInvalid--
}
numValid := len(tests) - numInvalid
for _, s1 := range tests {
// Create a directory for each of the interesting strings above
if err := testFs.MkdirAll(filepath.Join("normalization", s1), 0755); err != nil {
if err := testFs.MkdirAll(filepath.Join("normalization", s1), 0o755); err != nil {
t.Fatal(err)
}
@@ -217,7 +235,7 @@ func TestNormalization(t *testing.T) {
// file names. Ensure that the file doesn't exist when it's
// created. This detects and fails if there's file name
// normalization stuff at the filesystem level.
if fd, err := testFs.OpenFile(filepath.Join("normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
if fd, err := testFs.OpenFile(filepath.Join("normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0o644); err != nil {
t.Fatal(err)
} else {
fd.Write([]byte("test"))
@@ -241,7 +259,7 @@ func TestNormalization(t *testing.T) {
expectedNum := numValid*numValid + numValid + 1
if len(files) != expectedNum {
t.Errorf("Expected %d files, got %d", expectedNum, len(files))
t.Errorf("Expected %d files, got %d, numvalid %d", expectedNum, len(files), numValid)
}
// The file names should all be in NFC form.
@@ -262,11 +280,11 @@ func TestNormalizationDarwinCaseFS(t *testing.T) {
return
}
testFs := fs.NewFilesystem(testFsType, testFsLocation, new(fs.OptionDetectCaseConflicts))
testFs := newTestFs(new(fs.OptionDetectCaseConflicts))
testFs.RemoveAll("normalization")
defer testFs.RemoveAll("normalization")
testFs.MkdirAll("normalization", 0755)
testFs.MkdirAll("normalization", 0o755)
const (
inNFC = "\xC3\x84"
@@ -274,7 +292,7 @@ func TestNormalizationDarwinCaseFS(t *testing.T) {
)
// Create dir in NFC
if err := testFs.Mkdir(filepath.Join("normalization", "dir-"+inNFC), 0755); err != nil {
if err := testFs.Mkdir(filepath.Join("normalization", "dir-"+inNFC), 0o755); err != nil {
t.Fatal(err)
}
@@ -328,11 +346,11 @@ func TestWalkSymlinkUnix(t *testing.T) {
// Create a folder with a symlink in it
os.RemoveAll("_symlinks")
os.Mkdir("_symlinks", 0755)
os.Mkdir("_symlinks", 0o755)
defer os.RemoveAll("_symlinks")
os.Symlink("../testdata", "_symlinks/link")
fs := fs.NewFilesystem(testFsType, "_symlinks")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil, 0)
@@ -350,79 +368,6 @@ func TestWalkSymlinkUnix(t *testing.T) {
}
}
func TestWalkSymlinkWindows(t *testing.T) {
if !build.IsWindows {
t.Skip("skipping unsupported symlink test")
}
// Create a folder with a symlink in it
name := "_symlinks-win"
os.RemoveAll(name)
os.Mkdir(name, 0755)
defer os.RemoveAll(name)
testFs := fs.NewFilesystem(testFsType, name)
if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, "../testdata", "link"); err != nil {
// Probably we require permissions we don't have.
t.Skip(err)
}
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(testFs, path, nil, nil, 0)
// Verify that we got zero symlinks
if len(files) != 0 {
t.Errorf("expected zero symlinks, not %d", len(files))
}
}
}
func TestWalkRootSymlink(t *testing.T) {
// Create a folder with a symlink in it
tmp := t.TempDir()
testFs := fs.NewFilesystem(testFsType, tmp)
link := "link"
dest, _ := filepath.Abs("testdata/dir1")
destFs := fs.NewFilesystem(testFsType, dest)
if err := fs.DebugSymlinkForTestsOnly(destFs, testFs, ".", "link"); err != nil {
if build.IsWindows {
// Probably we require permissions we don't have.
t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
} else {
t.Fatal(err)
}
}
// Scan root with symlink at FS root
files := walkDir(fs.NewFilesystem(testFsType, filepath.Join(testFs.URI(), link)), ".", nil, nil, 0)
// Verify that we got two files
if len(files) != 2 {
t.Fatalf("expected two files, not %d", len(files))
}
// Scan symlink below FS root
files = walkDir(testFs, "link", nil, nil, 0)
// Verify that we got the one symlink, except on windows
if build.IsWindows {
if len(files) != 0 {
t.Errorf("expected no files, not %d", len(files))
}
} else if len(files) != 1 {
t.Errorf("expected one file, not %d", len(files))
}
// Scan path below symlink
files = walkDir(fs.NewFilesystem(testFsType, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
// Verify that we get nothing
if len(files) != 0 {
t.Errorf("expected no files, not %d", len(files))
}
}
func TestBlocksizeHysteresis(t *testing.T) {
// Verify that we select the right block size in the presence of old
// file information.
@@ -552,7 +497,7 @@ func TestScanOwnershipPOSIX(t *testing.T) {
fakeFS.Create("root-owned")
fakeFS.Create("user-owned")
fakeFS.Lchown("user-owned", "1234", "5678")
fakeFS.Mkdir("user-owned-dir", 0755)
fakeFS.Mkdir("user-owned-dir", 0o755)
fakeFS.Lchown("user-owned-dir", "2345", "6789")
expected := []struct {
@@ -682,14 +627,15 @@ var initOnce sync.Once
const (
testdataSize = 17<<20 + 1
testdataName = "_random.data"
testFsPath = "some_random_dir_path"
)
func BenchmarkHashFile(b *testing.B) {
initOnce.Do(initTestFile)
testFs := newDataFs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(testFsType, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), testFs, testdataName, protocol.MinBlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@@ -698,8 +644,9 @@ func BenchmarkHashFile(b *testing.B) {
b.ReportAllocs()
}
func initTestFile() {
fd, err := os.Create(testdataName)
func newDataFs() fs.Filesystem {
tfs := fs.NewFilesystem(fs.FilesystemTypeFake, rand.String(16)+"?content=true")
fd, err := tfs.Create(testdataName)
if err != nil {
panic(err)
}
@@ -712,6 +659,8 @@ func initTestFile() {
if err := fd.Close(); err != nil {
panic(err)
}
return tfs
}
func TestStopWalk(t *testing.T) {
@@ -782,9 +731,7 @@ func TestStopWalk(t *testing.T) {
}
func TestIssue4799(t *testing.T) {
tmp := t.TempDir()
fs := fs.NewFilesystem(testFsType, tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeFake, rand.String(16))
fd, err := fs.Create("foo")
if err != nil {
@@ -805,6 +752,7 @@ func TestRecurseInclude(t *testing.T) {
!ffile
*
`
testFs := newTestFs()
ignores := ignore.New(testFs, ignore.WithCache(true))
if err := ignores.Parse(bytes.NewBufferString(stignore), ".stignore"); err != nil {
t.Fatal(err)
@@ -830,7 +778,11 @@ func TestRecurseInclude(t *testing.T) {
filepath.Join("dir2", "dir21", "efile", "ign", "efile"),
}
if len(files) != len(expected) {
t.Fatalf("Got %d files %v, expected %d files at %v", len(files), files, len(expected), expected)
var filesString []string
for _, file := range files {
filesString = append(filesString, file.Name)
}
t.Fatalf("Got %d files %v, expected %d files at %v", len(files), filesString, len(expected), expected)
}
for i := range files {
if files[i].Name != expected[i] {
@@ -840,9 +792,7 @@ func TestRecurseInclude(t *testing.T) {
}
func TestIssue4841(t *testing.T) {
tmp := t.TempDir()
fs := fs.NewFilesystem(testFsType, tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeFake, rand.String(16))
fd, err := fs.Create("foo")
if err != nil {
@@ -883,6 +833,7 @@ func TestIssue4841(t *testing.T) {
// TestNotExistingError reproduces https://github.com/syncthing/syncthing/issues/5385
func TestNotExistingError(t *testing.T) {
sub := "notExisting"
testFs := newTestFs()
if _, err := testFs.Lstat(sub); !fs.IsNotExist(err) {
t.Fatalf("Lstat returned error %v, while nothing should exist there.", err)
}
@@ -900,7 +851,7 @@ func TestSkipIgnoredDirs(t *testing.T) {
fss := fs.NewFilesystem(fs.FilesystemTypeFake, "")
name := "foo/ignored"
err := fss.MkdirAll(name, 0777)
err := fss.MkdirAll(name, 0o777)
if err != nil {
t.Fatal(err)
}
@@ -940,7 +891,7 @@ func TestIncludedSubdir(t *testing.T) {
fss := fs.NewFilesystem(fs.FilesystemTypeFake, "")
name := filepath.Clean("foo/bar/included")
err := fss.MkdirAll(name, 0777)
err := fss.MkdirAll(name, 0o777)
if err != nil {
t.Fatal(err)
}
@@ -1023,17 +974,17 @@ func testConfig() (Config, context.CancelFunc) {
ctx, cancel := context.WithCancel(context.Background())
go evLogger.Serve(ctx)
return Config{
Filesystem: testFs,
Filesystem: newTestFs(),
Hashers: 2,
EventLogger: evLogger,
}, cancel
}
func BenchmarkWalk(b *testing.B) {
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, b.TempDir())
testFs := fs.NewFilesystem(fs.FilesystemTypeFake, rand.String(32))
for i := 0; i < 100; i++ {
if err := testFs.Mkdir(fmt.Sprintf("dir%d", i), 0755); err != nil {
if err := testFs.Mkdir(fmt.Sprintf("dir%d", i), 0o755); err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {