felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: switch database engine to sqlite (fixes #9954) (#9965)

Browse Source 
Switch the database from LevelDB to SQLite, for greater stability and
simpler code.

Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl>
Co-authored-by: bt90 <btom1990@googlemail.com>
This commit is contained in:
Jakob Borg
2025-03-29 13:50:08 +01:00
committed by GitHub
co-authored by Tommy van der Vorst bt90
parent b1c8f88a44
commit 025905fcdf
146 changed files with 8315 additions and 11984 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/fs/filesystem_test.go
+2 -2
View File
@@ -189,7 +189,7 @@ func TestRepro9677MissingMtimeFS(t *testing.T) {
testTime := time.Unix(1723491493, 123456789)
// Create a file with an mtime FS entry
firstFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB))
firstFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB, ""))
// Create a file, set its mtime and check that we get the expected mtime when stat-ing.
file, err := firstFS.Create(name)
@@ -231,6 +231,6 @@ func TestRepro9677MissingMtimeFS(t *testing.T) {
// be without mtime, even if requested:
NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{})
newFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB))
newFS := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("%v?insens=true&timeprecisionsecond=true", t.Name()), &OptionDetectCaseConflicts{}, NewMtimeOption(mtimeDB, ""))
checkMtime(newFS)
}
lib/fs/mtimefs.go
+30 -76
View File
@@ -7,21 +7,21 @@
package fs
import (
"errors"
"time"
)
// The database is where we store the virtual mtimes
type database interface {
Bytes(key string) (data []byte, ok bool, err error)
PutBytes(key string, data []byte) error
Delete(key string) error
GetMtime(folder, name string) (ondisk, virtual time.Time)
PutMtime(folder, name string, ondisk, virtual time.Time) error
DeleteMtime(folder, name string) error
}
type mtimeFS struct {
Filesystem
chtimes func(string, time.Time, time.Time) error
db database
folderID string
caseInsensitive bool
}
@@ -34,16 +34,18 @@ func WithCaseInsensitivity(v bool) MtimeFSOption {
}
type optionMtime struct {
db database
options []MtimeFSOption
db database
folderID string
options []MtimeFSOption
}
// NewMtimeOption makes any filesystem provide nanosecond mtime precision,
// regardless of what shenanigans the underlying filesystem gets up to.
func NewMtimeOption(db database, options ...MtimeFSOption) Option {
func NewMtimeOption(db database, folderID string, options ...MtimeFSOption) Option {
return &optionMtime{
db: db,
options: options,
db: db,
folderID: folderID,
options: options,
}
}
@@ -52,6 +54,7 @@ func (o *optionMtime) apply(fs Filesystem) Filesystem {
Filesystem: fs,
chtimes: fs.Chtimes, // for mocking it out in the tests
db: o.db,
folderID: o.folderID,
}
for _, opt := range o.options {
opt(f)
@@ -84,14 +87,11 @@ func (f *mtimeFS) Stat(name string) (FileInfo, error) {
return nil, err
}
mtimeMapping, err := f.load(name)
if err != nil {
return nil, err
}
if mtimeMapping.Real.Equal(info.ModTime()) {
ondisk, virtual := f.load(name)
if ondisk.Equal(info.ModTime()) {
info = mtimeFileInfo{
FileInfo: info,
mtime: mtimeMapping.Virtual,
mtime: virtual,
}
}
@@ -104,14 +104,11 @@ func (f *mtimeFS) Lstat(name string) (FileInfo, error) {
return nil, err
}
mtimeMapping, err := f.load(name)
if err != nil {
return nil, err
}
if mtimeMapping.Real.Equal(info.ModTime()) {
ondisk, virtual := f.load(name)
if ondisk.Equal(info.ModTime()) {
info = mtimeFileInfo{
FileInfo: info,
mtime: mtimeMapping.Virtual,
mtime: virtual,
}
}
@@ -150,43 +147,27 @@ func (*mtimeFS) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeMtime
}
func (f *mtimeFS) save(name string, real, virtual time.Time) {
func (f *mtimeFS) save(name string, ondisk, virtual time.Time) {
if f.caseInsensitive {
name = UnicodeLowercaseNormalized(name)
}
if real.Equal(virtual) {
if ondisk.Equal(virtual) {
// If the virtual time and the real on disk time are equal we don't
// need to store anything.
f.db.Delete(name)
_ = f.db.DeleteMtime(f.folderID, name)
return
}
mtime := MtimeMapping{
Real: real,
Virtual: virtual,
}
bs, _ := mtime.Marshal() // Can't fail
f.db.PutBytes(name, bs)
_ = f.db.PutMtime(f.folderID, name, ondisk, virtual)
}
func (f *mtimeFS) load(name string) (MtimeMapping, error) {
func (f *mtimeFS) load(name string) (ondisk, virtual time.Time) {
if f.caseInsensitive {
name = UnicodeLowercaseNormalized(name)
}
data, exists, err := f.db.Bytes(name)
if err != nil {
return MtimeMapping{}, err
} else if !exists {
return MtimeMapping{}, nil
}
var mtime MtimeMapping
if err := mtime.Unmarshal(data); err != nil {
return MtimeMapping{}, err
}
return mtime, nil
return f.db.GetMtime(f.folderID, name)
}
// The mtimeFileInfo is an os.FileInfo that lies about the ModTime().
@@ -211,14 +192,11 @@ func (f mtimeFile) Stat() (FileInfo, error) {
return nil, err
}
mtimeMapping, err := f.fs.load(f.Name())
if err != nil {
return nil, err
}
if mtimeMapping.Real.Equal(info.ModTime()) {
ondisk, virtual := f.fs.load(f.Name())
if ondisk.Equal(info.ModTime()) {
info = mtimeFileInfo{
FileInfo: info,
mtime: mtimeMapping.Virtual,
mtime: virtual,
}
}
@@ -230,38 +208,14 @@ func (f mtimeFile) unwrap() File {
return f.File
}
// MtimeMapping represents the mapping as stored in the database
type MtimeMapping struct {
// "Real" is the on disk timestamp
Real time.Time `json:"real"`
// "Virtual" is what want the timestamp to be
Virtual time.Time `json:"virtual"`
}
func (t *MtimeMapping) Marshal() ([]byte, error) {
bs0, _ := t.Real.MarshalBinary()
bs1, _ := t.Virtual.MarshalBinary()
return append(bs0, bs1...), nil
}
func (t *MtimeMapping) Unmarshal(bs []byte) error {
if err := t.Real.UnmarshalBinary(bs[:len(bs)/2]); err != nil {
return err
}
if err := t.Virtual.UnmarshalBinary(bs[len(bs)/2:]); err != nil {
return err
}
return nil
}
func GetMtimeMapping(fs Filesystem, file string) (MtimeMapping, error) {
func GetMtimeMapping(fs Filesystem, file string) (ondisk, virtual time.Time) {
fs, ok := unwrapFilesystem(fs, filesystemWrapperTypeMtime)
if !ok {
return MtimeMapping{}, errors.New("failed to unwrap")
return time.Time{}, time.Time{}
}
mtimeFs, ok := fs.(*mtimeFS)
if !ok {
return MtimeMapping{}, errors.New("unwrapping failed")
return time.Time{}, time.Time{}
}
return mtimeFs.load(file)
}
lib/fs/mtimefs_test.go
+9 -9
View File
@@ -226,20 +226,20 @@ func TestMtimeFSInsensitive(t *testing.T) {
// The mapStore is a simple database
type mapStore map[string][]byte
type mapStore map[string][2]time.Time
func (s mapStore) PutBytes(key string, data []byte) error {
s[key] = data
func (s mapStore) PutMtime(_, name string, real, virtual time.Time) error {
s[name] = [2]time.Time{real, virtual}
return nil
}
func (s mapStore) Bytes(key string) (data []byte, ok bool, err error) {
data, ok = s[key]
return
func (s mapStore) GetMtime(_, name string) (real, virtual time.Time) {
v := s[name]
return v[0], v[1]
}
func (s mapStore) Delete(key string) error {
delete(s, key)
func (s mapStore) DeleteMtime(_, name string) error {
delete(s, name)
return nil
}
@@ -260,7 +260,7 @@ func newMtimeFS(path string, db database, options ...MtimeFSOption) *mtimeFS {
}
func newMtimeFSWithWalk(path string, db database, options ...MtimeFSOption) (*mtimeFS, *walkFilesystem) {
fs := NewFilesystem(FilesystemTypeBasic, path, NewMtimeOption(db, options...))
fs := NewFilesystem(FilesystemTypeBasic, path, NewMtimeOption(db, "", options...))
wfs, _ := unwrapFilesystem(fs, filesystemWrapperTypeWalk)
mfs, _ := unwrapFilesystem(fs, filesystemWrapperTypeMtime)
return mfs.(*mtimeFS), wfs.(*walkFilesystem)