felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/model, lib/versioner: Drive version cleanup from scanner (fixes #6313) (#6693)

Browse Source 
This change adds a separate config for the cleanup interval, and runs that cleanup from the main folder loop.
This commit is contained in:
Jakob Borg
2020-07-14 10:48:50 +02:00
committed by GitHub
parent 245b4b98c4
commit aedc2d788f
15 changed files with 178 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/versioner/external.go
+5
View File
@@ -7,6 +7,7 @@
package versioner
import (
"context"
"errors"
"os"
"os/exec"
@@ -115,3 +116,7 @@ func (v external) GetVersions() (map[string][]FileVersion, error) {
func (v external) Restore(filePath string, versionTime time.Time) error {
return ErrRestorationNotSupported
}
func (v external) Clean(_ context.Context) error {
return nil
}
lib/versioner/simple.go
+5
View File
@@ -7,6 +7,7 @@
package versioner
import (
"context"
"strconv"
"time"
@@ -73,3 +74,7 @@ func (v simple) GetVersions() (map[string][]FileVersion, error) {
func (v simple) Restore(filepath string, versionTime time.Time) error {
return restoreFile(v.copyRangeMethod, v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
}
func (v simple) Clean(_ context.Context) error {
return nil
}
lib/versioner/staggered.go
+16 -46
View File
@@ -13,12 +13,8 @@ import (
"strconv"
"time"
"github.com/thejerf/suture"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
)
func init() {
@@ -32,15 +28,10 @@ type interval struct {
}
type staggered struct {
suture.Service
cleanInterval int64
folderFs fs.Filesystem
versionsFs fs.Filesystem
interval [4]interval
copyRangeMethod fs.CopyRangeMethod
mutex sync.Mutex
testCleanDone chan struct{}
}
func newStaggered(cfg config.FolderConfiguration) Versioner {
@@ -49,19 +40,14 @@ func newStaggered(cfg config.FolderConfiguration) Versioner {
if err != nil {
maxAge = 31536000 // Default: ~1 year
}
cleanInterval, err := strconv.ParseInt(params["cleanInterval"], 10, 0)
if err != nil {
cleanInterval = 3600 // Default: clean once per hour
}
// Backwards compatibility
params["fsPath"] = params["versionsPath"]
versionsFs := versionerFsFromFolderCfg(cfg)
s := &staggered{
cleanInterval: cleanInterval,
folderFs: cfg.Filesystem(),
versionsFs: versionsFs,
folderFs: cfg.Filesystem(),
versionsFs: versionsFs,
interval: [4]interval{
{30, 60 * 60}, // first hour -> 30 sec between versions
{60 * 60, 24 * 60 * 60}, // next day -> 1 h between versions
@@ -69,41 +55,18 @@ func newStaggered(cfg config.FolderConfiguration) Versioner {
{7 * 24 * 60 * 60, maxAge}, // next year -> 1 week between versions
},
copyRangeMethod: cfg.CopyRangeMethod,
mutex: sync.NewMutex(),
}
s.Service = util.AsService(s.serve, s.String())
l.Debugf("instantiated %#v", s)
return s
}
func (v *staggered) serve(ctx context.Context) {
v.clean()
if v.testCleanDone != nil {
close(v.testCleanDone)
}
tck := time.NewTicker(time.Duration(v.cleanInterval) * time.Second)
defer tck.Stop()
for {
select {
case <-tck.C:
v.clean()
case <-ctx.Done():
return
}
}
}
func (v *staggered) clean() {
l.Debugln("Versioner clean: Waiting for lock on", v.versionsFs)
v.mutex.Lock()
defer v.mutex.Unlock()
func (v *staggered) Clean(ctx context.Context) error {
l.Debugln("Versioner clean: Cleaning", v.versionsFs)
if _, err := v.versionsFs.Stat("."); fs.IsNotExist(err) {
// There is no need to clean a nonexistent dir.
return
return nil
}
versionsPerFile := make(map[string][]string)
@@ -113,6 +76,11 @@ func (v *staggered) clean() {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if f.IsDir() && !f.IsSymlink() {
dirTracker.addDir(path)
@@ -134,16 +102,22 @@ func (v *staggered) clean() {
if err := v.versionsFs.Walk(".", walkFn); err != nil {
l.Warnln("Versioner: error scanning versions dir", err)
return
return err
}
for _, versionList := range versionsPerFile {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
v.expire(versionList)
}
dirTracker.deleteEmptyDirs(v.versionsFs)
l.Debugln("Cleaner: Finished cleaning", v.versionsFs)
return nil
}
func (v *staggered) expire(versions []string) {
@@ -216,10 +190,6 @@ func (v *staggered) toRemove(versions []string, now time.Time) []string {
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v *staggered) Archive(filePath string) error {
l.Debugln("Waiting for lock on ", v.versionsFs)
v.mutex.Lock()
defer v.mutex.Unlock()
if err := archiveFile(v.copyRangeMethod, v.folderFs, v.versionsFs, filePath, TagFilename); err != nil {
return err
}
lib/versioner/trashcan.go
+11 -32
View File
@@ -12,11 +12,8 @@ import (
"strconv"
"time"
"github.com/thejerf/suture"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/util"
)
func init() {
@@ -25,7 +22,6 @@ func init() {
}
type trashcan struct {
suture.Service
folderFs fs.Filesystem
versionsFs fs.Filesystem
cleanoutDays int
@@ -42,7 +38,6 @@ func newTrashcan(cfg config.FolderConfiguration) Versioner {
cleanoutDays: cleanoutDays,
copyRangeMethod: cfg.CopyRangeMethod,
}
s.Service = util.AsService(s.serve, s.String())
l.Debugf("instantiated %#v", s)
return s
@@ -56,37 +51,16 @@ func (t *trashcan) Archive(filePath string) error {
})
}
func (t *trashcan) serve(ctx context.Context) {
l.Debugln(t, "starting")
defer l.Debugln(t, "stopping")
// Do the first cleanup one minute after startup.
timer := time.NewTimer(time.Minute)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return
case <-timer.C:
if t.cleanoutDays > 0 {
if err := t.cleanoutArchive(); err != nil {
l.Infoln("Cleaning trashcan:", err)
}
}
// Cleanups once a day should be enough.
timer.Reset(24 * time.Hour)
}
}
}
func (t *trashcan) String() string {
return fmt.Sprintf("trashcan@%p", t)
}
func (t *trashcan) cleanoutArchive() error {
func (t *trashcan) Clean(ctx context.Context) error {
if t.cleanoutDays <= 0 {
// no cleanout requested
return nil
}
if _, err := t.versionsFs.Lstat("."); fs.IsNotExist(err) {
return nil
}
@@ -98,6 +72,11 @@ func (t *trashcan) cleanoutArchive() error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if info.IsDir() && !info.IsSymlink() {
dirTracker.addDir(path)
lib/versioner/trashcan_test.go
+3 -2
View File
@@ -7,13 +7,14 @@
package versioner
import (
"github.com/syncthing/syncthing/lib/config"
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
)
@@ -65,7 +66,7 @@ func TestTrashcanCleanout(t *testing.T) {
}
versioner := newTrashcan(cfg).(*trashcan)
if err := versioner.cleanoutArchive(); err != nil {
if err := versioner.Clean(context.Background()); err != nil {
t.Fatal(err)
}
lib/versioner/versioner.go
+2
View File
@@ -9,6 +9,7 @@
package versioner
import (
"context"
"errors"
"fmt"
"time"
@@ -20,6 +21,7 @@ type Versioner interface {
Archive(filePath string) error
GetVersions() (map[string][]FileVersion, error)
Restore(filePath string, versionTime time.Time) error
Clean(context.Context) error
}
type FileVersion struct {