felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: configurable delete retention interval (#10030)

Browse Source 
Command line flag, as it also needs to be able to take effect during
migration.
This commit is contained in:
Jakob Borg
2025-04-03 09:55:19 +02:00
committed by GitHub
parent b88aea34b6
commit 8a2d8ebf81
5 changed files with 62 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/syncthing/main.go
+21 -20
View File
@@ -161,24 +161,25 @@ func (c *CLI) AfterApply() error {
type serveCmd struct { type serveCmd struct {
buildSpecificOptions buildSpecificOptions
AllowNewerConfig bool `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"` AllowNewerConfig bool `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"`
Audit bool `help:"Write events to audit file" env:"STAUDIT"` Audit bool `help:"Write events to audit file" env:"STAUDIT"`
AuditFile string `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"` AuditFile string `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"`
DBMaintenanceInterval time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTINTERVAL"` DBMaintenanceInterval time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTENANCEINTERVAL"`
GUIAddress string `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"` DBDeleteRetentionInterval time.Duration `help:"Database deleted item retention interval" default:"4320h" env:"STDBDELETERETENTIONINTERVAL"`
GUIAPIKey string `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"` GUIAddress string `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"`
LogFile string `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"` GUIAPIKey string `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"`
LogFlags int `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"` LogFile string `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"`
LogMaxFiles int `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STNUMLOGFILES"` LogFlags int `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"`
LogMaxSize int `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"` LogMaxFiles int `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STLOGMAXOLDFILES"`
NoBrowser bool `help:"Do not start browser" env:"STNOBROWSER"` LogMaxSize int `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"`
NoDefaultFolder bool `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"` NoBrowser bool `help:"Do not start browser" env:"STNOBROWSER"`
NoPortProbing bool `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"` NoDefaultFolder bool `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"`
NoRestart bool `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"` NoPortProbing bool `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
NoUpgrade bool `help:"Disable automatic upgrades" env:"STNOUPGRADE"` NoRestart bool `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"`
Paused bool `help:"Start with all devices and folders paused" env:"STPAUSED"` NoUpgrade bool `help:"Disable automatic upgrades" env:"STNOUPGRADE"`
Unpaused bool `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"` Paused bool `help:"Start with all devices and folders paused" env:"STPAUSED"`
Verbose bool `help:"Print verbose log output" env:"STVERBOSE"` Unpaused bool `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"`
Verbose bool `help:"Print verbose log output" env:"STVERBOSE"`
// Debug options below // Debug options below
DebugGUIAssetsDir string `help:"Directory to load GUI assets from" placeholder:"PATH" env:"STGUIASSETS"` DebugGUIAssetsDir string `help:"Directory to load GUI assets from" placeholder:"PATH" env:"STGUIASSETS"`
@@ -487,12 +488,12 @@ func (c *serveCmd) syncthingMain() {
}) })
} }
if err := syncthing.TryMigrateDatabase(); err != nil { if err := syncthing.TryMigrateDatabase(c.DBDeleteRetentionInterval); err != nil {
l.Warnln("Failed to migrate old-style database:", err) l.Warnln("Failed to migrate old-style database:", err)
os.Exit(1) os.Exit(1)
} }
sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database)) sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database), c.DBDeleteRetentionInterval)
if err != nil { if err != nil {
l.Warnln("Error opening database:", err) l.Warnln("Error opening database:", err)
os.Exit(1) os.Exit(1)
internal/db/sqlite/db.go
+14 -4
View File
@@ -17,10 +17,12 @@ import (
) )
type DB struct { type DB struct {
sql *sqlx.DB sql *sqlx.DB
localDeviceIdx int64 localDeviceIdx int64
updateLock sync.Mutex deleteRetention time.Duration
updatePoints int
updateLock sync.Mutex
updatePoints int
statementsMut sync.RWMutex statementsMut sync.RWMutex
statements map[string]*sqlx.Stmt statements map[string]*sqlx.Stmt
@@ -29,6 +31,14 @@ type DB struct {
var _ db.DB = (*DB)(nil) var _ db.DB = (*DB)(nil)
type Option func(*DB)
func WithDeleteRetention(d time.Duration) Option {
return func(s *DB) {
s.deleteRetention = d
}
}
func (s *DB) Close() error { func (s *DB) Close() error {
s.updateLock.Lock() s.updateLock.Lock()
s.statementsMut.Lock() s.statementsMut.Lock()
internal/db/sqlite/db_open.go
+12 -5
View File
@@ -21,7 +21,7 @@ import (
const maxDBConns = 128 const maxDBConns = 128
func Open(path string) (*DB, error) { func Open(path string, opts ...Option) (*DB, error) {
// Open the database with options to enable foreign keys and recursive // Open the database with options to enable foreign keys and recursive
// triggers (needed for the delete+insert triggers on row replace). // triggers (needed for the delete+insert triggers on row replace).
sqlDB, err := sqlx.Open(dbDriver, "file:"+path+"?"+commonOptions) sqlDB, err := sqlx.Open(dbDriver, "file:"+path+"?"+commonOptions)
@@ -36,7 +36,7 @@ func Open(path string) (*DB, error) {
// https://www.sqlite.org/pragma.html#pragma_optimize // https://www.sqlite.org/pragma.html#pragma_optimize
return nil, wrap(err, "PRAGMA optimize") return nil, wrap(err, "PRAGMA optimize")
} }
return openCommon(sqlDB) return openCommon(sqlDB, opts...)
} }
// Open the database with options suitable for the migration inserts. This // Open the database with options suitable for the migration inserts. This
@@ -73,7 +73,7 @@ func OpenTemp() (*DB, error) {
return Open(path) return Open(path)
} }
func openCommon(sqlDB *sqlx.DB) (*DB, error) { func openCommon(sqlDB *sqlx.DB, opts ...Option) (*DB, error) {
if _, err := sqlDB.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil { if _, err := sqlDB.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil {
return nil, wrap(err, "PRAGMA auto_vacuum") return nil, wrap(err, "PRAGMA auto_vacuum")
} }
@@ -85,8 +85,15 @@ func openCommon(sqlDB *sqlx.DB) (*DB, error) {
} }
db := &DB{ db := &DB{
sql: sqlDB, sql: sqlDB,
statements: make(map[string]*sqlx.Stmt), deleteRetention: defaultDeleteRetention,
statements: make(map[string]*sqlx.Stmt),
}
for _, opt := range opts {
opt(db)
}
if db.deleteRetention > 0 && db.deleteRetention < minDeleteRetention {
db.deleteRetention = minDeleteRetention
} }
if err := db.runScripts("sql/schema/*"); err != nil { if err := db.runScripts("sql/schema/*"); err != nil {
internal/db/sqlite/db_service.go
+11 -4
View File
@@ -15,9 +15,10 @@ import (
) )
const ( const (
internalMetaPrefix = "dbsvc" internalMetaPrefix = "dbsvc"
lastMaintKey = "lastMaint" lastMaintKey = "lastMaint"
MaxDeletedFileAge = 180 * 24 * time.Hour defaultDeleteRetention = 180 * 24 * time.Hour
minDeleteRetention = 24 * time.Hour
) )
type Service struct { type Service struct {
@@ -101,12 +102,18 @@ func (s *Service) periodic(ctx context.Context) error {
} }
func (s *Service) garbageCollectOldDeletedLocked() error { func (s *Service) garbageCollectOldDeletedLocked() error {
if s.sdb.deleteRetention <= 0 {
l.Debugln("Delete retention is infinite, skipping cleanup")
return nil
}
// Remove deleted files that are marked as not needed (we have processed // Remove deleted files that are marked as not needed (we have processed
// them) and they were deleted more than MaxDeletedFileAge ago. // them) and they were deleted more than MaxDeletedFileAge ago.
l.Debugln("Forgetting deleted files older than", s.sdb.deleteRetention)
res, err := s.sdb.stmt(` res, err := s.sdb.stmt(`
DELETE FROM files DELETE FROM files
WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0 WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0
`).Exec(time.Now().Add(-MaxDeletedFileAge).UnixNano()) `).Exec(time.Now().Add(-s.sdb.deleteRetention).UnixNano())
if err != nil { if err != nil {
return wrap(err) return wrap(err)
} }
lib/syncthing/utils.go
+4 -4
View File
@@ -158,8 +158,8 @@ func copyFile(src, dst string) error {
} }
// Opens a database // Opens a database
func OpenDatabase(path string) (newdb.DB, error) { func OpenDatabase(path string, deleteRetention time.Duration) (newdb.DB, error) {
sql, err := sqlite.Open(path) sql, err := sqlite.Open(path, sqlite.WithDeleteRetention(deleteRetention))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -170,7 +170,7 @@ func OpenDatabase(path string) (newdb.DB, error) {
} }
// Attempts migration of the old (LevelDB-based) database type to the new (SQLite-based) type // Attempts migration of the old (LevelDB-based) database type to the new (SQLite-based) type
func TryMigrateDatabase() error { func TryMigrateDatabase(deleteRetention time.Duration) error {
oldDBDir := locations.Get(locations.LegacyDatabase) oldDBDir := locations.Get(locations.LegacyDatabase)
if _, err := os.Lstat(oldDBDir); err != nil { if _, err := os.Lstat(oldDBDir); err != nil {
// No old database // No old database
@@ -251,7 +251,7 @@ func TryMigrateDatabase() error {
return err return err
} }
_ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool { _ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool {
if fi.Deleted && time.Since(fi.ModTime()) > sqlite.MaxDeletedFileAge { if deleteRetention > 0 && fi.Deleted && time.Since(fi.ModTime()) > deleteRetention {
// Skip deleted files that match the garbage collection // Skip deleted files that match the garbage collection
// criteria in the database // criteria in the database
return true return true