chore(sqlite): use normalised tables for file names and versions (#10383)
This changes the files table to use normalisation for the names and
versions. The idea is that these are often common between all remote
devices, and repeating an integer is more efficient than repeating a
long string. A new benchmark bears this out; for a database with 100k
files shared between 31 devices, with some worst case assumption on
version vector size, the database is reduced in size by 50% and the test
finishes quicker:
Current:
db_bench_test.go:322: Total size: 6263.70 MiB
--- PASS: TestBenchmarkSizeManyFilesRemotes (1084.89s)
New:
db_bench_test.go:326: Total size: 3049.95 MiB
--- PASS: TestBenchmarkSizeManyFilesRemotes (776.97s)
The other benchmarks end up about the same within the margin of
variability, with one possible exception being that RemoteNeed seems to
be a little slower on average:
old files/s new files/s
Update/n=RemoteNeed/size=1000-8 5.051k 4.654k
Update/n=RemoteNeed/size=2000-8 5.201k 4.384k
Update/n=RemoteNeed/size=4000-8 4.943k 4.242k
Update/n=RemoteNeed/size=8000-8 5.099k 3.527k
Update/n=RemoteNeed/size=16000-8 3.686k 3.847k
Update/n=RemoteNeed/size=30000-8 4.456k 3.482k
I'm not sure why, possibly that query can be optimised anyhow.
Signed-off-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
@@ -125,6 +125,9 @@ func (s *Service) periodic(ctx context.Context) error {
|
||||
if err := garbageCollectOldDeletedLocked(ctx, fdb); err != nil {
|
||||
return wrap(err)
|
||||
}
|
||||
if err := garbageCollectNamesAndVersions(ctx, fdb); err != nil {
|
||||
return wrap(err)
|
||||
}
|
||||
if err := garbageCollectBlocklistsAndBlocksLocked(ctx, fdb); err != nil {
|
||||
return wrap(err)
|
||||
}
|
||||
@@ -152,6 +155,34 @@ func tidy(ctx context.Context, db *sqlx.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func garbageCollectNamesAndVersions(ctx context.Context, fdb *folderDB) error {
|
||||
l := slog.With("folder", fdb.folderID, "fdb", fdb.baseName)
|
||||
|
||||
res, err := fdb.stmt(`
|
||||
DELETE FROM file_names
|
||||
WHERE NOT EXISTS (SELECT 1 FROM files f WHERE f.name_idx = idx)
|
||||
`).Exec()
|
||||
if err != nil {
|
||||
return wrap(err, "delete names")
|
||||
}
|
||||
if aff, err := res.RowsAffected(); err == nil {
|
||||
l.DebugContext(ctx, "Removed old file names", "affected", aff)
|
||||
}
|
||||
|
||||
res, err = fdb.stmt(`
|
||||
DELETE FROM file_versions
|
||||
WHERE NOT EXISTS (SELECT 1 FROM files f WHERE f.version_idx = idx)
|
||||
`).Exec()
|
||||
if err != nil {
|
||||
return wrap(err, "delete versions")
|
||||
}
|
||||
if aff, err := res.RowsAffected(); err == nil {
|
||||
l.DebugContext(ctx, "Removed old file versions", "affected", aff)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func garbageCollectOldDeletedLocked(ctx context.Context, fdb *folderDB) error {
|
||||
l := slog.With("folder", fdb.folderID, "fdb", fdb.baseName)
|
||||
if fdb.deleteRetention <= 0 {
|
||||
|
||||
Reference in New Issue
Block a user