fix(cmd): make database migration more robust to write errors (#10278)
Two things: - We could run into a write error, which would block the progress forever without an error. This because the writer routine exited, while the reader was just blocked on sending to it. - After a failed migration, inserts could fail with unique index constraint errors because we are reusing the sequence numbers from the original database. Add a drop folder to the start of migration to handle this. Additionally, the drop folder will clear out broken database files due to killed migrations.
This commit is contained in:
+15
-2
@@ -197,12 +197,20 @@ func TryMigrateDatabase(deleteRetention time.Duration) error {
|
|||||||
var writeErr error
|
var writeErr error
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
|
writerDone := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
defer close(writerDone)
|
||||||
var batch []protocol.FileInfo
|
var batch []protocol.FileInfo
|
||||||
files, blocks := 0, 0
|
files, blocks := 0, 0
|
||||||
t0 := time.Now()
|
t0 := time.Now()
|
||||||
t1 := time.Now()
|
t1 := time.Now()
|
||||||
|
|
||||||
|
if writeErr = sdb.DropFolder(folder); writeErr != nil {
|
||||||
|
slog.Error("Failed database drop", slogutil.Error(writeErr))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
for fi := range fis {
|
for fi := range fis {
|
||||||
batch = append(batch, fi)
|
batch = append(batch, fi)
|
||||||
files++
|
files++
|
||||||
@@ -210,6 +218,7 @@ func TryMigrateDatabase(deleteRetention time.Duration) error {
|
|||||||
if len(batch) == 1000 {
|
if len(batch) == 1000 {
|
||||||
writeErr = sdb.Update(folder, protocol.LocalDeviceID, batch)
|
writeErr = sdb.Update(folder, protocol.LocalDeviceID, batch)
|
||||||
if writeErr != nil {
|
if writeErr != nil {
|
||||||
|
slog.Error("Failed database write", slogutil.Error(writeErr))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
batch = batch[:0]
|
batch = batch[:0]
|
||||||
@@ -244,8 +253,12 @@ func TryMigrateDatabase(deleteRetention time.Duration) error {
|
|||||||
// criteria in the database
|
// criteria in the database
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
fis <- fi
|
select {
|
||||||
return true
|
case fis <- fi:
|
||||||
|
return true
|
||||||
|
case <-writerDone:
|
||||||
|
return false
|
||||||
|
}
|
||||||
})
|
})
|
||||||
close(fis)
|
close(fis)
|
||||||
snap.Release()
|
snap.Release()
|
||||||
|
|||||||
Reference in New Issue
Block a user