Switch the database from LevelDB to SQLite, for greater stability and simpler code. Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl> Co-authored-by: bt90 <btom1990@googlemail.com>
This commit is contained in:
Jakob Borg
co-authored by
Tommy van der Vorst
bt90
Tommy van der Vorst
bt90
parent
b1c8f88a44
commit
025905fcdf
@@ -41,5 +41,4 @@ func (p *profileCommand) Run(ctx Context) error {
|
||||
type debugCommand struct {
|
||||
File fileCommand `cmd:"" help:"Show information about a file (or directory/symlink)"`
|
||||
Profile profileCommand `cmd:"" help:"Save a profile to help figuring out what Syncthing does"`
|
||||
Index indexCommand `cmd:"" help:"Show information about the index (database)"`
|
||||
}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
// Copyright (C) 2014 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/alecthomas/kong"
|
||||
)
|
||||
|
||||
type indexCommand struct {
|
||||
Dump struct{} `cmd:"" help:"Print the entire db"`
|
||||
DumpSize struct{} `cmd:"" help:"Print the db size of different categories of information"`
|
||||
Check struct{} `cmd:"" help:"Check the database for inconsistencies"`
|
||||
Account struct{} `cmd:"" help:"Print key and value size statistics per key type"`
|
||||
}
|
||||
|
||||
func (*indexCommand) Run(kongCtx *kong.Context) error {
|
||||
switch kongCtx.Selected().Name {
|
||||
case "dump":
|
||||
return indexDump()
|
||||
case "dump-size":
|
||||
return indexDumpSize()
|
||||
case "check":
|
||||
return indexCheck()
|
||||
case "account":
|
||||
return indexAccount()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
// Copyright (C) 2020 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
)
|
||||
|
||||
// indexAccount prints key and data size statistics per class
|
||||
func indexAccount() error {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
it, err := ldb.NewPrefixIterator(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var ksizes [256]int
|
||||
var dsizes [256]int
|
||||
var counts [256]int
|
||||
var max [256]int
|
||||
|
||||
for it.Next() {
|
||||
key := it.Key()
|
||||
t := key[0]
|
||||
ds := len(it.Value())
|
||||
ks := len(key)
|
||||
s := ks + ds
|
||||
|
||||
counts[t]++
|
||||
ksizes[t] += ks
|
||||
dsizes[t] += ds
|
||||
if s > max[t] {
|
||||
max[t] = s
|
||||
}
|
||||
}
|
||||
|
||||
tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
|
||||
toti, totds, totks := 0, 0, 0
|
||||
for t := range ksizes {
|
||||
if ksizes[t] > 0 {
|
||||
// yes metric kilobytes 🤘
|
||||
fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
|
||||
toti += counts[t]
|
||||
totds += dsizes[t]
|
||||
totks += ksizes[t]
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
|
||||
tw.Flush()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
// Copyright (C) 2015 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/syncthing/syncthing/internal/gen/bep"
|
||||
"github.com/syncthing/syncthing/internal/gen/dbproto"
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
func indexDump() error {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
it, err := ldb.NewPrefixIterator(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for it.Next() {
|
||||
key := it.Key()
|
||||
switch key[0] {
|
||||
case db.KeyTypeDevice:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
device := binary.BigEndian.Uint32(key[1+4:])
|
||||
name := nulString(key[1+4+4:])
|
||||
fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
|
||||
|
||||
var f bep.FileInfo
|
||||
err := proto.Unmarshal(it.Value(), &f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" V:%v\n", &f)
|
||||
|
||||
case db.KeyTypeGlobal:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
name := nulString(key[1+4:])
|
||||
var flv dbproto.VersionList
|
||||
proto.Unmarshal(it.Value(), &flv)
|
||||
fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, &flv)
|
||||
|
||||
case db.KeyTypeBlock:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
hash := key[1+4 : 1+4+32]
|
||||
name := nulString(key[1+4+32:])
|
||||
fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
|
||||
|
||||
case db.KeyTypeDeviceStatistic:
|
||||
fmt.Printf("[dstat] K:%x V:%x\n", key, it.Value())
|
||||
|
||||
case db.KeyTypeFolderStatistic:
|
||||
fmt.Printf("[fstat] K:%x V:%x\n", key, it.Value())
|
||||
|
||||
case db.KeyTypeVirtualMtime:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
name := nulString(key[1+4:])
|
||||
val := it.Value()
|
||||
var realTime, virtualTime time.Time
|
||||
realTime.UnmarshalBinary(val[:len(val)/2])
|
||||
virtualTime.UnmarshalBinary(val[len(val)/2:])
|
||||
fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, realTime, virtualTime)
|
||||
|
||||
case db.KeyTypeFolderIdx:
|
||||
key := binary.BigEndian.Uint32(key[1:])
|
||||
fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
|
||||
|
||||
case db.KeyTypeDeviceIdx:
|
||||
key := binary.BigEndian.Uint32(key[1:])
|
||||
val := it.Value()
|
||||
device := "<nil>"
|
||||
if len(val) > 0 {
|
||||
dev, err := protocol.DeviceIDFromBytes(val)
|
||||
if err != nil {
|
||||
device = fmt.Sprintf("<invalid %d bytes>", len(val))
|
||||
} else {
|
||||
device = dev.String()
|
||||
}
|
||||
}
|
||||
fmt.Printf("[deviceidx] K:%d V:%s\n", key, device)
|
||||
|
||||
case db.KeyTypeIndexID:
|
||||
device := binary.BigEndian.Uint32(key[1:])
|
||||
folder := binary.BigEndian.Uint32(key[5:])
|
||||
fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
|
||||
|
||||
case db.KeyTypeFolderMeta:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
fmt.Printf("[foldermeta] F:%d", folder)
|
||||
var cs dbproto.CountsSet
|
||||
if err := proto.Unmarshal(it.Value(), &cs); err != nil {
|
||||
fmt.Printf(" (invalid)\n")
|
||||
} else {
|
||||
fmt.Printf(" V:%v\n", &cs)
|
||||
}
|
||||
|
||||
case db.KeyTypeMiscData:
|
||||
fmt.Printf("[miscdata] K:%q V:%q\n", key[1:], it.Value())
|
||||
|
||||
case db.KeyTypeSequence:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
seq := binary.BigEndian.Uint64(key[5:])
|
||||
fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
|
||||
|
||||
case db.KeyTypeNeed:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
file := string(key[5:])
|
||||
fmt.Printf("[need] F:%d V:%q\n", folder, file)
|
||||
|
||||
case db.KeyTypeBlockList:
|
||||
fmt.Printf("[blocklist] H:%x\n", key[1:])
|
||||
|
||||
case db.KeyTypeBlockListMap:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
hash := key[5:37]
|
||||
fileName := string(key[37:])
|
||||
fmt.Printf("[blocklistmap] F:%d H:%x N:%s\n", folder, hash, fileName)
|
||||
|
||||
case db.KeyTypeVersion:
|
||||
fmt.Printf("[version] H:%x", key[1:])
|
||||
var v bep.Vector
|
||||
err := proto.Unmarshal(it.Value(), &v)
|
||||
if err != nil {
|
||||
fmt.Printf(" (invalid)\n")
|
||||
} else {
|
||||
fmt.Printf(" V:%v\n", &v)
|
||||
}
|
||||
|
||||
case db.KeyTypePendingFolder:
|
||||
device := binary.BigEndian.Uint32(key[1:])
|
||||
folder := string(key[5:])
|
||||
var of dbproto.ObservedFolder
|
||||
proto.Unmarshal(it.Value(), &of)
|
||||
fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, &of)
|
||||
|
||||
case db.KeyTypePendingDevice:
|
||||
device := "<invalid>"
|
||||
dev, err := protocol.DeviceIDFromBytes(key[1:])
|
||||
if err == nil {
|
||||
device = dev.String()
|
||||
}
|
||||
var od dbproto.ObservedDevice
|
||||
proto.Unmarshal(it.Value(), &od)
|
||||
fmt.Printf("[pendingDevice] D:%v V:%v\n", device, &od)
|
||||
|
||||
default:
|
||||
fmt.Printf("[??? %d]\n %x\n %x\n", key[0], key, it.Value())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// Copyright (C) 2015 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
)
|
||||
|
||||
func indexDumpSize() error {
|
||||
type sizedElement struct {
|
||||
key string
|
||||
size int
|
||||
}
|
||||
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
it, err := ldb.NewPrefixIterator(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var elems []sizedElement
|
||||
for it.Next() {
|
||||
var ele sizedElement
|
||||
|
||||
key := it.Key()
|
||||
switch key[0] {
|
||||
case db.KeyTypeDevice:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
device := binary.BigEndian.Uint32(key[1+4:])
|
||||
name := nulString(key[1+4+4:])
|
||||
ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
|
||||
|
||||
case db.KeyTypeGlobal:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
name := nulString(key[1+4:])
|
||||
ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
|
||||
|
||||
case db.KeyTypeBlock:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
hash := key[1+4 : 1+4+32]
|
||||
name := nulString(key[1+4+32:])
|
||||
ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
|
||||
|
||||
case db.KeyTypeDeviceStatistic:
|
||||
ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
|
||||
|
||||
case db.KeyTypeFolderStatistic:
|
||||
ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
|
||||
|
||||
case db.KeyTypeVirtualMtime:
|
||||
ele.key = fmt.Sprintf("MTIME:%s", key[1:])
|
||||
|
||||
case db.KeyTypeFolderIdx:
|
||||
id := binary.BigEndian.Uint32(key[1:])
|
||||
ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
|
||||
|
||||
case db.KeyTypeDeviceIdx:
|
||||
id := binary.BigEndian.Uint32(key[1:])
|
||||
ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
|
||||
|
||||
default:
|
||||
ele.key = fmt.Sprintf("UNKNOWN:%x", key)
|
||||
}
|
||||
ele.size = len(it.Value())
|
||||
elems = append(elems, ele)
|
||||
}
|
||||
|
||||
sort.Slice(elems, func(i, j int) bool {
|
||||
return elems[i].size > elems[j].size
|
||||
})
|
||||
for _, ele := range elems {
|
||||
fmt.Println(ele.key, ele.size)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,434 +0,0 @@
|
||||
// Copyright (C) 2018 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/syncthing/syncthing/internal/gen/bep"
|
||||
"github.com/syncthing/syncthing/internal/gen/dbproto"
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
type fileInfoKey struct {
|
||||
folder uint32
|
||||
device uint32
|
||||
name string
|
||||
}
|
||||
|
||||
type globalKey struct {
|
||||
folder uint32
|
||||
name string
|
||||
}
|
||||
|
||||
type sequenceKey struct {
|
||||
folder uint32
|
||||
sequence uint64
|
||||
}
|
||||
|
||||
func indexCheck() (err error) {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
folders := make(map[uint32]string)
|
||||
devices := make(map[uint32]string)
|
||||
deviceToIDs := make(map[string]uint32)
|
||||
fileInfos := make(map[fileInfoKey]*bep.FileInfo)
|
||||
globals := make(map[globalKey]*dbproto.VersionList)
|
||||
sequences := make(map[sequenceKey]string)
|
||||
needs := make(map[globalKey]struct{})
|
||||
blocklists := make(map[string]struct{})
|
||||
versions := make(map[string]*bep.Vector)
|
||||
usedBlocklists := make(map[string]struct{})
|
||||
usedVersions := make(map[string]struct{})
|
||||
var localDeviceKey uint32
|
||||
success := true
|
||||
defer func() {
|
||||
if err == nil {
|
||||
if success {
|
||||
fmt.Println("Index check completed successfully.")
|
||||
} else {
|
||||
err = errors.New("Inconsistencies found in the index")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
it, err := ldb.NewPrefixIterator(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for it.Next() {
|
||||
key := it.Key()
|
||||
switch key[0] {
|
||||
case db.KeyTypeDevice:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
device := binary.BigEndian.Uint32(key[1+4:])
|
||||
name := nulString(key[1+4+4:])
|
||||
|
||||
var f bep.FileInfo
|
||||
err := proto.Unmarshal(it.Value(), &f)
|
||||
if err != nil {
|
||||
fmt.Println("Unable to unmarshal FileInfo:", err)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
|
||||
fileInfos[fileInfoKey{folder, device, name}] = &f
|
||||
|
||||
case db.KeyTypeGlobal:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
name := nulString(key[1+4:])
|
||||
var flv dbproto.VersionList
|
||||
if err := proto.Unmarshal(it.Value(), &flv); err != nil {
|
||||
fmt.Println("Unable to unmarshal VersionList:", err)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
globals[globalKey{folder, name}] = &flv
|
||||
|
||||
case db.KeyTypeFolderIdx:
|
||||
key := binary.BigEndian.Uint32(it.Key()[1:])
|
||||
folders[key] = string(it.Value())
|
||||
|
||||
case db.KeyTypeDeviceIdx:
|
||||
key := binary.BigEndian.Uint32(it.Key()[1:])
|
||||
devices[key] = string(it.Value())
|
||||
deviceToIDs[string(it.Value())] = key
|
||||
if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
|
||||
localDeviceKey = key
|
||||
}
|
||||
|
||||
case db.KeyTypeSequence:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
seq := binary.BigEndian.Uint64(key[5:])
|
||||
val := it.Value()
|
||||
sequences[sequenceKey{folder, seq}] = string(val[9:])
|
||||
|
||||
case db.KeyTypeNeed:
|
||||
folder := binary.BigEndian.Uint32(key[1:])
|
||||
name := nulString(key[1+4:])
|
||||
needs[globalKey{folder, name}] = struct{}{}
|
||||
|
||||
case db.KeyTypeBlockList:
|
||||
hash := string(key[1:])
|
||||
blocklists[hash] = struct{}{}
|
||||
|
||||
case db.KeyTypeVersion:
|
||||
hash := string(key[1:])
|
||||
var v bep.Vector
|
||||
if err := proto.Unmarshal(it.Value(), &v); err != nil {
|
||||
fmt.Println("Unable to unmarshal Vector:", err)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
versions[hash] = &v
|
||||
}
|
||||
}
|
||||
|
||||
if localDeviceKey == 0 {
|
||||
fmt.Println("Missing key for local device in device index (bailing out)")
|
||||
success = false
|
||||
return
|
||||
}
|
||||
|
||||
var missingSeq []sequenceKey
|
||||
for fk, fi := range fileInfos {
|
||||
if fk.name != fi.Name {
|
||||
fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
|
||||
success = false
|
||||
}
|
||||
|
||||
folder := folders[fk.folder]
|
||||
if folder == "" {
|
||||
fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
if devices[fk.device] == "" {
|
||||
fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
|
||||
success = false
|
||||
}
|
||||
|
||||
if fk.device == localDeviceKey {
|
||||
sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
|
||||
name, ok := sequences[sk]
|
||||
if !ok {
|
||||
fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
|
||||
missingSeq = append(missingSeq, sk)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
if name != fi.Name {
|
||||
fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
|
||||
success = false
|
||||
}
|
||||
}
|
||||
|
||||
if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
|
||||
key := string(fi.BlocksHash)
|
||||
if _, ok := blocklists[key]; !ok {
|
||||
fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
|
||||
success = false
|
||||
} else {
|
||||
usedBlocklists[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
if fi.VersionHash != nil {
|
||||
key := string(fi.VersionHash)
|
||||
if _, ok := versions[key]; !ok {
|
||||
fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
|
||||
success = false
|
||||
} else {
|
||||
usedVersions[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
_, ok := globals[globalKey{fk.folder, fk.name}]
|
||||
if !ok {
|
||||
fmt.Printf("Missing global for file %q\n", fi.Name)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Aggregate the ranges of missing sequence entries, print them
|
||||
|
||||
sort.Slice(missingSeq, func(a, b int) bool {
|
||||
if missingSeq[a].folder != missingSeq[b].folder {
|
||||
return missingSeq[a].folder < missingSeq[b].folder
|
||||
}
|
||||
return missingSeq[a].sequence < missingSeq[b].sequence
|
||||
})
|
||||
|
||||
var folder uint32
|
||||
var startSeq, prevSeq uint64
|
||||
for _, sk := range missingSeq {
|
||||
if folder != sk.folder || sk.sequence != prevSeq+1 {
|
||||
if folder != 0 {
|
||||
fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
|
||||
}
|
||||
startSeq = sk.sequence
|
||||
folder = sk.folder
|
||||
}
|
||||
prevSeq = sk.sequence
|
||||
}
|
||||
if folder != 0 {
|
||||
fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
|
||||
}
|
||||
|
||||
for gk, vl := range globals {
|
||||
folder := folders[gk.folder]
|
||||
if folder == "" {
|
||||
fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
|
||||
success = false
|
||||
}
|
||||
checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
|
||||
dev, ok := deviceToIDs[string(device)]
|
||||
if !ok {
|
||||
fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
|
||||
success = false
|
||||
}
|
||||
fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
|
||||
if !ok {
|
||||
fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
|
||||
success = false
|
||||
}
|
||||
|
||||
fiv := fi.Version
|
||||
if fi.VersionHash != nil {
|
||||
fiv = versions[string(fi.VersionHash)]
|
||||
}
|
||||
if !protocol.VectorFromWire(fiv).Equal(version) {
|
||||
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
|
||||
success = false
|
||||
}
|
||||
ffi := protocol.FileInfoFromDB(fi)
|
||||
if ffi.IsInvalid() != invalid {
|
||||
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, ffi.IsInvalid())
|
||||
success = false
|
||||
}
|
||||
if ffi.IsDeleted() != deleted {
|
||||
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, ffi.IsDeleted())
|
||||
success = false
|
||||
}
|
||||
}
|
||||
for i, fv := range vl.Versions {
|
||||
ver := protocol.VectorFromWire(fv.Version)
|
||||
for _, device := range fv.Devices {
|
||||
checkGlobal(i, device, ver, false, fv.Deleted)
|
||||
}
|
||||
for _, device := range fv.InvalidDevices {
|
||||
checkGlobal(i, device, ver, true, fv.Deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// If we need this file we should have a need entry for it. False
|
||||
// positives from needsLocally for deleted files, where we might
|
||||
// legitimately lack an entry if we never had it, and ignored files.
|
||||
if needsLocally(vl) {
|
||||
_, ok := needs[gk]
|
||||
if !ok {
|
||||
fv, _ := vlGetGlobal(vl)
|
||||
devB, _ := fvFirstDevice(fv)
|
||||
dev := deviceToIDs[string(devB)]
|
||||
fi := protocol.FileInfoFromDB(fileInfos[fileInfoKey{gk.folder, dev, gk.name}])
|
||||
if !fi.IsDeleted() && !fi.IsIgnored() {
|
||||
fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
seenSeq := make(map[fileInfoKey]uint64)
|
||||
for sk, name := range sequences {
|
||||
folder := folders[sk.folder]
|
||||
if folder == "" {
|
||||
fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
|
||||
if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
|
||||
fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
|
||||
success = false
|
||||
}
|
||||
seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
|
||||
|
||||
fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
|
||||
if !ok {
|
||||
fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
if fi.Sequence != int64(sk.sequence) {
|
||||
fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
|
||||
success = false
|
||||
}
|
||||
}
|
||||
|
||||
for nk := range needs {
|
||||
folder := folders[nk.folder]
|
||||
if folder == "" {
|
||||
fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
|
||||
vl, ok := globals[nk]
|
||||
if !ok {
|
||||
fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
|
||||
success = false
|
||||
continue
|
||||
}
|
||||
|
||||
if !needsLocally(vl) {
|
||||
fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
|
||||
success = false
|
||||
}
|
||||
}
|
||||
|
||||
if d := len(blocklists) - len(usedBlocklists); d > 0 {
|
||||
fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
|
||||
}
|
||||
if d := len(versions) - len(usedVersions); d > 0 {
|
||||
fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func needsLocally(vl *dbproto.VersionList) bool {
|
||||
gfv, gok := vlGetGlobal(vl)
|
||||
if !gok { // That's weird, but we hardly need something non-existent
|
||||
return false
|
||||
}
|
||||
fv, ok := vlGet(vl, protocol.LocalDeviceID[:])
|
||||
return db.Need(gfv, ok, protocol.VectorFromWire(fv.Version))
|
||||
}
|
||||
|
||||
// Get returns a FileVersion that contains the given device and whether it has
|
||||
// been found at all.
|
||||
func vlGet(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool) {
|
||||
_, i, _, ok := vlFindDevice(vl, device)
|
||||
if !ok {
|
||||
return &dbproto.FileVersion{}, false
|
||||
}
|
||||
return vl.Versions[i], true
|
||||
}
|
||||
|
||||
// GetGlobal returns the current global FileVersion. The returned FileVersion
|
||||
// may be invalid, if all FileVersions are invalid. Returns false only if
|
||||
// VersionList is empty.
|
||||
func vlGetGlobal(vl *dbproto.VersionList) (*dbproto.FileVersion, bool) {
|
||||
i := vlFindGlobal(vl)
|
||||
if i == -1 {
|
||||
return nil, false
|
||||
}
|
||||
return vl.Versions[i], true
|
||||
}
|
||||
|
||||
// findGlobal returns the first version that isn't invalid, or if all versions are
|
||||
// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
|
||||
func vlFindGlobal(vl *dbproto.VersionList) int {
|
||||
for i := range vl.Versions {
|
||||
if !fvIsInvalid(vl.Versions[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
if len(vl.Versions) == 0 {
|
||||
return -1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// findDevice returns whether the device is in InvalidVersions or Versions and
|
||||
// in InvalidDevices or Devices (true for invalid), the positions in the version
|
||||
// and device slices and whether it has been found at all.
|
||||
func vlFindDevice(vl *dbproto.VersionList, device []byte) (bool, int, int, bool) {
|
||||
for i, v := range vl.Versions {
|
||||
if j := deviceIndex(v.Devices, device); j != -1 {
|
||||
return false, i, j, true
|
||||
}
|
||||
if j := deviceIndex(v.InvalidDevices, device); j != -1 {
|
||||
return true, i, j, true
|
||||
}
|
||||
}
|
||||
return false, -1, -1, false
|
||||
}
|
||||
|
||||
func deviceIndex(devices [][]byte, device []byte) int {
|
||||
for i, dev := range devices {
|
||||
if bytes.Equal(device, dev) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func fvFirstDevice(fv *dbproto.FileVersion) ([]byte, bool) {
|
||||
if len(fv.Devices) != 0 {
|
||||
return fv.Devices[0], true
|
||||
}
|
||||
if len(fv.InvalidDevices) != 0 {
|
||||
return fv.InvalidDevices[0], true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func fvIsInvalid(fv *dbproto.FileVersion) bool {
|
||||
return fv == nil || len(fv.Devices) == 0
|
||||
}
|
||||
@@ -17,8 +17,6 @@ import (
|
||||
"path/filepath"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/db/backend"
|
||||
"github.com/syncthing/syncthing/lib/locations"
|
||||
)
|
||||
|
||||
func responseToBArray(response *http.Response) ([]byte, error) {
|
||||
@@ -133,10 +131,6 @@ func prettyPrintResponse(response *http.Response) error {
|
||||
return prettyPrintJSON(data)
|
||||
}
|
||||
|
||||
func getDB() (backend.Backend, error) {
|
||||
return backend.OpenLevelDBRO(locations.Get(locations.Database))
|
||||
}
|
||||
|
||||
func nulString(bs []byte) string {
|
||||
for i := range bs {
|
||||
if bs[i] == 0 {
|
||||
|
||||
+52
-54
@@ -22,6 +22,7 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -38,10 +39,10 @@ import (
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/decrypt"
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/generate"
|
||||
"github.com/syncthing/syncthing/internal/db"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/dialer"
|
||||
"github.com/syncthing/syncthing/lib/events"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
@@ -139,42 +140,41 @@ var entrypoint struct {
|
||||
// serveOptions are the options for the `syncthing serve` command.
|
||||
type serveOptions struct {
|
||||
cmdutil.CommonOptions
|
||||
AllowNewerConfig bool `help:"Allow loading newer than current config version"`
|
||||
Audit bool `help:"Write events to audit file"`
|
||||
AuditFile string `name:"auditfile" placeholder:"PATH" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)"`
|
||||
BrowserOnly bool `help:"Open GUI in browser"`
|
||||
DataDir string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
|
||||
DeviceID bool `help:"Show the device ID"`
|
||||
GenerateDir string `name:"generate" placeholder:"PATH" help:"Generate key and config in specified dir, then exit"` // DEPRECATED: replaced by subcommand!
|
||||
GUIAddress string `name:"gui-address" placeholder:"URL" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")"`
|
||||
GUIAPIKey string `name:"gui-apikey" placeholder:"API-KEY" help:"Override GUI API key"`
|
||||
LogFile string `name:"logfile" default:"${logFile}" placeholder:"PATH" help:"Log file name (see below)"`
|
||||
LogFlags int `name:"logflags" default:"${logFlags}" placeholder:"BITS" help:"Select information in log line prefix (see below)"`
|
||||
LogMaxFiles int `placeholder:"N" default:"${logMaxFiles}" name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)"`
|
||||
LogMaxSize int `placeholder:"BYTES" default:"${logMaxSize}" help:"Maximum size of any file (zero to disable log rotation)"`
|
||||
NoBrowser bool `help:"Do not start browser"`
|
||||
NoRestart bool `env:"STNORESTART" help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash"`
|
||||
NoUpgrade bool `env:"STNOUPGRADE" help:"Disable automatic upgrades"`
|
||||
Paths bool `help:"Show configuration paths"`
|
||||
Paused bool `help:"Start with all devices and folders paused"`
|
||||
Unpaused bool `help:"Start with all devices and folders unpaused"`
|
||||
Upgrade bool `help:"Perform upgrade"`
|
||||
UpgradeCheck bool `help:"Check for available upgrade"`
|
||||
UpgradeTo string `placeholder:"URL" help:"Force upgrade directly from specified URL"`
|
||||
Verbose bool `help:"Print verbose log output"`
|
||||
Version bool `help:"Show version"`
|
||||
AllowNewerConfig bool `help:"Allow loading newer than current config version"`
|
||||
Audit bool `help:"Write events to audit file"`
|
||||
AuditFile string `name:"auditfile" placeholder:"PATH" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)"`
|
||||
BrowserOnly bool `help:"Open GUI in browser"`
|
||||
DataDir string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
|
||||
DeviceID bool `help:"Show the device ID"`
|
||||
GenerateDir string `name:"generate" placeholder:"PATH" help:"Generate key and config in specified dir, then exit"` // DEPRECATED: replaced by subcommand!
|
||||
GUIAddress string `name:"gui-address" placeholder:"URL" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")"`
|
||||
GUIAPIKey string `name:"gui-apikey" placeholder:"API-KEY" help:"Override GUI API key"`
|
||||
LogFile string `name:"logfile" default:"${logFile}" placeholder:"PATH" help:"Log file name (see below)"`
|
||||
LogFlags int `name:"logflags" default:"${logFlags}" placeholder:"BITS" help:"Select information in log line prefix (see below)"`
|
||||
LogMaxFiles int `placeholder:"N" default:"${logMaxFiles}" name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)"`
|
||||
LogMaxSize int `placeholder:"BYTES" default:"${logMaxSize}" help:"Maximum size of any file (zero to disable log rotation)"`
|
||||
NoBrowser bool `help:"Do not start browser"`
|
||||
NoRestart bool `env:"STNORESTART" help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash"`
|
||||
NoUpgrade bool `env:"STNOUPGRADE" help:"Disable automatic upgrades"`
|
||||
Paths bool `help:"Show configuration paths"`
|
||||
Paused bool `help:"Start with all devices and folders paused"`
|
||||
Unpaused bool `help:"Start with all devices and folders unpaused"`
|
||||
Upgrade bool `help:"Perform upgrade"`
|
||||
UpgradeCheck bool `help:"Check for available upgrade"`
|
||||
UpgradeTo string `placeholder:"URL" help:"Force upgrade directly from specified URL"`
|
||||
Verbose bool `help:"Print verbose log output"`
|
||||
Version bool `help:"Show version"`
|
||||
DBMaintenanceInterval time.Duration `env:"STDBMAINTINTERVAL" help:"Database maintenance interval" default:"8h"`
|
||||
|
||||
// Debug options below
|
||||
DebugDBIndirectGCInterval time.Duration `env:"STGCINDIRECTEVERY" help:"Database indirection GC interval"`
|
||||
DebugDBRecheckInterval time.Duration `env:"STRECHECKDBEVERY" help:"Database metadata recalculation interval"`
|
||||
DebugGUIAssetsDir string `placeholder:"PATH" help:"Directory to load GUI assets from" env:"STGUIASSETS"`
|
||||
DebugPerfStats bool `env:"STPERFSTATS" help:"Write running performance statistics to perf-$pid.csv (Unix only)"`
|
||||
DebugProfileBlock bool `env:"STBLOCKPROFILE" help:"Write block profiles to block-$pid-$timestamp.pprof every 20 seconds"`
|
||||
DebugProfileCPU bool `help:"Write a CPU profile to cpu-$pid.pprof on exit" env:"STCPUPROFILE"`
|
||||
DebugProfileHeap bool `env:"STHEAPPROFILE" help:"Write heap profiles to heap-$pid-$timestamp.pprof each time heap usage increases"`
|
||||
DebugProfilerListen string `placeholder:"ADDR" env:"STPROFILER" help:"Network profiler listen address"`
|
||||
DebugResetDatabase bool `name:"reset-database" help:"Reset the database, forcing a full rescan and resync"`
|
||||
DebugResetDeltaIdxs bool `name:"reset-deltas" help:"Reset delta index IDs, forcing a full index exchange"`
|
||||
DebugGUIAssetsDir string `placeholder:"PATH" help:"Directory to load GUI assets from" env:"STGUIASSETS"`
|
||||
DebugPerfStats bool `env:"STPERFSTATS" help:"Write running performance statistics to perf-$pid.csv (Unix only)"`
|
||||
DebugProfileBlock bool `env:"STBLOCKPROFILE" help:"Write block profiles to block-$pid-$timestamp.pprof every 20 seconds"`
|
||||
DebugProfileCPU bool `help:"Write a CPU profile to cpu-$pid.pprof on exit" env:"STCPUPROFILE"`
|
||||
DebugProfileHeap bool `env:"STHEAPPROFILE" help:"Write heap profiles to heap-$pid-$timestamp.pprof each time heap usage increases"`
|
||||
DebugProfilerListen string `placeholder:"ADDR" env:"STPROFILER" help:"Network profiler listen address"`
|
||||
DebugResetDatabase bool `name:"reset-database" help:"Reset the database, forcing a full rescan and resync"`
|
||||
DebugResetDeltaIdxs bool `name:"reset-deltas" help:"Reset delta index IDs, forcing a full index exchange"`
|
||||
|
||||
// Internal options, not shown to users
|
||||
InternalRestarting bool `env:"STRESTART" hidden:"1"`
|
||||
@@ -592,8 +592,12 @@ func syncthingMain(options serveOptions) {
|
||||
})
|
||||
}
|
||||
|
||||
dbFile := locations.Get(locations.Database)
|
||||
ldb, err := syncthing.OpenDBBackend(dbFile, cfgWrapper.Options().DatabaseTuning)
|
||||
if err := syncthing.TryMigrateDatabase(); err != nil {
|
||||
l.Warnln("Failed to migrate old-style database:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database))
|
||||
if err != nil {
|
||||
l.Warnln("Error opening database:", err)
|
||||
os.Exit(1)
|
||||
@@ -602,11 +606,11 @@ func syncthingMain(options serveOptions) {
|
||||
// Check if auto-upgrades is possible, and if yes, and it's enabled do an initial
|
||||
// upgrade immediately. The auto-upgrade routine can only be started
|
||||
// later after App is initialised.
|
||||
|
||||
autoUpgradePossible := autoUpgradePossible(options)
|
||||
if autoUpgradePossible && cfgWrapper.Options().AutoUpgradeEnabled() {
|
||||
// try to do upgrade directly and log the error if relevant.
|
||||
release, err := initialAutoUpgradeCheck(db.NewMiscDataNamespace(ldb))
|
||||
miscDB := db.NewMiscDB(sdb)
|
||||
release, err := initialAutoUpgradeCheck(miscDB)
|
||||
if err == nil {
|
||||
err = upgrade.To(release)
|
||||
}
|
||||
@@ -617,7 +621,7 @@ func syncthingMain(options serveOptions) {
|
||||
l.Infoln("Initial automatic upgrade:", err)
|
||||
}
|
||||
} else {
|
||||
l.Infof("Upgraded to %q, exiting now.", release.Tag)
|
||||
l.Infof("Upgraded to %q, should exit now.", release.Tag)
|
||||
os.Exit(svcutil.ExitUpgrade.AsInt())
|
||||
}
|
||||
}
|
||||
@@ -629,24 +633,17 @@ func syncthingMain(options serveOptions) {
|
||||
}
|
||||
|
||||
appOpts := syncthing.Options{
|
||||
NoUpgrade: options.NoUpgrade,
|
||||
ProfilerAddr: options.DebugProfilerListen,
|
||||
ResetDeltaIdxs: options.DebugResetDeltaIdxs,
|
||||
Verbose: options.Verbose,
|
||||
DBRecheckInterval: options.DebugDBRecheckInterval,
|
||||
DBIndirectGCInterval: options.DebugDBIndirectGCInterval,
|
||||
NoUpgrade: options.NoUpgrade,
|
||||
ProfilerAddr: options.DebugProfilerListen,
|
||||
ResetDeltaIdxs: options.DebugResetDeltaIdxs,
|
||||
Verbose: options.Verbose,
|
||||
DBMaintenanceInterval: options.DBMaintenanceInterval,
|
||||
}
|
||||
if options.Audit {
|
||||
appOpts.AuditWriter = auditWriter(options.AuditFile)
|
||||
}
|
||||
if dur, err := time.ParseDuration(os.Getenv("STRECHECKDBEVERY")); err == nil {
|
||||
appOpts.DBRecheckInterval = dur
|
||||
}
|
||||
if dur, err := time.ParseDuration(os.Getenv("STGCINDIRECTEVERY")); err == nil {
|
||||
appOpts.DBIndirectGCInterval = dur
|
||||
}
|
||||
|
||||
app, err := syncthing.New(cfgWrapper, ldb, evLogger, cert, appOpts)
|
||||
app, err := syncthing.New(cfgWrapper, sdb, evLogger, cert, appOpts)
|
||||
if err != nil {
|
||||
l.Warnln("Failed to start Syncthing:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
@@ -692,6 +689,7 @@ func syncthingMain(options serveOptions) {
|
||||
pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
runtime.KeepAlive(lf) // ensure lock is still held to this point
|
||||
os.Exit(int(status))
|
||||
}
|
||||
|
||||
@@ -833,7 +831,7 @@ func autoUpgrade(cfg config.Wrapper, app *syncthing.App, evLogger events.Logger)
|
||||
}
|
||||
}
|
||||
|
||||
func initialAutoUpgradeCheck(misc *db.NamespacedKV) (upgrade.Release, error) {
|
||||
func initialAutoUpgradeCheck(misc *db.Typed) (upgrade.Release, error) {
|
||||
if last, ok, err := misc.Time(upgradeCheckKey); err == nil && ok && time.Since(last) < upgradeCheckInterval {
|
||||
return upgrade.Release{}, errTooEarlyUpgradeCheck
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user