felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: switch database engine to sqlite (fixes #9954) (#9965)

Browse Source 
Switch the database from LevelDB to SQLite, for greater stability and
simpler code.

Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl>
Co-authored-by: bt90 <btom1990@googlemail.com>
This commit is contained in:
Jakob Borg
2025-03-29 13:50:08 +01:00
committed by GitHub
co-authored by Tommy van der Vorst bt90
parent b1c8f88a44
commit 025905fcdf
146 changed files with 8315 additions and 11984 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/api/api_test.go
+56 -108
View File
@@ -27,12 +27,12 @@ import (
"github.com/d4l3k/messagediff"
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/assets"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
connmocks "github.com/syncthing/syncthing/lib/connections/mocks"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
discovermocks "github.com/syncthing/syncthing/lib/discover/mocks"
"github.com/syncthing/syncthing/lib/events"
eventmocks "github.com/syncthing/syncthing/lib/events/mocks"
@@ -84,8 +84,14 @@ func TestStopAfterBrokenConfig(t *testing.T) {
}
w := config.Wrap("/dev/null", cfg, protocol.LocalDeviceID, events.NoopLogger)
mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
kdb := db.NewMiscDataNamespace(mdb)
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mdb.Close()
})
kdb := db.NewMiscDB(mdb)
srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, events.NoopLogger, nil, nil, nil, nil, nil, nil, false, kdb).(*service)
srv.started = make(chan string)
@@ -217,11 +223,7 @@ type httpTestCase struct {
func TestAPIServiceRequests(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, apiCfg)
cases := []httpTestCase{
// /rest/db
@@ -598,11 +600,7 @@ func TestHTTPLogin(t *testing.T) {
APIKey: testAPIKey,
SendBasicAuthPrompt: sendBasicAuthPrompt,
})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, cfg)
url := baseURL + path
t.Run(fmt.Sprintf("%d path", expectedOkStatus), func(t *testing.T) {
@@ -795,13 +793,9 @@ func TestHTTPLogin(t *testing.T) {
w := initConfig(initialPassword, t)
{
baseURL, cancel, err := startHTTPWithShutdownTimeout(w, shutdownTimeout)
baseURL := startHTTPWithShutdownTimeout(t, w, shutdownTimeout)
cfgPath := baseURL + "/rest/config"
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusOK {
@@ -813,12 +807,8 @@ func TestHTTPLogin(t *testing.T) {
httpRequest(http.MethodPut, cfgPath, cfg, "", "", testAPIKey, "", "", "", nil, t)
}
{
baseURL, cancel, err := startHTTP(w)
baseURL := startHTTP(t, w)
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusForbidden {
@@ -837,13 +827,9 @@ func TestHTTPLogin(t *testing.T) {
w := initConfig(initialPassword, t)
{
baseURL, cancel, err := startHTTPWithShutdownTimeout(w, shutdownTimeout)
baseURL := startHTTPWithShutdownTimeout(t, w, shutdownTimeout)
cfgPath := baseURL + "/rest/config/gui"
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusOK {
@@ -855,12 +841,8 @@ func TestHTTPLogin(t *testing.T) {
httpRequest(http.MethodPut, cfgPath, cfg.GUI, "", "", testAPIKey, "", "", "", nil, t)
}
{
baseURL, cancel, err := startHTTP(w)
baseURL := startHTTP(t, w)
path := baseURL + "/meta.js"
t.Cleanup(cancel)
if err != nil {
t.Fatal(err)
}
resp := httpGetBasicAuth(path, "user", initialPassword)
if resp.StatusCode != http.StatusForbidden {
@@ -885,11 +867,7 @@ func TestHtmlFormLogin(t *testing.T) {
Password: "$2a$10$IdIZTxTg/dCNuNEGlmLynOjqg4B1FvDKuIV5e0BB3pnWVHNb8.GSq", // bcrypt of "räksmörgås" in UTF-8
SendBasicAuthPrompt: false,
})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, cfg)
loginUrl := baseURL + "/rest/noauth/auth/password"
resourceUrl := baseURL + "/meta.js"
@@ -1030,11 +1008,7 @@ func TestApiCache(t *testing.T) {
RawAddress: "127.0.0.1:0",
APIKey: testAPIKey,
})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, cfg)
httpGet := func(url string, bearer string) *http.Response {
return httpGet(url, "", "", "", bearer, nil, t)
@@ -1059,11 +1033,11 @@ func TestApiCache(t *testing.T) {
})
}
func startHTTP(cfg config.Wrapper) (string, context.CancelFunc, error) {
return startHTTPWithShutdownTimeout(cfg, 0)
func startHTTP(t *testing.T, cfg config.Wrapper) string {
return startHTTPWithShutdownTimeout(t, cfg, 0)
}
func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Duration) (string, context.CancelFunc, error) {
func startHTTPWithShutdownTimeout(t *testing.T, cfg config.Wrapper, shutdownTimeout time.Duration) string {
m := new(modelmocks.Model)
assetDir := "../../gui"
eventSub := new(eventmocks.BufferedSubscription)
@@ -1086,12 +1060,18 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
// Instantiate the API service
urService := ur.New(cfg, m, connections, false)
mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
kdb := db.NewMiscDataNamespace(mdb)
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mdb.Close()
})
kdb := db.NewMiscDB(mdb)
svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, mockedSummary, errorLog, systemLog, false, kdb).(*service)
svc.started = addrChan
if shutdownTimeout > 0*time.Millisecond {
if shutdownTimeout > 0 {
svc.shutdownTimeout = shutdownTimeout
}
@@ -1101,14 +1081,14 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
})
supervisor.Add(svc)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
supervisor.ServeBackground(ctx)
// Make sure the API service is listening, and get the URL to use.
addr := <-addrChan
tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
if err != nil {
cancel()
return "", cancel, fmt.Errorf("weird address from API service: %w", err)
t.Fatal(fmt.Errorf("weird address from API service: %w", err))
}
host, _, _ := net.SplitHostPort(cfg.GUI().RawAddress)
@@ -1117,17 +1097,13 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
}
baseURL := fmt.Sprintf("http://%s", net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port)))
return baseURL, cancel, nil
return baseURL
}
func TestCSRFRequired(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal("Unexpected error from getting base URL:", err)
}
t.Cleanup(cancel)
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Minute,
@@ -1245,11 +1221,7 @@ func TestCSRFRequired(t *testing.T) {
func TestRandomString(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1304,7 +1276,7 @@ func TestConfigPostOK(t *testing.T) {
]
}`))
resp, err := testConfigPost(cfg)
resp, err := testConfigPost(t, cfg)
if err != nil {
t.Fatal(err)
}
@@ -1325,7 +1297,7 @@ func TestConfigPostDupFolder(t *testing.T) {
]
}`))
resp, err := testConfigPost(cfg)
resp, err := testConfigPost(t, cfg)
if err != nil {
t.Fatal(err)
}
@@ -1334,12 +1306,10 @@ func TestConfigPostDupFolder(t *testing.T) {
}
}
func testConfigPost(data io.Reader) (*http.Response, error) {
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
return nil, err
}
defer cancel()
func testConfigPost(t *testing.T, data io.Reader) (*http.Response, error) {
t.Helper()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1356,11 +1326,7 @@ func TestHostCheck(t *testing.T) {
cfg := newMockedConfig()
cfg.GUIReturns(config.GUIConfiguration{RawAddress: "127.0.0.1:0"})
baseURL, cancel, err := startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, cfg)
// A normal HTTP get to the localhost-bound service should succeed
@@ -1419,11 +1385,7 @@ func TestHostCheck(t *testing.T) {
RawAddress: "127.0.0.1:0",
InsecureSkipHostCheck: true,
})
baseURL, cancel, err = startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL = startHTTP(t, cfg)
// A request with a suspicious Host header should be allowed
@@ -1445,11 +1407,7 @@ func TestHostCheck(t *testing.T) {
cfg.GUIReturns(config.GUIConfiguration{
RawAddress: "0.0.0.0:0",
})
baseURL, cancel, err = startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL = startHTTP(t, cfg)
// A request with a suspicious Host header should be allowed
@@ -1476,11 +1434,7 @@ func TestHostCheck(t *testing.T) {
cfg.GUIReturns(config.GUIConfiguration{
RawAddress: "[::1]:0",
})
baseURL, cancel, err = startHTTP(cfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL = startHTTP(t, cfg)
// A normal HTTP get to the localhost-bound service should succeed
@@ -1568,11 +1522,7 @@ func TestAddressIsLocalhost(t *testing.T) {
func TestAccessControlAllowOriginHeader(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1596,11 +1546,7 @@ func TestAccessControlAllowOriginHeader(t *testing.T) {
func TestOptionsRequest(t *testing.T) {
t.Parallel()
baseURL, cancel, err := startHTTP(apiCfg)
if err != nil {
t.Fatal(err)
}
defer cancel()
baseURL := startHTTP(t, apiCfg)
cli := &http.Client{
Timeout: time.Second,
}
@@ -1632,8 +1578,14 @@ func TestEventMasks(t *testing.T) {
cfg := newMockedConfig()
defSub := new(eventmocks.BufferedSubscription)
diskSub := new(eventmocks.BufferedSubscription)
mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
kdb := db.NewMiscDataNamespace(mdb)
mdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mdb.Close()
})
kdb := db.NewMiscDB(mdb)
svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, events.NoopLogger, nil, nil, nil, nil, nil, nil, false, kdb).(*service)
if mask := svc.getEventMask(""); mask != DefaultEventMask {
@@ -1780,11 +1732,7 @@ func TestConfigChanges(t *testing.T) {
cfgCtx, cfgCancel := context.WithCancel(context.Background())
go w.Serve(cfgCtx)
defer cfgCancel()
baseURL, cancel, err := startHTTP(w)
if err != nil {
t.Fatal("Unexpected error from getting base URL:", err)
}
defer cancel()
baseURL := startHTTP(t, w)
cli := &http.Client{
Timeout: time.Minute,