diff --git a/cmd/infra/ursrv/serve/serve.go b/cmd/infra/ursrv/serve/serve.go index bddece60d..803b869de 100644 --- a/cmd/infra/ursrv/serve/serve.go +++ b/cmd/infra/ursrv/serve/serve.go @@ -27,7 +27,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/puzpuzpuz/xsync/v3" "github.com/syncthing/syncthing/internal/blob" - "github.com/syncthing/syncthing/internal/blob/azureblob" "github.com/syncthing/syncthing/internal/blob/s3" "github.com/syncthing/syncthing/internal/slogutil" "github.com/syncthing/syncthing/lib/build" @@ -49,10 +48,6 @@ type CLI struct { S3Bucket string `name:"s3-bucket" env:"UR_S3_BUCKET"` S3AccessKeyID string `name:"s3-access-key-id" env:"UR_S3_ACCESS_KEY_ID"` S3SecretKey string `name:"s3-secret-key" env:"UR_S3_SECRET_KEY"` - - AzureBlobAccount string `name:"azure-blob-account" env:"UR_AZUREBLOB_ACCOUNT"` - AzureBlobKey string `name:"azure-blob-key" env:"UR_AZUREBLOB_KEY"` - AzureBlobContainer string `name:"azure-blob-container" env:"UR_AZUREBLOB_CONTAINER"` } var ( @@ -138,12 +133,6 @@ func (cli *CLI) Run() error { slog.Error("Failed to create S3 session", slogutil.Error(err)) return err } - } else if cli.AzureBlobAccount != "" { - blobs, err = azureblob.NewBlobStore(cli.AzureBlobAccount, cli.AzureBlobKey, cli.AzureBlobContainer) - if err != nil { - slog.Error("Failed to create Azure blob store", slogutil.Error(err)) - return err - } } if _, err := os.Stat(cli.DumpFile); err != nil && blobs != nil { diff --git a/cmd/stdiscosrv/database.go b/cmd/stdiscosrv/database.go index 3ef147f44..d9c9d619b 100644 --- a/cmd/stdiscosrv/database.go +++ b/cmd/stdiscosrv/database.go @@ -113,7 +113,7 @@ func (s *inMemoryStore) merge(key *protocol.DeviceID, addrs []*discosrv.Database } if oldRec, ok := s.m.Load(*key); ok { - newRec = merge(oldRec, newRec) + newRec = merge(newRec, oldRec) } s.m.Store(*key, newRec) @@ -135,7 +135,13 @@ func (s *inMemoryStore) get(key *protocol.DeviceID) (*discosrv.DatabaseRecord, e return &discosrv.DatabaseRecord{}, nil } - rec.Addresses = expire(rec.Addresses, s.clock.Now()) + naddresses, changed := expire(rec.Addresses, s.clock.Now()) + if changed { + rec = &discosrv.DatabaseRecord{ + Addresses: naddresses, + Seen: rec.Seen, + } + } databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc() return rec, nil } @@ -184,12 +190,12 @@ func (s *inMemoryStore) expireAndCalculateStatistics() { } n++ - addresses := expire(rec.Addresses, now) - if len(addresses) == 0 { - rec.Addresses = nil - s.m.Store(key, rec) - } else if len(addresses) != len(rec.Addresses) { - rec.Addresses = addresses + addresses, changed := expire(rec.Addresses, now) + if changed { + rec = &discosrv.DatabaseRecord{ + Addresses: addresses, + Seen: rec.Seen, + } s.m.Store(key, rec) } @@ -371,9 +377,9 @@ func (s *inMemoryStore) read() (int, error) { } slices.SortFunc(rec.Addresses, Cmp) - rec.Addresses = slices.CompactFunc(rec.Addresses, Equal) + rec.Addresses, _ = expire(slices.CompactFunc(rec.Addresses, Equal), s.clock.Now()) s.m.Store(key, &discosrv.DatabaseRecord{ - Addresses: expire(rec.Addresses, s.clock.Now()), + Addresses: rec.Addresses, Seen: rec.Seen, }) nr++ @@ -384,7 +390,7 @@ func (s *inMemoryStore) read() (int, error) { // merge returns the merged result of the two database records a and b. The // result is the union of the two address sets, with the newer expiry time // chosen for any duplicates. The address list in a is overwritten and -// reused for the result. +// reused for the result; b is not modified. func merge(a, b *discosrv.DatabaseRecord) *discosrv.DatabaseRecord { // Both lists must be sorted for this to work. @@ -415,25 +421,33 @@ func merge(a, b *discosrv.DatabaseRecord) *discosrv.DatabaseRecord { return a } -// expire returns the list of addresses after removing expired entries. -// Expiration happen in place, so the slice given as the parameter is -// destroyed. Internal order is preserved. -func expire(addrs []*discosrv.DatabaseAddress, now time.Time) []*discosrv.DatabaseAddress { +// expire returns the list of addresses after removing expired entries. A +// new slice is allocated if any changes are required, and the changed +// boolean indicates whether that happened or not. +func expire(addrs []*discosrv.DatabaseAddress, now time.Time) (result []*discosrv.DatabaseAddress, changed bool) { cutoff := now.UnixNano() - naddrs := addrs[:0] - for i := range addrs { - if i > 0 && addrs[i].Address == addrs[i-1].Address { - // Skip duplicates - continue - } - if addrs[i].Expires >= cutoff { - naddrs = append(naddrs, addrs[i]) + remains := 0 + for _, a := range addrs { + if a.Expires < cutoff { + changed = true + } else { + remains++ } } - if len(naddrs) == 0 { - return nil + if !changed { + return addrs, false } - return naddrs + if remains == 0 { + return nil, true + } + + naddrs := make([]*discosrv.DatabaseAddress, 0, remains) + for _, a := range addrs { + if a.Expires >= cutoff { + naddrs = append(naddrs, a) + } + } + return naddrs, true } func Cmp(d, other *discosrv.DatabaseAddress) (n int) { diff --git a/cmd/stdiscosrv/database_test.go b/cmd/stdiscosrv/database_test.go index 496e0184b..0c8d5946b 100644 --- a/cmd/stdiscosrv/database_test.go +++ b/cmd/stdiscosrv/database_test.go @@ -161,7 +161,7 @@ func TestFilter(t *testing.T) { } for _, tc := range cases { - res := expire(tc.a, time.Unix(0, 10)) + res, _ := expire(tc.a, time.Unix(0, 10)) if fmt.Sprint(res) != fmt.Sprint(tc.b) { t.Errorf("Incorrect result %v, expected %v", res, tc.b) } diff --git a/cmd/stdiscosrv/main.go b/cmd/stdiscosrv/main.go index c61ff4d76..4e2c04889 100644 --- a/cmd/stdiscosrv/main.go +++ b/cmd/stdiscosrv/main.go @@ -22,7 +22,6 @@ import ( "github.com/thejerf/suture/v4" "github.com/syncthing/syncthing/internal/blob" - "github.com/syncthing/syncthing/internal/blob/azureblob" "github.com/syncthing/syncthing/internal/blob/s3" "github.com/syncthing/syncthing/internal/slogutil" _ "github.com/syncthing/syncthing/lib/automaxprocs" @@ -77,10 +76,6 @@ type CLI struct { DBS3AccessKeyID string `name:"db-s3-access-key-id" group:"Database (S3 backup)" hidden:"true" help:"S3 access key ID for database" env:"DISCOVERY_DB_S3_ACCESS_KEY_ID"` DBS3SecretKey string `name:"db-s3-secret-key" group:"Database (S3 backup)" hidden:"true" help:"S3 secret key for database" env:"DISCOVERY_DB_S3_SECRET_KEY"` - DBAzureBlobAccount string `name:"db-azure-blob-account" env:"DISCOVERY_DB_AZUREBLOB_ACCOUNT"` - DBAzureBlobKey string `name:"db-azure-blob-key" env:"DISCOVERY_DB_AZUREBLOB_KEY"` - DBAzureBlobContainer string `name:"db-azure-blob-container" env:"DISCOVERY_DB_AZUREBLOB_CONTAINER"` - AMQPAddress string `group:"AMQP replication" hidden:"true" help:"Address to AMQP broker" env:"DISCOVERY_AMQP_ADDRESS"` Debug bool `short:"d" help:"Print debug output" env:"DISCOVERY_DEBUG"` @@ -134,8 +129,6 @@ func main() { var err error if cli.DBS3Endpoint != "" { blobs, err = s3.NewSession(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, cli.DBS3AccessKeyID, cli.DBS3SecretKey) - } else if cli.DBAzureBlobAccount != "" { - blobs, err = azureblob.NewBlobStore(cli.DBAzureBlobAccount, cli.DBAzureBlobKey, cli.DBAzureBlobContainer) } if err != nil { slog.Error("Failed to create blob store", "error", err) diff --git a/go.mod b/go.mod index 2a223a776..8a491e35a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.24.0 require ( github.com/AudriusButkevicius/recli v0.0.7 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 github.com/alecthomas/kong v1.12.1 github.com/aws/aws-sdk-go v1.55.8 github.com/calmh/incontainer v1.0.0 @@ -54,8 +53,6 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -90,6 +87,7 @@ require ( github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.10.0 // indirect diff --git a/go.sum b/go.sum index c15bced16..b21184af5 100644 --- a/go.sum +++ b/go.sum @@ -2,20 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/AudriusButkevicius/recli v0.0.7 h1:9zjbYlTupi+W5SJXm2cR2sV2mJAIg1sIfDcsW7hrkPM= github.com/AudriusButkevicius/recli v0.0.7/go.mod h1:Nhfib1j/VFnLrXL9cHgA+/n2O6P5THuWelOnbfPNd78= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= @@ -81,8 +69,6 @@ github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqw github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= -github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= -github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -195,8 +181,6 @@ github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5 github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= -github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/internal/blob/azureblob/azureblob.go b/internal/blob/azureblob/azureblob.go deleted file mode 100644 index a574ccc46..000000000 --- a/internal/blob/azureblob/azureblob.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (C) 2025 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -package azureblob - -import ( - "context" - "io" - "time" - - stblob "github.com/syncthing/syncthing/internal/blob" - - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" -) - -var _ stblob.Store = (*BlobStore)(nil) - -type BlobStore struct { - client *azblob.Client - container string -} - -func NewBlobStore(accountName, accountKey, containerName string) (*BlobStore, error) { - credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) - if err != nil { - return nil, err - } - url := "https://" + accountName + ".blob.core.windows.net/" - sc, err := azblob.NewClientWithSharedKeyCredential(url, credential, &azblob.ClientOptions{}) - if err != nil { - return nil, err - } - // This errors when the container already exists, which we ignore. - _, _ = sc.CreateContainer(context.Background(), containerName, &container.CreateOptions{}) - return &BlobStore{ - client: sc, - container: containerName, - }, nil -} - -func (a *BlobStore) Upload(ctx context.Context, key string, data io.Reader) error { - _, err := a.client.UploadStream(ctx, a.container, key, data, &blockblob.UploadStreamOptions{}) - return err -} - -func (a *BlobStore) Download(ctx context.Context, key string, w stblob.Writer) error { - resp, err := a.client.DownloadStream(ctx, a.container, key, &blob.DownloadStreamOptions{}) - if err != nil { - return err - } - defer resp.Body.Close() - _, err = io.Copy(w, resp.Body) - return err -} - -func (a *BlobStore) LatestKey(ctx context.Context) (string, error) { - opts := &azblob.ListBlobsFlatOptions{} - pager := a.client.NewListBlobsFlatPager(a.container, opts) - var latest string - var lastModified time.Time - for pager.More() { - page, err := pager.NextPage(ctx) - if err != nil { - return "", err - } - for _, blob := range page.Segment.BlobItems { - if latest == "" || blob.Properties.LastModified.After(lastModified) { - latest = *blob.Name - lastModified = *blob.Properties.LastModified - } - } - } - return latest, nil -}