feat(ursrv): new metrics based approach

cmd/stdiscosrv/database.go

@@ -26,6 +26,8 @@ import (
 
 	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/s3"
 )
 
 type clock interface {
@@ -48,27 +50,38 @@ type inMemoryStore struct {
 	m             *xsync.MapOf[protocol.DeviceID, DatabaseRecord]
 	dir           string
 	flushInterval time.Duration
-	s3            *s3Copier
+	s3            *s3.Session
+	objKey        string
 	clock         clock
 }
 
-func newInMemoryStore(dir string, flushInterval time.Duration, s3 *s3Copier) *inMemoryStore {
+func newInMemoryStore(dir string, flushInterval time.Duration, s3sess *s3.Session) *inMemoryStore {
+	hn, err := os.Hostname()
+	if err != nil {
+		hn = rand.String(8)
+	}
 	s := &inMemoryStore{
 		m:             xsync.NewMapOf[protocol.DeviceID, DatabaseRecord](),
 		dir:           dir,
 		flushInterval: flushInterval,
-		s3:            s3,
+		s3:            s3sess,
+		objKey:        hn + ".db",
 		clock:         defaultClock{},
 	}
 	nr, err := s.read()
-	if os.IsNotExist(err) && s3 != nil {
+	if os.IsNotExist(err) && s3sess != nil {
 		// Try to read from AWS
+		latestKey, cerr := s3sess.LatestKey()
+		if cerr != nil {
+			log.Println("Error reading database from S3:", err)
+			return s
+		}
 		fd, cerr := os.Create(path.Join(s.dir, "records.db"))
 		if cerr != nil {
 			log.Println("Error creating database file:", err)
 			return s
 		}
-		if err := s3.downloadLatest(fd); err != nil {
+		if cerr := s3sess.Download(fd, latestKey); cerr != nil {
 			log.Printf("Error reading database from S3: %v", err)
 		}
 		_ = fd.Close()
@@ -303,7 +316,7 @@ func (s *inMemoryStore) write() (err error) {
 			return nil
 		}
 		defer fd.Close()
-		if err := s.s3.upload(fd); err != nil {
+		if err := s.s3.Upload(fd, s.objKey); err != nil {
 			log.Printf("Error uploading database to S3: %v", err)
 		}
 		log.Println("Finished uploading database")

cmd/stdiscosrv/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/s3"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/thejerf/suture/v4"
 )
@@ -117,14 +118,13 @@ func main() {
 	})
 
 	// If configured, use S3 for database backups.
-	var s3c *s3Copier
+	var s3c *s3.Session
 	if cli.DBS3Endpoint != "" {
-		hostname, err := os.Hostname()
+		var err error
+		s3c, err = s3.NewSession(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
 		if err != nil {
-			log.Fatalf("Failed to get hostname: %v", err)
+			log.Fatalf("Failed to create S3 session: %v", err)
 		}
-		key := hostname + ".db"
-		s3c = newS3Copier(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, key, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
 	}
 
 	// Start the database.

cmd/stdiscosrv/s3.go

@@ -1,97 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"io"
-	"log"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-)
-
-type s3Copier struct {
-	endpoint    string
-	region      string
-	bucket      string
-	key         string
-	accessKeyID string
-	secretKey   string
-}
-
-func newS3Copier(endpoint, region, bucket, key, accessKeyID, secretKey string) *s3Copier {
-	return &s3Copier{
-		endpoint:    endpoint,
-		region:      region,
-		bucket:      bucket,
-		key:         key,
-		accessKeyID: accessKeyID,
-		secretKey:   secretKey,
-	}
-}
-
-func (s *s3Copier) upload(r io.Reader) error {
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String(s.region),
-		Endpoint:    aws.String(s.endpoint),
-		Credentials: credentials.NewStaticCredentials(s.accessKeyID, s.secretKey, ""),
-	})
-	if err != nil {
-		return err
-	}
-
-	uploader := s3manager.NewUploader(sess)
-	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(s.key),
-		Body:   r,
-	})
-	return err
-}
-
-func (s *s3Copier) downloadLatest(w io.WriterAt) error {
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String(s.region),
-		Endpoint:    aws.String(s.endpoint),
-		Credentials: credentials.NewStaticCredentials(s.accessKeyID, s.secretKey, ""),
-	})
-	if err != nil {
-		return err
-	}
-
-	svc := s3.New(sess)
-	resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.bucket)})
-	if err != nil {
-		return err
-	}
-
-	var lastKey string
-	var lastModified time.Time
-	var lastSize int64
-	for _, item := range resp.Contents {
-		if item.LastModified.After(lastModified) && *item.Size > lastSize {
-			lastKey = *item.Key
-			lastModified = *item.LastModified
-			lastSize = *item.Size
-		} else if lastModified.Sub(*item.LastModified) < 5*time.Minute && *item.Size > lastSize {
-			lastKey = *item.Key
-			lastSize = *item.Size
-		}
-	}
-
-	log.Println("Downloading database from", lastKey)
-	downloader := s3manager.NewDownloader(sess)
-	_, err = downloader.Download(w, &s3.GetObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(lastKey),
-	})
-	return err
-}