felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

Merge branch 'main' into v2

Browse Source 
* main:
  feat(stdiscosrv): configurable desired not-found rate
  chore(blobs): generalised blob storage
  chore(stdiscosrv): path style s3
  feat(ursv): add os/arch/distribution metric
  chore(strelaypoolsrv): limit number of returned relays
  build(infra): run in Docker environment for pushes
  chore(stupgrades): expose latest release as a metric
  feat(api, gui): allow authentication bypass for metrics (#10045)
This commit is contained in:
Jakob Borg
2025-04-13 09:44:09 +02:00
parent 780b8fd3bc ed97e365b2
commit ed6575411f
18 changed files with 274 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/infra/strelaypoolsrv/main.go
+7
View File
@@ -29,6 +29,7 @@ import (
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/geoip"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/relay/client"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/tlsutil"
@@ -110,6 +111,7 @@ var (
requestProcessors = 8
geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY")
geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
maxRelaysReturned = 100
requests chan request
@@ -141,6 +143,7 @@ func main() {
flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
flag.IntVar(&maxRelaysReturned, "max-relays-returned", maxRelaysReturned, "Maximum number of relays returned for a normal endpoint query")
flag.Parse()
@@ -331,6 +334,10 @@ func handleEndpointShort(rw http.ResponseWriter, r *http.Request) {
relays = append(relays, relayShort{URL: slimURL(r.URL)})
}
mut.RUnlock()
if len(relays) > maxRelaysReturned {
rand.Shuffle(relays)
relays = relays[:maxRelaysReturned]
}
_ = json.NewEncoder(rw).Encode(map[string][]relayShort{
"relays": relays,
cmd/infra/stupgrades/main.go
+22 -3
View File
@@ -258,9 +258,10 @@ func filterForCompabitility(rels []upgrade.Release, ua, osv string) []upgrade.Re
}
type cachedReleases struct {
url string
mut sync.RWMutex
current []upgrade.Release
url string
mut sync.RWMutex
current []upgrade.Release
latestRel, latestPre string
}
func (c *cachedReleases) Releases() []upgrade.Release {
@@ -274,8 +275,26 @@ func (c *cachedReleases) Update(ctx context.Context) error {
if err != nil {
return err
}
latestRel, latestPre := "", ""
for _, rel := range rels {
if !rel.Prerelease && latestRel == "" {
latestRel = rel.Tag
}
if rel.Prerelease && latestPre == "" {
latestPre = rel.Tag
}
if latestRel != "" && latestPre != "" {
break
}
}
c.mut.Lock()
c.current = rels
if latestRel != c.latestRel || latestPre != c.latestPre {
metricLatestReleaseInfo.DeleteLabelValues(c.latestRel, c.latestPre)
metricLatestReleaseInfo.WithLabelValues(latestRel, latestPre).Set(1)
c.latestRel = latestRel
c.latestPre = latestPre
}
c.mut.Unlock()
return nil
}
cmd/infra/stupgrades/metrics.go
+6
View File
@@ -27,4 +27,10 @@ var (
Subsystem: "upgrade",
Name: "http_requests",
}, []string{"target", "result"})
metricLatestReleaseInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "upgrade",
Name: "latest_release_info",
Help: "Release information",
}, []string{"latest_release", "latest_pre"})
)
cmd/infra/ursrv/serve/serve.go
+34 -18
View File
@@ -26,9 +26,11 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/puzpuzpuz/xsync/v3"
"github.com/syncthing/syncthing/internal/blob"
"github.com/syncthing/syncthing/internal/blob/azureblob"
"github.com/syncthing/syncthing/internal/blob/s3"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/geoip"
"github.com/syncthing/syncthing/lib/s3"
"github.com/syncthing/syncthing/lib/ur/contract"
)
@@ -40,11 +42,15 @@ type CLI struct {
DumpFile string `env:"UR_DUMP_FILE" default:"reports.jsons.gz"`
DumpInterval time.Duration `env:"UR_DUMP_INTERVAL" default:"5m"`
S3Endpoint string `name:"s3-endpoint" hidden:"true" env:"UR_S3_ENDPOINT"`
S3Region string `name:"s3-region" hidden:"true" env:"UR_S3_REGION"`
S3Bucket string `name:"s3-bucket" hidden:"true" env:"UR_S3_BUCKET"`
S3AccessKeyID string `name:"s3-access-key-id" hidden:"true" env:"UR_S3_ACCESS_KEY_ID"`
S3SecretKey string `name:"s3-secret-key" hidden:"true" env:"UR_S3_SECRET_KEY"`
S3Endpoint string `name:"s3-endpoint" env:"UR_S3_ENDPOINT"`
S3Region string `name:"s3-region" env:"UR_S3_REGION"`
S3Bucket string `name:"s3-bucket" env:"UR_S3_BUCKET"`
S3AccessKeyID string `name:"s3-access-key-id" env:"UR_S3_ACCESS_KEY_ID"`
S3SecretKey string `name:"s3-secret-key" env:"UR_S3_SECRET_KEY"`
AzureBlobAccount string `name:"azure-blob-account" env:"UR_AZUREBLOB_ACCOUNT"`
AzureBlobKey string `name:"azure-blob-key" env:"UR_AZUREBLOB_KEY"`
AzureBlobContainer string `name:"azure-blob-container" env:"UR_AZUREBLOB_CONTAINER"`
}
var (
@@ -77,6 +83,7 @@ var (
{regexp.MustCompile(`\ssyncthing@archlinux`), "Arch (3rd party)"},
{regexp.MustCompile(`@debian`), "Debian (3rd party)"},
{regexp.MustCompile(`@fedora`), "Fedora (3rd party)"},
{regexp.MustCompile(`@openSUSE`), "openSUSE (3rd party)"},
{regexp.MustCompile(`\sbrew@`), "Homebrew (3rd party)"},
{regexp.MustCompile(`\sroot@buildkitsandbox`), "LinuxServer.io (3rd party)"},
{regexp.MustCompile(`\sports@freebsd`), "FreeBSD (3rd party)"},
@@ -119,19 +126,25 @@ func (cli *CLI) Run() error {
go geo.Serve(context.TODO())
}
// s3
// Blob storage
var s3sess *s3.Session
var blobs blob.Store
if cli.S3Endpoint != "" {
s3sess, err = s3.NewSession(cli.S3Endpoint, cli.S3Region, cli.S3Bucket, cli.S3AccessKeyID, cli.S3SecretKey)
blobs, err = s3.NewSession(cli.S3Endpoint, cli.S3Region, cli.S3Bucket, cli.S3AccessKeyID, cli.S3SecretKey)
if err != nil {
slog.Error("Failed to create S3 session", "error", err)
return err
}
} else if cli.AzureBlobAccount != "" {
blobs, err = azureblob.NewBlobStore(cli.AzureBlobAccount, cli.AzureBlobKey, cli.AzureBlobContainer)
if err != nil {
slog.Error("Failed to create Azure blob store", "error", err)
return err
}
}
if _, err := os.Stat(cli.DumpFile); err != nil && s3sess != nil {
if err := cli.downloadDumpFile(s3sess); err != nil {
if _, err := os.Stat(cli.DumpFile); err != nil && blobs != nil {
if err := cli.downloadDumpFile(blobs); err != nil {
slog.Error("Failed to download dump file", "error", err)
}
}
@@ -153,7 +166,7 @@ func (cli *CLI) Run() error {
go func() {
for range time.Tick(cli.DumpInterval) {
if err := cli.saveDumpFile(srv, s3sess); err != nil {
if err := cli.saveDumpFile(srv, blobs); err != nil {
slog.Error("Failed to write dump file", "error", err)
}
}
@@ -192,8 +205,8 @@ func (cli *CLI) Run() error {
return metricsSrv.Serve(urListener)
}
func (cli *CLI) downloadDumpFile(s3sess *s3.Session) error {
latestKey, err := s3sess.LatestKey()
func (cli *CLI) downloadDumpFile(blobs blob.Store) error {
latestKey, err := blobs.LatestKey(context.Background())
if err != nil {
return fmt.Errorf("list latest S3 key: %w", err)
}
@@ -201,7 +214,7 @@ func (cli *CLI) downloadDumpFile(s3sess *s3.Session) error {
if err != nil {
return fmt.Errorf("create dump file: %w", err)
}
if err := s3sess.Download(fd, latestKey); err != nil {
if err := blobs.Download(context.Background(), latestKey, fd); err != nil {
_ = fd.Close()
return fmt.Errorf("download dump file: %w", err)
}
@@ -212,7 +225,7 @@ func (cli *CLI) downloadDumpFile(s3sess *s3.Session) error {
return nil
}
func (cli *CLI) saveDumpFile(srv *server, s3sess *s3.Session) error {
func (cli *CLI) saveDumpFile(srv *server, blobs blob.Store) error {
fd, err := os.Create(cli.DumpFile + ".tmp")
if err != nil {
return fmt.Errorf("creating dump file: %w", err)
@@ -233,13 +246,13 @@ func (cli *CLI) saveDumpFile(srv *server, s3sess *s3.Session) error {
}
slog.Info("Dump file saved")
if s3sess != nil {
if blobs != nil {
key := fmt.Sprintf("reports-%s.jsons.gz", time.Now().UTC().Format("2006-01-02"))
fd, err := os.Open(cli.DumpFile)
if err != nil {
return fmt.Errorf("opening dump file: %w", err)
}
if err := s3sess.Upload(fd, key); err != nil {
if err := blobs.Upload(context.Background(), key, fd); err != nil {
return fmt.Errorf("uploading dump file: %w", err)
}
_ = fd.Close()
@@ -351,6 +364,9 @@ func (s *server) addReport(rep *contract.Report) bool {
break
}
}
rep.DistDist = rep.Distribution
rep.DistOS = rep.OS
rep.DistArch = rep.Arch
_, loaded := s.reports.LoadAndStore(rep.UniqueID, rep)
return loaded
cmd/stdiscosrv/apisrv.go
+3 -3
View File
@@ -66,7 +66,7 @@ type contextKey int
const idKey contextKey = iota
func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP, compression bool) *apiSrv {
func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP, compression bool, desiredNotFoundRate float64) *apiSrv {
return &apiSrv{
addr: addr,
cert: cert,
@@ -77,13 +77,13 @@ func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator,
seenTracker: &retryAfterTracker{
name: "seenTracker",
bucketStarts: time.Now(),
desiredRate: 250,
desiredRate: desiredNotFoundRate / 2,
currentDelay: notFoundRetryUnknownMinSeconds,
},
notSeenTracker: &retryAfterTracker{
name: "notSeenTracker",
bucketStarts: time.Now(),
desiredRate: 250,
desiredRate: desiredNotFoundRate / 2,
currentDelay: notFoundRetryUnknownMaxSeconds / 2,
},
}
cmd/stdiscosrv/apisrv_test.go
+1 -1
View File
@@ -111,7 +111,7 @@ func BenchmarkAPIRequests(b *testing.B) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go db.Serve(ctx)
api := newAPISrv("127.0.0.1:0", tls.Certificate{}, db, nil, true, true)
api := newAPISrv("127.0.0.1:0", tls.Certificate{}, db, nil, true, true, 1000)
srv := httptest.NewServer(http.HandlerFunc(api.handler))
kf := b.TempDir() + "/cert"
cmd/stdiscosrv/database.go
+16 -16
View File
@@ -24,11 +24,11 @@ import (
"github.com/puzpuzpuz/xsync/v3"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/blob"
"github.com/syncthing/syncthing/internal/gen/discosrv"
"github.com/syncthing/syncthing/internal/protoutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/s3"
)
type clock interface {
@@ -51,12 +51,12 @@ type inMemoryStore struct {
m *xsync.MapOf[protocol.DeviceID, *discosrv.DatabaseRecord]
dir string
flushInterval time.Duration
s3 *s3.Session
blobs blob.Store
objKey string
clock clock
}
func newInMemoryStore(dir string, flushInterval time.Duration, s3sess *s3.Session) *inMemoryStore {
func newInMemoryStore(dir string, flushInterval time.Duration, blobs blob.Store) *inMemoryStore {
hn, err := os.Hostname()
if err != nil {
hn = rand.String(8)
@@ -65,25 +65,25 @@ func newInMemoryStore(dir string, flushInterval time.Duration, s3sess *s3.Sessio
m: xsync.NewMapOf[protocol.DeviceID, *discosrv.DatabaseRecord](),
dir: dir,
flushInterval: flushInterval,
s3: s3sess,
blobs: blobs,
objKey: hn + ".db",
clock: defaultClock{},
}
nr, err := s.read()
if os.IsNotExist(err) && s3sess != nil {
// Try to read from AWS
latestKey, cerr := s3sess.LatestKey()
if os.IsNotExist(err) && blobs != nil {
// Try to read from blob storage
latestKey, cerr := blobs.LatestKey(context.Background())
if cerr != nil {
log.Println("Error reading database from S3:", err)
log.Println("Error finding database from blob storage:", cerr)
return s
}
fd, cerr := os.Create(path.Join(s.dir, "records.db"))
if cerr != nil {
log.Println("Error creating database file:", err)
log.Println("Error creating database file:", cerr)
return s
}
if cerr := s3sess.Download(fd, latestKey); cerr != nil {
log.Printf("Error reading database from S3: %v", err)
if cerr := blobs.Download(context.Background(), latestKey, fd); cerr != nil {
log.Printf("Error downloading database from blob storage: %v", cerr)
}
_ = fd.Close()
nr, err = s.read()
@@ -310,16 +310,16 @@ func (s *inMemoryStore) write() (err error) {
return err
}
// Upload to S3
if s.s3 != nil {
// Upload to blob storage
if s.blobs != nil {
fd, err = os.Open(dbf)
if err != nil {
log.Printf("Error uploading database to S3: %v", err)
log.Printf("Error uploading database to blob storage: %v", err)
return nil
}
defer fd.Close()
if err := s.s3.Upload(fd, s.objKey); err != nil {
log.Printf("Error uploading database to S3: %v", err)
if err := s.blobs.Upload(context.Background(), s.objKey, fd); err != nil {
log.Printf("Error uploading database to blob storage: %v", err)
}
log.Println("Finished uploading database")
}
cmd/stdiscosrv/main.go
+25 -16
View File
@@ -21,11 +21,13 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/internal/blob"
"github.com/syncthing/syncthing/internal/blob/azureblob"
"github.com/syncthing/syncthing/internal/blob/s3"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/s3"
"github.com/syncthing/syncthing/lib/tlsutil"
)
@@ -58,12 +60,13 @@ const (
var debug = false
type CLI struct {
Cert string `group:"Listen" help:"Certificate file" default:"./cert.pem" env:"DISCOVERY_CERT_FILE"`
Key string `group:"Listen" help:"Key file" default:"./key.pem" env:"DISCOVERY_KEY_FILE"`
HTTP bool `group:"Listen" help:"Listen on HTTP (behind an HTTPS proxy)" env:"DISCOVERY_HTTP"`
Compression bool `group:"Listen" help:"Enable GZIP compression of responses" env:"DISCOVERY_COMPRESSION"`
Listen string `group:"Listen" help:"Listen address" default:":8443" env:"DISCOVERY_LISTEN"`
MetricsListen string `group:"Listen" help:"Metrics listen address" env:"DISCOVERY_METRICS_LISTEN"`
Cert string `group:"Listen" help:"Certificate file" default:"./cert.pem" env:"DISCOVERY_CERT_FILE"`
Key string `group:"Listen" help:"Key file" default:"./key.pem" env:"DISCOVERY_KEY_FILE"`
HTTP bool `group:"Listen" help:"Listen on HTTP (behind an HTTPS proxy)" env:"DISCOVERY_HTTP"`
Compression bool `group:"Listen" help:"Enable GZIP compression of responses" env:"DISCOVERY_COMPRESSION"`
Listen string `group:"Listen" help:"Listen address" default:":8443" env:"DISCOVERY_LISTEN"`
MetricsListen string `group:"Listen" help:"Metrics listen address" env:"DISCOVERY_METRICS_LISTEN"`
DesiredNotFoundRate float64 `group:"Listen" help:"Desired maximum rate of not-found replies (/s)" default:"1000"`
DBDir string `group:"Database" help:"Database directory" default:"." env:"DISCOVERY_DB_DIR"`
DBFlushInterval time.Duration `group:"Database" help:"Interval between database flushes" default:"5m" env:"DISCOVERY_DB_FLUSH_INTERVAL"`
@@ -74,6 +77,10 @@ type CLI struct {
DBS3AccessKeyID string `name:"db-s3-access-key-id" group:"Database (S3 backup)" hidden:"true" help:"S3 access key ID for database" env:"DISCOVERY_DB_S3_ACCESS_KEY_ID"`
DBS3SecretKey string `name:"db-s3-secret-key" group:"Database (S3 backup)" hidden:"true" help:"S3 secret key for database" env:"DISCOVERY_DB_S3_SECRET_KEY"`
DBAzureBlobAccount string `name:"db-azure-blob-account" env:"DISCOVERY_DB_AZUREBLOB_ACCOUNT"`
DBAzureBlobKey string `name:"db-azure-blob-key" env:"DISCOVERY_DB_AZUREBLOB_KEY"`
DBAzureBlobContainer string `name:"db-azure-blob-container" env:"DISCOVERY_DB_AZUREBLOB_CONTAINER"`
AMQPAddress string `group:"AMQP replication" hidden:"true" help:"Address to AMQP broker" env:"DISCOVERY_AMQP_ADDRESS"`
Debug bool `short:"d" help:"Print debug output" env:"DISCOVERY_DEBUG"`
@@ -117,18 +124,20 @@ func main() {
Timeout: 2 * time.Minute,
})
// If configured, use S3 for database backups.
var s3c *s3.Session
// If configured, use blob storage for database backups.
var blobs blob.Store
var err error
if cli.DBS3Endpoint != "" {
var err error
s3c, err = s3.NewSession(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
if err != nil {
log.Fatalf("Failed to create S3 session: %v", err)
}
blobs, err = s3.NewSession(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
} else if cli.DBAzureBlobAccount != "" {
blobs, err = azureblob.NewBlobStore(cli.DBAzureBlobAccount, cli.DBAzureBlobKey, cli.DBAzureBlobContainer)
}
if err != nil {
log.Fatalf("Failed to create blob store: %v", err)
}
// Start the database.
db := newInMemoryStore(cli.DBDir, cli.DBFlushInterval, s3c)
db := newInMemoryStore(cli.DBDir, cli.DBFlushInterval, blobs)
main.Add(db)
// If we have an AMQP broker for replication, start that
@@ -141,7 +150,7 @@ func main() {
}
// Start the main API server.
qs := newAPISrv(cli.Listen, cert, db, repl, cli.HTTP, cli.Compression)
qs := newAPISrv(cli.Listen, cert, db, repl, cli.HTTP, cli.Compression, cli.DesiredNotFoundRate)
main.Add(qs)
// If we have a metrics port configured, start a metrics handler.