felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

Merge branch 'infrastructure'

Browse Source 
* infrastructure:
  build: let infra containers builds fail individually
  chore(ur): move structs to reduce dependency chain
  chore(stcrashreceiver): add profiler on metrics port
  chore(stcrashreceiver): compact diskstore in-memory representation
  chore(stcrashreceiver): better source cache & metrics
  chore(stcrashreceiver): metrics on ignore matches
This commit is contained in:
Jakob Borg
2026-05-23 09:36:25 +02:00
parent 8a3a06f7ca 05b4f6abda
commit 49a2688caa
9 changed files with 92 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-infra-dockers.yaml
+1
View File
@@ -22,6 +22,7 @@ jobs:
if: github.repository_owner == 'syncthing'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
pkg:
- stcrashreceiver
cmd/infra/stcrashreceiver/diskstore.go
+14 -4
View File
@@ -136,15 +136,25 @@ func (d *diskStore) Exists(path string) bool {
}
func (d *diskStore) clean() {
for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
f := d.currentFiles[0]
numDeleted := 0
for idx := range d.currentFiles {
if len(d.currentFiles)-numDeleted < d.maxFiles && d.currentSize < d.maxBytes {
break
}
f := d.currentFiles[idx]
log.Println("Removing", f.path)
if err := os.Remove(f.path); err != nil {
log.Println("Failed to remove file:", err)
}
d.currentFiles = d.currentFiles[1:]
d.currentSize -= f.size
numDeleted = idx + 1
}
// Compact currentFiles
copy(d.currentFiles, d.currentFiles[numDeleted:])
d.currentFiles = d.currentFiles[:len(d.currentFiles)-numDeleted]
var oldest time.Duration
if len(d.currentFiles) > 0 {
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
@@ -158,7 +168,7 @@ func (d *diskStore) clean() {
}
func (d *diskStore) inventory() error {
d.currentFiles = nil
d.currentFiles = d.currentFiles[:0]
d.currentSize = 0
err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
cmd/infra/stcrashreceiver/main.go
+7 -4
View File
@@ -20,6 +20,7 @@ import (
"io"
"log"
"net/http"
"net/http/pprof"
"os"
"path/filepath"
"regexp"
@@ -29,7 +30,7 @@ import (
raven "github.com/getsentry/raven-go"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/ur"
"github.com/syncthing/syncthing/lib/ur/contract"
)
const maxRequestSize = 1 << 20 // 1 MiB
@@ -89,6 +90,7 @@ func main() {
if params.MetricsListen != "" {
mmux := http.NewServeMux()
mmux.Handle("/metrics", promhttp.Handler())
mmux.HandleFunc("/debug/pprof/", pprof.Index)
go func() {
if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil {
log.Fatalln("HTTP serve metrics:", err)
@@ -123,12 +125,13 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
return
}
if _, ok := ignore.match(bs); ok {
if pat, ok := ignore.match(bs); ok {
metricIgnoreMatchesTotal.WithLabelValues(pat).Inc()
result = "ignored"
return
}
var reports []ur.FailureReport
var reports []contract.FailureReport
err = json.Unmarshal(bs, &reports)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
@@ -176,7 +179,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
}
}
func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
func saveFailureWithGoroutines(data contract.FailureData, failureDir string) (string, error) {
bs := make([]byte, len(data.Description)+len(data.Goroutines))
copy(bs, data.Description)
copy(bs[len(data.Description):], data.Goroutines)
cmd/infra/stcrashreceiver/metrics.go
+15
View File
@@ -42,4 +42,19 @@ var (
Subsystem: "crashreceiver",
Name: "sentry_reports_total",
}, []string{"result"})
metricIgnoreMatchesTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "ignore_matches_total",
}, []string{"pattern"})
metricSourceCodeLoadsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "source_code_loads_total",
}, []string{"result"})
metricSourceCodeCacheSize = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "source_code_cache_size",
})
)
cmd/infra/stcrashreceiver/sourcecodeloader.go
+25 -10
View File
@@ -15,23 +15,33 @@ import (
"strings"
"sync"
"time"
lru "github.com/hashicorp/golang-lru/v2"
)
const (
urlPrefix = "https://raw.githubusercontent.com/syncthing/syncthing/"
httpTimeout = 10 * time.Second
urlPrefix = "https://raw.githubusercontent.com/syncthing/syncthing/"
httpTimeout = 10 * time.Second
maxCacheEntries = 1000
)
type cacheKey struct {
version string
file string
}
type githubSourceCodeLoader struct {
mut sync.Mutex
version string
cache map[string]map[string][][]byte // version -> file -> lines
client *http.Client
cache *lru.TwoQueueCache[cacheKey, [][]byte] // version & file -> lines
client *http.Client
}
func newGithubSourceCodeLoader() *githubSourceCodeLoader {
cache, _ := lru.New2Q[cacheKey, [][]byte](maxCacheEntries)
return &githubSourceCodeLoader{
cache: make(map[string]map[string][][]byte),
cache: cache,
client: &http.Client{Timeout: httpTimeout},
}
}
@@ -39,9 +49,6 @@ func newGithubSourceCodeLoader() *githubSourceCodeLoader {
func (l *githubSourceCodeLoader) LockWithVersion(version string) {
l.mut.Lock()
l.version = version
if _, ok := l.cache[version]; !ok {
l.cache[version] = make(map[string][][]byte)
}
}
func (l *githubSourceCodeLoader) Unlock() {
@@ -50,11 +57,13 @@ func (l *githubSourceCodeLoader) Unlock() {
func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
filename = filepath.ToSlash(filename)
lines, ok := l.cache[l.version][filename]
key := cacheKey{version: l.version, file: filename}
lines, ok := l.cache.Get(key)
if !ok {
// Cache whatever we managed to find (or nil if nothing, so we don't try again)
defer func() {
l.cache[l.version][filename] = lines
l.cache.Add(key, lines)
metricSourceCodeCacheSize.Set(float64(l.cache.Len()))
}()
knownPrefixes := []string{"/lib/", "/cmd/"}
@@ -73,19 +82,25 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
resp, err := l.client.Get(url)
if err != nil {
fmt.Println("Loading source:", err)
metricSourceCodeLoadsTotal.WithLabelValues("failed").Inc()
return nil, 0
}
if resp.StatusCode != http.StatusOK {
fmt.Println("Loading source:", resp.Status)
metricSourceCodeLoadsTotal.WithLabelValues("failed").Inc()
return nil, 0
}
data, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
fmt.Println("Loading source:", err.Error())
metricSourceCodeLoadsTotal.WithLabelValues("failed").Inc()
return nil, 0
}
lines = bytes.Split(data, []byte{'\n'})
metricSourceCodeLoadsTotal.WithLabelValues("loaded").Inc()
} else {
metricSourceCodeLoadsTotal.WithLabelValues("cached").Inc()
}
return getLineFromLines(lines, line, context)
cmd/infra/stcrashreceiver/stcrashreceiver.go
+1
View File
@@ -90,6 +90,7 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
first := string(bytes.TrimSpace(bytes.Split(bs, []byte("\n"))[0]))
if pat, ok := r.ignore.match(bs); ok {
metricIgnoreMatchesTotal.WithLabelValues(pat).Inc()
result = "ignored"
log.Printf("Ignored report %s, matched: %s (%s)", reportID[:8], pat, first)
return
lib/model/indexhandler.go
+2 -2
View File
@@ -20,7 +20,7 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/ur"
"github.com/syncthing/syncthing/lib/ur/contract"
)
type indexHandler struct {
@@ -470,7 +470,7 @@ func (s *indexHandler) logSequenceAnomaly(msg string, extra map[string]any) {
extraStrs[k] = fmt.Sprint(v)
}
s.evLogger.Log(events.Failure, ur.FailureData{
s.evLogger.Log(events.Failure, contract.FailureData{
Description: msg,
Extra: extraStrs,
})
lib/ur/contract/contract.go
+13
View File
@@ -282,3 +282,16 @@ func clear(v interface{}, since int) error {
}
return nil
}
type FailureReport struct {
FailureData
Count int
Version string
}
type FailureData struct {
Description string
Goroutines string
Extra map[string]string
}
lib/ur/failurereporting.go
+14 -26
View File
@@ -23,6 +23,7 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/thejerf/suture/v4"
)
@@ -39,23 +40,10 @@ var (
invalidEventDataType = "failure event data is not a string"
)
type FailureReport struct {
FailureData
Count int
Version string
}
type FailureData struct {
Description string
Goroutines string
Extra map[string]string
}
func FailureDataWithGoroutines(description string) FailureData {
func FailureDataWithGoroutines(description string) contract.FailureData {
var buf strings.Builder
pprof.Lookup("goroutine").WriteTo(&buf, 1)
return FailureData{
return contract.FailureData{
Description: description,
Goroutines: buf.String(),
Extra: make(map[string]string),
@@ -86,7 +74,7 @@ type failureHandler struct {
type failureStat struct {
first, last time.Time
count int
data FailureData
data contract.FailureData
}
func (h *failureHandler) Serve(ctx context.Context) error {
@@ -105,24 +93,24 @@ func (h *failureHandler) Serve(ctx context.Context) error {
if !ok {
// Just to be safe - shouldn't ever happen, as
// evChan is set to nil when unsubscribing.
h.addReport(FailureData{Description: evChanClosed}, time.Now())
h.addReport(contract.FailureData{Description: evChanClosed}, time.Now())
evChan = nil
continue
}
var data FailureData
var data contract.FailureData
switch d := e.Data.(type) {
case string:
data.Description = d
case FailureData:
case contract.FailureData:
data = d
default:
// Same here, shouldn't ever happen.
h.addReport(FailureData{Description: invalidEventDataType}, time.Now())
h.addReport(contract.FailureData{Description: invalidEventDataType}, time.Now())
continue
}
h.addReport(data, e.Time)
case <-timer.C:
reports := make([]FailureReport, 0, len(h.buf))
reports := make([]contract.FailureReport, 0, len(h.buf))
now := time.Now()
for descr, stat := range h.buf {
if now.Sub(stat.last) > minDelay || now.Sub(stat.first) > maxDelay {
@@ -152,7 +140,7 @@ func (h *failureHandler) Serve(ctx context.Context) error {
if sub != nil {
sub.Unsubscribe()
if len(h.buf) > 0 {
reports := make([]FailureReport, 0, len(h.buf))
reports := make([]contract.FailureReport, 0, len(h.buf))
for _, stat := range h.buf {
reports = append(reports, newFailureReport(stat))
}
@@ -179,7 +167,7 @@ func (h *failureHandler) applyOpts(opts config.OptionsConfiguration, sub events.
return url, nil, nil
}
func (h *failureHandler) addReport(data FailureData, evTime time.Time) {
func (h *failureHandler) addReport(data contract.FailureData, evTime time.Time) {
if stat, ok := h.buf[data.Description]; ok {
stat.last = evTime
stat.count++
@@ -204,7 +192,7 @@ func (*failureHandler) String() string {
return "FailureHandler"
}
func sendFailureReports(ctx context.Context, reports []FailureReport, url string) {
func sendFailureReports(ctx context.Context, reports []contract.FailureReport, url string) {
var b bytes.Buffer
if err := json.NewEncoder(&b).Encode(reports); err != nil {
panic(err)
@@ -235,8 +223,8 @@ func sendFailureReports(ctx context.Context, reports []FailureReport, url string
resp.Body.Close()
}
func newFailureReport(stat *failureStat) FailureReport {
return FailureReport{
func newFailureReport(stat *failureStat) contract.FailureReport {
return contract.FailureReport{
FailureData: stat.data,
Count: stat.count,
Version: build.LongVersion,