chore(cmd): clean up commands (#9705)
Move infrastructure related commands to under `cmd/infra` and development stuff to `cmd/dev`. The default build command builds the regular user facing binaries: syncthing, stdiscosrv, and strelaysrv.
This commit is contained in:
Jakob Borg
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,198 @@
|
||||
// Copyright (C) 2023 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type diskStore struct {
|
||||
dir string
|
||||
inbox chan diskEntry
|
||||
maxBytes int64
|
||||
maxFiles int
|
||||
|
||||
currentFiles []currentFile
|
||||
currentSize int64
|
||||
}
|
||||
|
||||
type diskEntry struct {
|
||||
path string
|
||||
data []byte
|
||||
}
|
||||
|
||||
type currentFile struct {
|
||||
path string
|
||||
size int64
|
||||
mtime int64
|
||||
}
|
||||
|
||||
func (d *diskStore) Serve(ctx context.Context) {
|
||||
if err := os.MkdirAll(d.dir, 0o700); err != nil {
|
||||
log.Println("Creating directory:", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := d.inventory(); err != nil {
|
||||
log.Println("Failed to inventory disk store:", err)
|
||||
}
|
||||
d.clean()
|
||||
|
||||
cleanTimer := time.NewTicker(time.Minute)
|
||||
inventoryTimer := time.NewTicker(24 * time.Hour)
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
gw := gzip.NewWriter(buf)
|
||||
for {
|
||||
select {
|
||||
case entry := <-d.inbox:
|
||||
path := d.fullPath(entry.path)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
log.Println("Creating directory:", err)
|
||||
continue
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
gw.Reset(buf)
|
||||
if _, err := gw.Write(entry.data); err != nil {
|
||||
log.Println("Failed to compress crash report:", err)
|
||||
continue
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
log.Println("Failed to compress crash report:", err)
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
|
||||
log.Printf("Failed to write %s: %v", entry.path, err)
|
||||
_ = os.Remove(path)
|
||||
continue
|
||||
}
|
||||
|
||||
d.currentSize += int64(buf.Len())
|
||||
d.currentFiles = append(d.currentFiles, currentFile{
|
||||
size: int64(len(entry.data)),
|
||||
path: path,
|
||||
})
|
||||
|
||||
case <-cleanTimer.C:
|
||||
d.clean()
|
||||
|
||||
case <-inventoryTimer.C:
|
||||
if err := d.inventory(); err != nil {
|
||||
log.Println("Failed to inventory disk store:", err)
|
||||
}
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *diskStore) Put(path string, data []byte) bool {
|
||||
select {
|
||||
case d.inbox <- diskEntry{
|
||||
path: path,
|
||||
data: data,
|
||||
}:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (d *diskStore) Get(path string) ([]byte, error) {
|
||||
path = d.fullPath(path)
|
||||
bs, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gr, err := gzip.NewReader(bytes.NewReader(bs))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer gr.Close()
|
||||
return io.ReadAll(gr)
|
||||
}
|
||||
|
||||
func (d *diskStore) Exists(path string) bool {
|
||||
path = d.fullPath(path)
|
||||
_, err := os.Lstat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (d *diskStore) clean() {
|
||||
for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
|
||||
f := d.currentFiles[0]
|
||||
log.Println("Removing", f.path)
|
||||
if err := os.Remove(f.path); err != nil {
|
||||
log.Println("Failed to remove file:", err)
|
||||
}
|
||||
d.currentFiles = d.currentFiles[1:]
|
||||
d.currentSize -= f.size
|
||||
}
|
||||
var oldest time.Duration
|
||||
if len(d.currentFiles) > 0 {
|
||||
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
|
||||
}
|
||||
|
||||
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
|
||||
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
|
||||
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
|
||||
|
||||
log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
|
||||
}
|
||||
|
||||
func (d *diskStore) inventory() error {
|
||||
d.currentFiles = nil
|
||||
d.currentSize = 0
|
||||
err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if filepath.Ext(path) != ".gz" {
|
||||
return nil
|
||||
}
|
||||
d.currentSize += info.Size()
|
||||
d.currentFiles = append(d.currentFiles, currentFile{
|
||||
path: path,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime().Unix(),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
sort.Slice(d.currentFiles, func(i, j int) bool {
|
||||
return d.currentFiles[i].mtime < d.currentFiles[j].mtime
|
||||
})
|
||||
var oldest time.Duration
|
||||
if len(d.currentFiles) > 0 {
|
||||
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
|
||||
}
|
||||
|
||||
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
|
||||
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
|
||||
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
|
||||
|
||||
log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *diskStore) fullPath(path string) string {
|
||||
return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
// Copyright (C) 2019 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Command stcrashreceiver is a trivial HTTP server that allows two things:
|
||||
//
|
||||
// - uploading files (crash reports) named like a SHA256 hash using a PUT request
|
||||
// - checking whether such file exists using a HEAD request
|
||||
//
|
||||
// Typically this should be deployed behind something that manages HTTPS.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
raven "github.com/getsentry/raven-go"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/ur"
|
||||
)
|
||||
|
||||
const maxRequestSize = 1 << 20 // 1 MiB
|
||||
|
||||
type cli struct {
|
||||
Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
|
||||
DSN string `help:"Sentry DSN" env:"SENTRY_DSN"`
|
||||
Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
|
||||
MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
|
||||
MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
|
||||
SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
|
||||
DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
|
||||
MetricsListen string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"`
|
||||
IngorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var params cli
|
||||
kong.Parse(¶ms)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
ds := &diskStore{
|
||||
dir: filepath.Join(params.Dir, "crash_reports"),
|
||||
inbox: make(chan diskEntry, params.DiskQueue),
|
||||
maxFiles: params.MaxDiskFiles,
|
||||
maxBytes: params.MaxDiskSizeMB << 20,
|
||||
}
|
||||
go ds.Serve(context.Background())
|
||||
|
||||
ss := &sentryService{
|
||||
dsn: params.DSN,
|
||||
inbox: make(chan sentryRequest, params.SentryQueue),
|
||||
}
|
||||
go ss.Serve(context.Background())
|
||||
|
||||
var ip *ignorePatterns
|
||||
if params.IngorePatterns != "" {
|
||||
var err error
|
||||
ip, err = loadIgnorePatterns(params.IngorePatterns)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load ignore patterns: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
cr := &crashReceiver{
|
||||
store: ds,
|
||||
sentry: ss,
|
||||
ignore: ip,
|
||||
}
|
||||
|
||||
mux.Handle("/", cr)
|
||||
mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
|
||||
w.Write([]byte("OK"))
|
||||
})
|
||||
|
||||
if params.MetricsListen != "" {
|
||||
mmux := http.NewServeMux()
|
||||
mmux.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil {
|
||||
log.Fatalln("HTTP serve metrics:", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if params.DSN != "" {
|
||||
mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"), ip))
|
||||
}
|
||||
|
||||
log.SetOutput(os.Stdout)
|
||||
if err := http.ListenAndServe(params.Listen, mux); err != nil {
|
||||
log.Fatalln("HTTP serve:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http.ResponseWriter, req *http.Request) {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
result := "failure"
|
||||
defer func() {
|
||||
metricFailureReportsTotal.WithLabelValues(result).Inc()
|
||||
}()
|
||||
|
||||
lr := io.LimitReader(req.Body, maxRequestSize)
|
||||
bs, err := io.ReadAll(lr)
|
||||
req.Body.Close()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
|
||||
if ignore.match(bs) {
|
||||
result = "ignored"
|
||||
return
|
||||
}
|
||||
|
||||
var reports []ur.FailureReport
|
||||
err = json.Unmarshal(bs, &reports)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
}
|
||||
if len(reports) == 0 {
|
||||
// Shouldn't happen
|
||||
log.Printf("Got zero failure reports")
|
||||
return
|
||||
}
|
||||
|
||||
version, err := build.ParseVersion(reports[0].Version)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
}
|
||||
for _, r := range reports {
|
||||
pkt := packet(version, "failure")
|
||||
pkt.Message = r.Description
|
||||
pkt.Extra = raven.Extra{
|
||||
"count": r.Count,
|
||||
}
|
||||
for k, v := range r.Extra {
|
||||
pkt.Extra[k] = v
|
||||
}
|
||||
if r.Goroutines != "" {
|
||||
url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
|
||||
if err != nil {
|
||||
log.Println("Saving failure report:", err)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
pkt.Extra["goroutinesURL"] = url
|
||||
}
|
||||
message := sanitizeMessageLDB(r.Description)
|
||||
pkt.Fingerprint = []string{message}
|
||||
|
||||
if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
|
||||
log.Println("Failed to send failure report:", err)
|
||||
} else {
|
||||
log.Println("Sent failure report:", r.Description)
|
||||
result = "success"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
|
||||
bs := make([]byte, len(data.Description)+len(data.Goroutines))
|
||||
copy(bs, data.Description)
|
||||
copy(bs[len(data.Description):], data.Goroutines)
|
||||
id := fmt.Sprintf("%x", sha256.Sum256(bs))
|
||||
path := fullPathCompressed(failureDir, id)
|
||||
err := compressAndWrite(bs, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return reportServer + path, nil
|
||||
}
|
||||
|
||||
type ignorePatterns struct {
|
||||
patterns []*regexp.Regexp
|
||||
}
|
||||
|
||||
func loadIgnorePatterns(path string) (*ignorePatterns, error) {
|
||||
bs, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var patterns []*regexp.Regexp
|
||||
for _, line := range strings.Split(string(bs), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
re, err := regexp.Compile(line)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
patterns = append(patterns, re)
|
||||
}
|
||||
|
||||
log.Printf("Loaded %d ignore patterns", len(patterns))
|
||||
return &ignorePatterns{patterns: patterns}, nil
|
||||
}
|
||||
|
||||
func (i *ignorePatterns) match(report []byte) bool {
|
||||
if i == nil {
|
||||
return false
|
||||
}
|
||||
for _, re := range i.patterns {
|
||||
if re.Match(report) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright (C) 2023 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
metricCrashReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "crash_reports_total",
|
||||
}, []string{"result"})
|
||||
metricFailureReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "failure_reports_total",
|
||||
}, []string{"result"})
|
||||
metricDiskstoreFilesTotal = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_files_total",
|
||||
})
|
||||
metricDiskstoreBytesTotal = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_bytes_total",
|
||||
})
|
||||
metricDiskstoreOldestAgeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_oldest_age_seconds",
|
||||
})
|
||||
)
|
||||
@@ -0,0 +1,242 @@
|
||||
// Copyright (C) 2019 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
raven "github.com/getsentry/raven-go"
|
||||
"github.com/maruel/panicparse/v2/stack"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
)
|
||||
|
||||
const reportServer = "https://crash.syncthing.net/report/"
|
||||
|
||||
var loader = newGithubSourceCodeLoader()
|
||||
|
||||
func init() {
|
||||
raven.SetSourceCodeLoader(loader)
|
||||
}
|
||||
|
||||
var (
|
||||
clients = make(map[string]*raven.Client)
|
||||
clientsMut sync.Mutex
|
||||
)
|
||||
|
||||
type sentryService struct {
|
||||
dsn string
|
||||
inbox chan sentryRequest
|
||||
}
|
||||
|
||||
type sentryRequest struct {
|
||||
reportID string
|
||||
userID string
|
||||
data []byte
|
||||
}
|
||||
|
||||
func (s *sentryService) Serve(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case req := <-s.inbox:
|
||||
pkt, err := parseCrashReport(req.reportID, req.data)
|
||||
if err != nil {
|
||||
log.Println("Failed to parse crash report:", err)
|
||||
continue
|
||||
}
|
||||
if err := sendReport(s.dsn, pkt, req.userID); err != nil {
|
||||
log.Println("Failed to send crash report:", err)
|
||||
}
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sentryService) Send(reportID, userID string, data []byte) bool {
|
||||
select {
|
||||
case s.inbox <- sentryRequest{reportID, userID, data}:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func sendReport(dsn string, pkt *raven.Packet, userID string) error {
|
||||
pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
|
||||
|
||||
clientsMut.Lock()
|
||||
defer clientsMut.Unlock()
|
||||
|
||||
cli, ok := clients[dsn]
|
||||
if !ok {
|
||||
var err error
|
||||
cli, err = raven.New(dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clients[dsn] = cli
|
||||
}
|
||||
|
||||
// The client sets release and such on the packet before sending, in the
|
||||
// misguided idea that it knows this better than than the packet we give
|
||||
// it. So we copy the values from the packet to the client first...
|
||||
cli.SetRelease(pkt.Release)
|
||||
cli.SetEnvironment(pkt.Environment)
|
||||
|
||||
defer cli.Wait()
|
||||
_, errC := cli.Capture(pkt, nil)
|
||||
return <-errC
|
||||
}
|
||||
|
||||
func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
|
||||
parts := bytes.SplitN(report, []byte("\n"), 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, errors.New("no first line")
|
||||
}
|
||||
|
||||
version, err := build.ParseVersion(string(parts[0]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
report = parts[1]
|
||||
|
||||
foundPanic := false
|
||||
var subjectLine []byte
|
||||
for {
|
||||
parts = bytes.SplitN(report, []byte("\n"), 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, errors.New("no panic line found")
|
||||
}
|
||||
|
||||
line := parts[0]
|
||||
report = parts[1]
|
||||
|
||||
if foundPanic {
|
||||
// The previous line was our "Panic at ..." header. We are now
|
||||
// at the beginning of the real panic trace and this is our
|
||||
// subject line.
|
||||
subjectLine = line
|
||||
break
|
||||
} else if bytes.HasPrefix(line, []byte("Panic at")) {
|
||||
foundPanic = true
|
||||
}
|
||||
}
|
||||
|
||||
r := bytes.NewReader(report)
|
||||
ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
if ctx == nil || len(ctx.Goroutines) == 0 {
|
||||
return nil, errors.New("no goroutines found")
|
||||
}
|
||||
|
||||
// Lock the source code loader to the version we are processing here.
|
||||
if version.Commit != "" {
|
||||
// We have a commit hash, so we know exactly which source to use
|
||||
loader.LockWithVersion(version.Commit)
|
||||
} else if strings.HasPrefix(version.Tag, "v") {
|
||||
// Lets hope the tag is close enough
|
||||
loader.LockWithVersion(version.Tag)
|
||||
} else {
|
||||
// Last resort
|
||||
loader.LockWithVersion("main")
|
||||
}
|
||||
defer loader.Unlock()
|
||||
|
||||
var trace raven.Stacktrace
|
||||
for _, gr := range ctx.Goroutines {
|
||||
if gr.First {
|
||||
trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
|
||||
for i, sc := range gr.Stack.Calls {
|
||||
trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name, sc.RemoteSrcPath, sc.Line, 3, nil)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
pkt := packet(version, "crash")
|
||||
pkt.Message = string(subjectLine)
|
||||
pkt.Extra = raven.Extra{
|
||||
"url": reportServer + path,
|
||||
}
|
||||
pkt.Interfaces = []raven.Interface{&trace}
|
||||
pkt.Fingerprint = crashReportFingerprint(pkt.Message)
|
||||
|
||||
return pkt, nil
|
||||
}
|
||||
|
||||
var (
|
||||
indexRe = regexp.MustCompile(`\[[-:0-9]+\]`)
|
||||
sizeRe = regexp.MustCompile(`(length|capacity) [0-9]+`)
|
||||
ldbPosRe = regexp.MustCompile(`(\(pos=)([0-9]+)\)`)
|
||||
ldbChecksumRe = regexp.MustCompile(`(want=0x)([a-z0-9]+)( got=0x)([a-z0-9]+)`)
|
||||
ldbFileRe = regexp.MustCompile(`(\[file=)([0-9]+)(\.ldb\])`)
|
||||
ldbInternalKeyRe = regexp.MustCompile(`(internal key ")[^"]+(", len=)[0-9]+`)
|
||||
ldbPathRe = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
|
||||
)
|
||||
|
||||
func sanitizeMessageLDB(message string) string {
|
||||
message = ldbPosRe.ReplaceAllString(message, "${1}x)")
|
||||
message = ldbFileRe.ReplaceAllString(message, "${1}x${3}")
|
||||
message = ldbChecksumRe.ReplaceAllString(message, "${1}X${3}X")
|
||||
message = ldbInternalKeyRe.ReplaceAllString(message, "${1}x${2}x")
|
||||
message = ldbPathRe.ReplaceAllString(message, "$1 x: ")
|
||||
return message
|
||||
}
|
||||
|
||||
func crashReportFingerprint(message string) []string {
|
||||
// Do not fingerprint on the stack in case of db corruption or fatal
|
||||
// db io error - where it occurs doesn't matter.
|
||||
orig := message
|
||||
message = sanitizeMessageLDB(message)
|
||||
if message != orig {
|
||||
return []string{message}
|
||||
}
|
||||
|
||||
message = indexRe.ReplaceAllString(message, "[x]")
|
||||
message = sizeRe.ReplaceAllString(message, "$1 x")
|
||||
|
||||
// {{ default }} is what sentry uses as a fingerprint by default. While
|
||||
// never specified, the docs point at this being some hash derived from the
|
||||
// stack trace. Here we include the filtered panic message on top of that.
|
||||
// https://docs.sentry.io/platforms/go/data-management/event-grouping/sdk-fingerprinting/#basic-example
|
||||
return []string{"{{ default }}", message}
|
||||
}
|
||||
|
||||
func packet(version build.VersionParts, reportType string) *raven.Packet {
|
||||
pkt := &raven.Packet{
|
||||
Platform: "go",
|
||||
Release: version.Tag,
|
||||
Environment: version.Environment(),
|
||||
Tags: raven.Tags{
|
||||
raven.Tag{Key: "version", Value: version.Version},
|
||||
raven.Tag{Key: "tag", Value: version.Tag},
|
||||
raven.Tag{Key: "codename", Value: version.Codename},
|
||||
raven.Tag{Key: "runtime", Value: version.Runtime},
|
||||
raven.Tag{Key: "goos", Value: version.GOOS},
|
||||
raven.Tag{Key: "goarch", Value: version.GOARCH},
|
||||
raven.Tag{Key: "builder", Value: version.Builder},
|
||||
raven.Tag{Key: "report_type", Value: reportType},
|
||||
},
|
||||
}
|
||||
if version.Commit != "" {
|
||||
pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.Commit})
|
||||
}
|
||||
for _, tag := range version.Extra {
|
||||
pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
|
||||
}
|
||||
return pkt
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Copyright (C) 2019 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseReport(t *testing.T) {
|
||||
bs, err := os.ReadFile("_testdata/panic.log")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
pkt, err := parseCrashReport("1/2/345", bs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bs, err = pkt.JSON()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("%s\n", bs)
|
||||
}
|
||||
|
||||
func TestCrashReportFingerprint(t *testing.T) {
|
||||
cases := []struct {
|
||||
message, exp string
|
||||
ldb bool
|
||||
}{
|
||||
{
|
||||
message: "panic: leveldb/table: corruption on data-block (pos=51308946): checksum mismatch, want=0xa89f9aa0 got=0xd27cc4c7 [file=004003.ldb]",
|
||||
exp: "panic: leveldb/table: corruption on data-block (pos=x): checksum mismatch, want=0xX got=0xX [file=x.ldb]",
|
||||
ldb: true,
|
||||
},
|
||||
{
|
||||
message: "panic: leveldb/table: corruption on table-footer (pos=248): bad magic number [file=001370.ldb]",
|
||||
exp: "panic: leveldb/table: corruption on table-footer (pos=x): bad magic number [file=x.ldb]",
|
||||
ldb: true,
|
||||
},
|
||||
{
|
||||
message: "panic: runtime error: slice bounds out of range [4294967283:4194304]",
|
||||
exp: "panic: runtime error: slice bounds out of range [x]",
|
||||
},
|
||||
{
|
||||
message: "panic: runtime error: slice bounds out of range [-2:]",
|
||||
exp: "panic: runtime error: slice bounds out of range [x]",
|
||||
},
|
||||
{
|
||||
message: "panic: runtime error: slice bounds out of range [:4294967283] with capacity 32768",
|
||||
exp: "panic: runtime error: slice bounds out of range [x] with capacity x",
|
||||
},
|
||||
{
|
||||
message: "panic: runtime error: index out of range [0] with length 0",
|
||||
exp: "panic: runtime error: index out of range [x] with length x",
|
||||
},
|
||||
{
|
||||
message: `panic: leveldb: internal key "\x01", len=1: invalid length`,
|
||||
exp: `panic: leveldb: internal key "x", len=x: invalid length`,
|
||||
ldb: true,
|
||||
},
|
||||
{
|
||||
message: `panic: write /var/syncthing/config/index-v0.14.0.db/2732813.log: cannot allocate memory`,
|
||||
exp: `panic: write x: cannot allocate memory`,
|
||||
ldb: true,
|
||||
},
|
||||
{
|
||||
message: `panic: filling Blocks: read C:\Users\Serv-Resp-Tizayuca\AppData\Local\Syncthing\index-v0.14.0.db\006561.ldb: Error de datos (comprobación de redundancia cíclica).`,
|
||||
exp: `panic: filling Blocks: read x: Error de datos (comprobación de redundancia cíclica).`,
|
||||
ldb: true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range cases {
|
||||
fingerprint := crashReportFingerprint(tc.message)
|
||||
|
||||
expLen := 2
|
||||
if tc.ldb {
|
||||
expLen = 1
|
||||
}
|
||||
if l := len(fingerprint); l != expLen {
|
||||
t.Errorf("tc %v: Unexpected fingerprint length: %v != %v", i, l, expLen)
|
||||
} else if msg := fingerprint[expLen-1]; msg != tc.exp {
|
||||
t.Errorf("tc %v:\n\"%v\" !=\n\"%v\"", i, msg, tc.exp)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// Copyright (C) 2019 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
urlPrefix = "https://raw.githubusercontent.com/syncthing/syncthing/"
|
||||
httpTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
type githubSourceCodeLoader struct {
|
||||
mut sync.Mutex
|
||||
version string
|
||||
cache map[string]map[string][][]byte // version -> file -> lines
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newGithubSourceCodeLoader() *githubSourceCodeLoader {
|
||||
return &githubSourceCodeLoader{
|
||||
cache: make(map[string]map[string][][]byte),
|
||||
client: &http.Client{Timeout: httpTimeout},
|
||||
}
|
||||
}
|
||||
|
||||
func (l *githubSourceCodeLoader) LockWithVersion(version string) {
|
||||
l.mut.Lock()
|
||||
l.version = version
|
||||
if _, ok := l.cache[version]; !ok {
|
||||
l.cache[version] = make(map[string][][]byte)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *githubSourceCodeLoader) Unlock() {
|
||||
l.mut.Unlock()
|
||||
}
|
||||
|
||||
func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
|
||||
filename = filepath.ToSlash(filename)
|
||||
lines, ok := l.cache[l.version][filename]
|
||||
if !ok {
|
||||
// Cache whatever we managed to find (or nil if nothing, so we don't try again)
|
||||
defer func() {
|
||||
l.cache[l.version][filename] = lines
|
||||
}()
|
||||
|
||||
knownPrefixes := []string{"/lib/", "/cmd/"}
|
||||
var idx int
|
||||
for _, pref := range knownPrefixes {
|
||||
idx = strings.Index(filename, pref)
|
||||
if idx >= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx == -1 {
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
url := urlPrefix + l.version + filename[idx:]
|
||||
resp, err := l.client.Get(url)
|
||||
|
||||
if err != nil {
|
||||
fmt.Println("Loading source:", err)
|
||||
return nil, 0
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println("Loading source:", resp.Status)
|
||||
return nil, 0
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
fmt.Println("Loading source:", err.Error())
|
||||
return nil, 0
|
||||
}
|
||||
lines = bytes.Split(data, []byte{'\n'})
|
||||
}
|
||||
|
||||
return getLineFromLines(lines, line, context)
|
||||
}
|
||||
|
||||
func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
|
||||
if lines == nil {
|
||||
// cached error from ReadFile: return no lines
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
line-- // stack trace lines are 1-indexed
|
||||
start := line - context
|
||||
var idx int
|
||||
if start < 0 {
|
||||
start = 0
|
||||
idx = line
|
||||
} else {
|
||||
idx = context
|
||||
}
|
||||
end := line + context + 1
|
||||
if line >= len(lines) {
|
||||
return nil, 0
|
||||
}
|
||||
if end > len(lines) {
|
||||
end = len(lines)
|
||||
}
|
||||
return lines[start:end], idx
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Copyright (C) 2019 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type crashReceiver struct {
|
||||
store *diskStore
|
||||
sentry *sentryService
|
||||
ignore *ignorePatterns
|
||||
|
||||
ignoredMut sync.RWMutex
|
||||
ignored map[string]struct{}
|
||||
}
|
||||
|
||||
func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
// The final path component should be a SHA256 hash in hex, so 64 hex
|
||||
// characters. We don't care about case on the request but use lower
|
||||
// case internally.
|
||||
reportID := strings.ToLower(path.Base(req.URL.Path))
|
||||
if len(reportID) != 64 {
|
||||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, c := range reportID {
|
||||
if c >= 'a' && c <= 'f' {
|
||||
continue
|
||||
}
|
||||
if c >= '0' && c <= '9' {
|
||||
continue
|
||||
}
|
||||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
switch req.Method {
|
||||
case http.MethodGet:
|
||||
r.serveGet(reportID, w, req)
|
||||
case http.MethodHead:
|
||||
r.serveHead(reportID, w, req)
|
||||
case http.MethodPut:
|
||||
r.servePut(reportID, w, req)
|
||||
default:
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
// serveGet responds to GET requests by serving the uncompressed report.
|
||||
func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http.Request) {
|
||||
bs, err := r.store.Get(reportID)
|
||||
if err != nil {
|
||||
http.Error(w, "Not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Write(bs)
|
||||
}
|
||||
|
||||
// serveHead responds to HEAD requests by checking if the named report
|
||||
// already exists in the system.
|
||||
func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
|
||||
r.ignoredMut.RLock()
|
||||
_, ignored := r.ignored[reportID]
|
||||
r.ignoredMut.RUnlock()
|
||||
if ignored {
|
||||
return // found
|
||||
}
|
||||
if !r.store.Exists(reportID) {
|
||||
http.Error(w, "Not found", http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// servePut accepts and stores the given report.
|
||||
func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
|
||||
result := "receive_failure"
|
||||
defer func() {
|
||||
metricCrashReportsTotal.WithLabelValues(result).Inc()
|
||||
}()
|
||||
|
||||
r.ignoredMut.RLock()
|
||||
_, ignored := r.ignored[reportID]
|
||||
r.ignoredMut.RUnlock()
|
||||
if ignored {
|
||||
result = "ignored_cached"
|
||||
io.Copy(io.Discard, req.Body)
|
||||
return // found
|
||||
}
|
||||
|
||||
// Read at most maxRequestSize of report data.
|
||||
log.Println("Receiving report", reportID)
|
||||
lr := io.LimitReader(req.Body, maxRequestSize)
|
||||
bs, err := io.ReadAll(lr)
|
||||
if err != nil {
|
||||
log.Println("Reading report:", err)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if r.ignore.match(bs) {
|
||||
r.ignoredMut.Lock()
|
||||
if r.ignored == nil {
|
||||
r.ignored = make(map[string]struct{})
|
||||
}
|
||||
r.ignored[reportID] = struct{}{}
|
||||
r.ignoredMut.Unlock()
|
||||
result = "ignored"
|
||||
return
|
||||
}
|
||||
|
||||
result = "success"
|
||||
|
||||
// Store the report
|
||||
if !r.store.Put(reportID, bs) {
|
||||
log.Println("Failed to store report (queue full):", reportID)
|
||||
result = "queue_failure"
|
||||
}
|
||||
|
||||
// Send the report to Sentry
|
||||
if !r.sentry.Send(reportID, userIDFor(req), bs) {
|
||||
log.Println("Failed to send report to sentry (queue full):", reportID)
|
||||
result = "sentry_failure"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// Copyright (C) 2021 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// userIDFor returns a string we can use as the user ID for the purpose of
|
||||
// counting affected users. It's the truncated hash of a salt, the user
|
||||
// remote IP, and the current month.
|
||||
func userIDFor(req *http.Request) string {
|
||||
addr := req.RemoteAddr
|
||||
if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
|
||||
addr = fwd
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(addr); err == nil {
|
||||
addr = host
|
||||
}
|
||||
now := time.Now().Format("200601")
|
||||
salt := "stcrashreporter"
|
||||
hash := sha256.Sum256([]byte(salt + addr + now))
|
||||
return fmt.Sprintf("%x", hash[:8])
|
||||
}
|
||||
|
||||
// 01234567890abcdef... => 01/23
|
||||
func dirFor(base string) string {
|
||||
return filepath.Join(base[0:2], base[2:4])
|
||||
}
|
||||
|
||||
func fullPathCompressed(root, reportID string) string {
|
||||
return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
|
||||
}
|
||||
|
||||
func compressAndWrite(bs []byte, fullPath string) error {
|
||||
// Compress the report for storage
|
||||
buf := new(bytes.Buffer)
|
||||
gw := gzip.NewWriter(buf)
|
||||
_, _ = gw.Write(bs) // can't fail
|
||||
gw.Close()
|
||||
|
||||
// Create an output file with the compressed report
|
||||
return os.WriteFile(fullPath, buf.Bytes(), 0644)
|
||||
}
|
||||
Reference in New Issue
Block a user