felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/geoip, cmd/relaypoolsrv, cmd/ursrv: Automatically manage GeoIP updates (#9342)

Browse Source 
This adds a small package `geoip` which knows how to download and manage
the Maxmind GeoLite2 database we use. This removes the need for various
scripts to download and manage the geoip database, something that today
happens on Docker startup for the relay pool server and using various
hand written hacks for the usage reporting server.

The database is downloaded when needed and then refreshed on a
best-effort basis weekly.
This commit is contained in:
Jakob Borg
2024-05-18 20:31:49 +03:00
committed by GitHub
parent 57d399317e
commit ba6ac2f604
8 changed files with 212 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/strelaypoolsrv/main.go
+20 -20
View File
@@ -21,12 +21,12 @@ import (
"time"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/oschwald/geoip2-golang"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
"github.com/syncthing/syncthing/lib/assets"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/geoip"
"github.com/syncthing/syncthing/lib/httpcache"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
@@ -100,11 +100,12 @@ var (
debug bool
permRelaysFile string
ipHeader string
geoipPath string
proto string
statsRefresh = time.Minute
requestQueueLen = 64
requestProcessors = 8
geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY")
geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
requests chan request
@@ -130,34 +131,38 @@ func main() {
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
flag.Parse()
requests = make(chan request, requestQueueLen)
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir())
if err != nil {
log.Fatalln("Failed to create GeoIP provider:", err)
}
go geoip.Serve(context.TODO())
var listener net.Listener
var err error
if permRelaysFile != "" {
permanentRelays = loadRelays(permRelaysFile)
permanentRelays = loadRelays(permRelaysFile, geoip)
}
testCert = createTestCertificate()
for i := 0; i < requestProcessors; i++ {
go requestProcessor()
go requestProcessor(geoip)
}
// Load relays from cache in the background.
// Load them in a serial fashion to make sure any genuine requests
// are not dropped.
go func() {
for _, relay := range loadRelays(knownRelaysFile) {
for _, relay := range loadRelays(knownRelaysFile, geoip) {
resultChan := make(chan result)
requests <- request{relay, resultChan, nil}
result := <-resultChan
@@ -425,19 +430,19 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
}
}
func requestProcessor() {
func requestProcessor(geoip *geoip.Provider) {
for request := range requests {
if request.queueTimer != nil {
request.queueTimer.ObserveDuration()
}
timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
handleRelayTest(request)
handleRelayTest(request, geoip)
timer.ObserveDuration()
}
}
func handleRelayTest(request request) {
func handleRelayTest(request request, geoip *geoip.Provider) {
if debug {
log.Println("Request for", request.relay)
}
@@ -450,7 +455,7 @@ func handleRelayTest(request request) {
}
stats := fetchStats(request.relay)
location := getLocation(request.relay.uri.Host)
location := getLocation(request.relay.uri.Host, geoip)
mut.Lock()
if stats != nil {
@@ -523,7 +528,7 @@ func evict(relay *relay) func() {
}
}
func loadRelays(file string) []*relay {
func loadRelays(file string, geoip *geoip.Provider) []*relay {
content, err := os.ReadFile(file)
if err != nil {
log.Println("Failed to load relays: " + err.Error())
@@ -547,7 +552,7 @@ func loadRelays(file string) []*relay {
relays = append(relays, &relay{
URL: line,
Location: getLocation(uri.Host),
Location: getLocation(uri.Host, geoip),
uri: uri,
})
if debug {
@@ -580,21 +585,16 @@ func createTestCertificate() tls.Certificate {
return cert
}
func getLocation(host string) location {
func getLocation(host string, geoip *geoip.Provider) location {
timer := prometheus.NewTimer(locationLookupSeconds)
defer timer.ObserveDuration()
db, err := geoip2.Open(geoipPath)
if err != nil {
return location{}
}
defer db.Close()
addr, err := net.ResolveTCPAddr("tcp", host)
if err != nil {
return location{}
}
city, err := db.City(addr.IP)
city, err := geoip.City(addr.IP)
if err != nil {
return location{}
}
cmd/ursrv/serve/serve.go
+22 -21
View File
@@ -8,6 +8,7 @@ package serve
import (
"bytes"
"context"
"database/sql"
"embed"
"encoding/json"
@@ -17,6 +18,7 @@ import (
"log"
"net"
"net/http"
"os"
"regexp"
"sort"
"strconv"
@@ -26,20 +28,21 @@ import (
"unicode"
_ "github.com/lib/pq" // PostgreSQL driver
"github.com/oschwald/geoip2-golang"
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"github.com/syncthing/syncthing/lib/geoip"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/ur/contract"
)
type CLI struct {
Debug bool `env:"UR_DEBUG"`
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"`
Debug bool `env:"UR_DEBUG"`
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"`
GeoIPAccountID int `env:"UR_GEOIP_ACCOUNT_ID"`
}
//go:embed static
@@ -189,10 +192,16 @@ func (cli *CLI) Run() error {
log.Fatalln("listen:", err)
}
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir())
if err != nil {
log.Fatalln("geoip:", err)
}
go geoip.Serve(context.TODO())
srv := &server{
db: db,
debug: cli.Debug,
geoIPPath: cli.GeoIPPath,
db: db,
debug: cli.Debug,
geoip: geoip,
}
http.HandleFunc("/", srv.rootHandler)
http.HandleFunc("/newdata", srv.newDataHandler)
@@ -213,9 +222,9 @@ func (cli *CLI) Run() error {
}
type server struct {
debug bool
db *sql.DB
geoIPPath string
debug bool
db *sql.DB
geoip *geoip.Provider
cacheMut sync.Mutex
cachedIndex []byte
@@ -238,7 +247,7 @@ func (s *server) cacheRefresher() {
}
func (s *server) refreshCacheLocked() error {
rep := getReport(s.db, s.geoIPPath)
rep := getReport(s.db, s.geoip)
buf := new(bytes.Buffer)
err := tpl.Execute(buf, rep)
if err != nil {
@@ -492,15 +501,7 @@ type weightedLocation struct {
Weight int `json:"weight"`
}
func getReport(db *sql.DB, geoIPPath string) map[string]interface{} {
geoip, err := geoip2.Open(geoIPPath)
if err != nil {
log.Println("opening geoip db", err)
geoip = nil
} else {
defer geoip.Close()
}
func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} {
nodes := 0
countriesTotal := 0
var versions []string