chore(cmd): clean up commands (#9705)
Move infrastructure related commands to under `cmd/infra` and development stuff to `cmd/dev`. The default build command builds the regular user facing binaries: syncthing, stdiscosrv, and strelaysrv.
This commit is contained in:
Jakob Borg
@@ -0,0 +1,22 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 The Syncthing Project
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# relaypoolsrv
|
||||
|
||||
This is the relay pool server for the `syncthing` project, which allows
|
||||
community hosted [relaysrv](https://github.com/syncthing/relaysrv)'s to join
|
||||
the public pool.
|
||||
|
||||
Servers that join the pool are then advertised to users of `syncthing` as
|
||||
potential connection points for those who are unable to connect directly due
|
||||
to NAT or firewall issues.
|
||||
|
||||
There is very little reason why you'd want to run this yourself, as
|
||||
`relaypoolsrv` is just used for announcement and lookup of public relay
|
||||
servers. If you are looking to setup a private or a public relay, please
|
||||
check the documentation for
|
||||
[relaysrv](https://github.com/syncthing/relaysrv), which also explains how
|
||||
to join the default public pool.
|
||||
|
||||
See `relaypoolsrv -help` for configuration options.
|
||||
|
||||
##### Third-party attributions
|
||||
|
||||
[oschwald/geoip2-golang](https://github.com/oschwald/geoip2-golang), [oschwald/maxminddb-golang](https://github.com/oschwald/maxminddb-golang), Copyright (C) 2015 [Gregory J. Oschwald](mailto:oschwald@gmail.com).
|
||||
|
||||
[lib/pq](https://github.com/lib/pq)</a>, Copyright (C) 2011-2013 'pq' Contributors Portions Copyright (C) 2011 Blake Mizerany.
|
||||
@@ -0,0 +1 @@
|
||||
gui.files.go
|
||||
@@ -0,0 +1,10 @@
|
||||
// Copyright (C) 2018 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:generate go run ../../../../script/genassets.go -o gui.files.go ../gui
|
||||
|
||||
// Package auto contains auto generated files for web assets.
|
||||
package auto
|
||||
@@ -0,0 +1,16 @@
|
||||
// Copyright (C) 2021 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build noassets
|
||||
// +build noassets
|
||||
|
||||
package auto
|
||||
|
||||
import "github.com/syncthing/syncthing/lib/assets"
|
||||
|
||||
func Assets() map[string]assets.Asset {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en" ng-app="syncthing" ng-controller="relayDataController">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<meta name="description" content=""/>
|
||||
<meta name="author" content=""/>
|
||||
|
||||
<title>Relay stats</title>
|
||||
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
|
||||
<link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
|
||||
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
|
||||
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.6.0/dist/leaflet.css"
|
||||
integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
|
||||
crossorigin=""/>
|
||||
<script src="https://unpkg.com/leaflet@1.6.0/dist/leaflet.js"
|
||||
integrity="sha512-gZwIG9x3wUXg2hdXF6+rVkLF/0Vi9U8D2Ntg4Ga5I5BZpVkVxlJWbSQtXPSiUTtC0TjtGOmxa1AJPuV0CPthew=="
|
||||
crossorigin=""></script>
|
||||
|
||||
<style>
|
||||
#map {
|
||||
height: 600px;
|
||||
}
|
||||
.ng-cloak {
|
||||
display: none;
|
||||
}
|
||||
table {
|
||||
font-size: 11px !important;
|
||||
width: 100%;
|
||||
border: 1px;
|
||||
|
||||
}
|
||||
td {
|
||||
padding: 0px !important;
|
||||
}
|
||||
tfoot td {
|
||||
font-weight: bold;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body class="ng-cloak">
|
||||
<div class="container">
|
||||
<h1>Relay Pool Data</h1>
|
||||
<div ng-if="relays === undefined" class="text-center">
|
||||
<img src="https://cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
|
||||
<p>Please wait while we gather data…</p>
|
||||
</div>
|
||||
<div>
|
||||
<div ng-show="relays !== undefined" class="ng-hide">
|
||||
<p>
|
||||
The relays listed on this page are not managed or vetted by the Syncthing project.
|
||||
Each relay is the responsibility of the relay operator.
|
||||
Currently {{ relays.length }} relays are online.
|
||||
</p>
|
||||
</div>
|
||||
<div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
|
||||
<p>The circle size represents how much bytes the relay has transferred relatively to other relays.</p>
|
||||
</div>
|
||||
<div>
|
||||
<table class="table table-striped table-condensed table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th rowspan="2">Address</td>
|
||||
<th rowspan="2">
|
||||
<a ng-click="sortType = 'stats.numActiveSessions'; sortReverse = !sortReverse">
|
||||
Sessions
|
||||
<span ng-show="sortType == 'stats.numActiveSessions' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.numActiveSessions' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th rowspan="2">
|
||||
<a ng-click="sortType = 'stats.numConnections'; sortReverse = !sortReverse">
|
||||
Connections
|
||||
<span ng-show="sortType == 'stats.numConnections' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.numConnections' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th rowspan="2">
|
||||
<a ng-click="sortType = 'stats.bytesProxied'; sortReverse = !sortReverse">
|
||||
Data relayed
|
||||
<span ng-show="sortType == 'stats.bytesProxied' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.bytesProxied' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th colspan="6" class="text-center">Transfer rate in the last period</th>
|
||||
<th rowspan="2">
|
||||
<a ng-click="sortType = 'stats.uptimeSeconds'; sortReverse = !sortReverse">
|
||||
Uptime hours
|
||||
<span ng-show="sortType == 'stats.uptimeSeconds' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th rowspan="2">
|
||||
<a ng-click="sortType = 'stats.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
|
||||
Provided by
|
||||
<span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>
|
||||
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
|
||||
10s
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th>
|
||||
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
|
||||
1m
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th>
|
||||
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
|
||||
5m
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th>
|
||||
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
|
||||
15m
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th>
|
||||
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
|
||||
30m
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
<th>
|
||||
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
|
||||
60m
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fas fa-caret-down"></span>
|
||||
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fas fa-caret-up"></span>
|
||||
</a>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare" ng-mouseover="relay.showMarker()" ng-mouseleave="relay.hideMarker()">
|
||||
<td>{{ relay.address }}</td>
|
||||
<td ng-if="!relay.stats" colspan="11"></td>
|
||||
<td ng-if-start="relay.stats">{{ relay.stats.numActiveSessions }}</td>
|
||||
<td>{{ relay.stats.numConnections }}</td>
|
||||
<td>{{ relay.stats.bytesProxied | bytes }}</td>
|
||||
<td>{{ relay.stats.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
|
||||
<td>{{ relay.stats.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
|
||||
<td>{{ relay.stats.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
|
||||
<td>{{ relay.stats.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
|
||||
<td>{{ relay.stats.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
|
||||
<td>{{ relay.stats.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
|
||||
<td ng-if="relay.stats.uptimeSeconds != undefined">{{ relay.stats.uptimeSeconds/60/60 | number:0 }}</td>
|
||||
<td ng-if="relay.stats.uptimeSeconds == undefined"></td>
|
||||
<td title="{{ relay.stats.options['provided-by'] || '' }}" ng-if-end>
|
||||
{{ relay.stats.options['provided-by'] || '' | limitTo:50 }}
|
||||
<span ng-if="(relay.stats.options['provided-by'] || '').length > 50">…
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
<tfoot>
|
||||
<tr>
|
||||
<td>Totals</td>
|
||||
<td>{{ totals.numActiveSessions }}</td>
|
||||
<td>{{ totals.numConnections }}</td>
|
||||
<td>{{ totals.bytesProxied | bytes }}</td>
|
||||
<td>{{ totals.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
|
||||
<td>{{ totals.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
|
||||
<td>{{ totals.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
|
||||
<td>{{ totals.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
|
||||
<td>{{ totals.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
|
||||
<td>{{ totals.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
|
||||
<td>{{ totals.uptimeSeconds/60/60 | number:0 }} hours</td>
|
||||
<td>{{ relays.length }} relays</td>
|
||||
</tr>
|
||||
</tfoor>
|
||||
</table>
|
||||
</div>
|
||||
<hr>
|
||||
<p>
|
||||
This product includes GeoLite2 data created by MaxMind, available from
|
||||
<a href="https://www.maxmind.com">https://www.maxmind.com</a>.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.4.min.js"></script>
|
||||
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
|
||||
<script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
|
||||
</body>
|
||||
|
||||
<script>
|
||||
angular.module('syncthing', [
|
||||
])
|
||||
.config(['$httpProvider', function($httpProvider) {
|
||||
$httpProvider.defaults.timeout = 5000;
|
||||
}])
|
||||
.filter('bytes', function() {
|
||||
return function(bytes, precision) {
|
||||
if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
|
||||
if (typeof precision === 'undefined') precision = 1;
|
||||
|
||||
var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
|
||||
number = Math.floor(Math.log(bytes) / Math.log(1024));
|
||||
|
||||
var value = (bytes / Math.pow(1000, Math.floor(number)));
|
||||
if (!isFinite(value)) {
|
||||
value = 0;
|
||||
precision = 0;
|
||||
}
|
||||
if (!isFinite(number)) {
|
||||
units = 'bytes';
|
||||
} else {
|
||||
units = units[number];
|
||||
}
|
||||
return value.toFixed(precision) + ' ' + units;
|
||||
}
|
||||
})
|
||||
.controller('relayDataController', ['$scope', '$rootScope', '$http', '$q', '$compile', '$timeout', function($scope, $rootScope, $http, $q, $compile, $timeout) {
|
||||
$scope.totals = {
|
||||
bytesProxied: 0,
|
||||
goMaxProcs: 0,
|
||||
kbps10s1m5m15m30m60m: [0, 0, 0, 0, 0, 0],
|
||||
numActiveSessions: 0,
|
||||
numConnections: 0,
|
||||
numPendingSessionKeys: 0,
|
||||
numProxies: 0,
|
||||
uptimeSeconds: 0,
|
||||
};
|
||||
$scope.map = L.map('map').setView([40.90296, 1.90925], 2);
|
||||
L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',
|
||||
{
|
||||
attribution: 'Leaflet',
|
||||
maxZoom: 17
|
||||
}).addTo($scope.map);
|
||||
$scope.tooltipTemplate = $('#infoTemplate').html();
|
||||
$scope.usedLocations = {};
|
||||
$scope.sortType = 'stats.numActiveSessions';
|
||||
$scope.sortReverse = true;
|
||||
$scope.sortCompare = function(a, b) {
|
||||
if (a.value == b.value) {
|
||||
return 0;
|
||||
}
|
||||
if (a.type == "undefined" || a.type == "null") {
|
||||
return -1;
|
||||
}
|
||||
if (b.type == "undefined" || b.type == "null") {
|
||||
return 1;
|
||||
}
|
||||
return a.value > b.value ? 1 : -1;
|
||||
}
|
||||
|
||||
$http.get("/endpoint/full").then(function(response) {
|
||||
$scope.relays = response.data.relays;
|
||||
|
||||
angular.forEach($scope.relays, function(relay) {
|
||||
relay.uri = constructURI(relay.url);
|
||||
relay.address = relay.url.split('/')[2];
|
||||
|
||||
addMarkerToMap(relay);
|
||||
|
||||
if (relay.stats) {
|
||||
angular.forEach($scope.totals, function(value, key) {
|
||||
if (typeof $scope.totals[key] == 'number') {
|
||||
$scope.totals[key] += relay.stats[key];
|
||||
} else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
|
||||
angular.forEach($scope.totals[key], function(value, index) {
|
||||
$scope.totals[key][index] += relay.stats[key][index];
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// After the totals were calculated, add circles.
|
||||
angular.forEach($scope.relays, function(relay) {
|
||||
if (relay.stats) {
|
||||
addCircleToMap(relay);
|
||||
}
|
||||
});
|
||||
|
||||
if ($scope.relays.length == 1) {
|
||||
//Center to only relay with zoom
|
||||
$scope.map.panTo(new L.LatLng(relays[0].location.latitude, relays[0].location.longitude));
|
||||
$scope.map.setZoom(13);
|
||||
}
|
||||
});
|
||||
|
||||
function addMarkerToMap(relay) {
|
||||
var loc = relay.location.latitude + "," + relay.location.longitude;
|
||||
|
||||
// Deal with overlapping markers
|
||||
while (loc in $scope.usedLocations) {
|
||||
var locParts = loc.split(',');
|
||||
locParts = [parseFloat(locParts[0]), parseFloat(locParts[1])];
|
||||
locParts[Math.round(Math.random())] += 0.5 * (Math.random() >= 0.5 ? 1 : -1);
|
||||
loc = locParts.join(',');
|
||||
}
|
||||
|
||||
$scope.usedLocations[loc] = true;
|
||||
|
||||
var locParts = loc.split(',');
|
||||
|
||||
relay.marker = new L.Marker([relay.location.latitude, relay.location.longitude],{
|
||||
title: relay.url,
|
||||
});
|
||||
|
||||
var scope = $rootScope.$new(true);
|
||||
scope.relay = relay;
|
||||
|
||||
var icon = new L.Icon({
|
||||
iconSize: [18, 28], // size of the icon
|
||||
iconAnchor: [9, 28], // point of the icon which will correspond to marker's location
|
||||
shadowAnchor: [0, 0], // the same for the shadow
|
||||
popupAnchor: [0, -27], // popup anchor
|
||||
shadowSize: [0,0],
|
||||
iconUrl: 'https://cdn.rawgit.com/pointhi/leaflet-color-markers/master/img/marker-icon-red.png',
|
||||
shadowUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/images/marker-shadow.png',
|
||||
});
|
||||
|
||||
relay.marker = new L.marker(new L.latLng(locParts[0], locParts[1]),{icon})
|
||||
.bindPopup($compile($scope.tooltipTemplate)(scope)[0],{})
|
||||
.on('mouseover', function (e) {
|
||||
this.openPopup();
|
||||
}).on('mouseout', function (e) {
|
||||
this.closePopup();
|
||||
}).addTo($scope.map);
|
||||
|
||||
relay.showMarker = function() {
|
||||
relay.marker.openPopup();
|
||||
}
|
||||
|
||||
relay.hideMarker = function() {
|
||||
relay.marker.closePopup();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function addCircleToMap(relay) {
|
||||
console.log(relay.location.latitude)
|
||||
L.circle([relay.location.latitude, relay.location.longitude],
|
||||
{
|
||||
radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
|
||||
color: "FF0000",
|
||||
fillColor: "#FF0000",
|
||||
fillOpacity: 0.35,
|
||||
}).addTo($scope.map);
|
||||
}
|
||||
|
||||
function constructURI(url) {
|
||||
var uri = document.createElement('a');
|
||||
|
||||
// HAX, otherwise doesn't work
|
||||
uri.href = url.replace('relay://', 'http://');
|
||||
|
||||
// Convert query string to object
|
||||
uri.args = {};
|
||||
angular.forEach(uri.search.replace(/^\?/, '').split('&'), function(query) {
|
||||
var split = query.split('=');
|
||||
uri.args[split[0]] = split[1];
|
||||
});
|
||||
|
||||
return uri;
|
||||
}
|
||||
}]);
|
||||
</script>
|
||||
|
||||
<script type="text/template" id="infoTemplate">
|
||||
<div>
|
||||
<p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.stats.options['provided-by']">provided by <u>{{ relay.stats.options['provided-by'] }}</u></span></p>
|
||||
<div ng-if="relay.stats">
|
||||
<span ng-if="relay.stats.startTime">Start time: {{ relay.stats.startTime | date:"medium" }}</br></span>
|
||||
<span ng-if="relay.stats.bytesProxied != undefined">Proxied: {{ relay.stats.bytesProxied | bytes }}</br></span>
|
||||
<span ng-if="relay.stats.numActiveSessions != undefined">Sessions: {{ relay.stats.numActiveSessions }}</br></span>
|
||||
<span ng-if="relay.stats.numConnections != undefined">Clients: {{ relay.stats.numConnections }}</br></span>
|
||||
<span ng-if="relay.stats.options.pools">Pools: {{ relay.stats.options.pools.join(', ') }}</br></span>
|
||||
<span ng-if="relay.stats.options['global-rate'] != undefined">
|
||||
<span ng-if="relay.stats.options['global-rate'] > 0">Global rate limit: {{ relay.stats.options['global-rate'] | bytes }}/s</span>
|
||||
<span ng-if="relay.stats.options['global-rate'] == 0">Global rate limit: unlimited</span>
|
||||
<br/>
|
||||
</span>
|
||||
<span ng-if="relay.stats.options['per-session-rate'] != undefined">
|
||||
<span ng-if="relay.stats.options['per-session-rate'] > 0">Session rate limit: {{ relay.stats.options['per-session-rate'] | bytes }}/s</span>
|
||||
<span ng-if="relay.stats.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
|
||||
<br/>
|
||||
</span>
|
||||
</div>
|
||||
<div ng-if="!relay.stats">
|
||||
Data unavailable.
|
||||
<div>
|
||||
</div>
|
||||
</script>
|
||||
</html>
|
||||
@@ -0,0 +1,710 @@
|
||||
// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto"
|
||||
"github.com/syncthing/syncthing/lib/assets"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/geoip"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/relay/client"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
)
|
||||
|
||||
type location struct {
|
||||
Latitude float64 `json:"latitude"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
City string `json:"city"`
|
||||
Country string `json:"country"`
|
||||
Continent string `json:"continent"`
|
||||
}
|
||||
|
||||
type relay struct {
|
||||
URL string `json:"url"`
|
||||
Location location `json:"location"`
|
||||
uri *url.URL
|
||||
Stats *stats `json:"stats"`
|
||||
StatsRetrieved time.Time `json:"statsRetrieved"`
|
||||
}
|
||||
|
||||
type relayShort struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type stats struct {
|
||||
StartTime time.Time `json:"startTime"`
|
||||
UptimeSeconds int `json:"uptimeSeconds"`
|
||||
PendingSessionKeys int `json:"numPendingSessionKeys"`
|
||||
ActiveSessions int `json:"numActiveSessions"`
|
||||
Connections int `json:"numConnections"`
|
||||
Proxies int `json:"numProxies"`
|
||||
BytesProxied int `json:"bytesProxied"`
|
||||
GoVersion string `json:"goVersion"`
|
||||
GoOS string `json:"goOS"`
|
||||
GoArch string `json:"goArch"`
|
||||
GoMaxProcs int `json:"goMaxProcs"`
|
||||
GoRoutines int `json:"goNumRoutine"`
|
||||
Rates []int64 `json:"kbps10s1m5m15m30m60m"`
|
||||
Options struct {
|
||||
NetworkTimeout int `json:"network-timeout"`
|
||||
PintInterval int `json:"ping-interval"`
|
||||
MessageTimeout int `json:"message-timeout"`
|
||||
SessionRate int `json:"per-session-rate"`
|
||||
GlobalRate int `json:"global-rate"`
|
||||
Pools []string `json:"pools"`
|
||||
ProvidedBy string `json:"provided-by"`
|
||||
} `json:"options"`
|
||||
}
|
||||
|
||||
func (r relay) String() string {
|
||||
return r.URL
|
||||
}
|
||||
|
||||
type request struct {
|
||||
relay *relay
|
||||
result chan result
|
||||
queueTimer *prometheus.Timer
|
||||
}
|
||||
|
||||
type result struct {
|
||||
err error
|
||||
eviction time.Duration
|
||||
}
|
||||
|
||||
var (
|
||||
testCert tls.Certificate
|
||||
knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
|
||||
listen = ":80"
|
||||
metricsListen = ":8081"
|
||||
dir string
|
||||
evictionTime = time.Hour
|
||||
debug bool
|
||||
permRelaysFile string
|
||||
ipHeader string
|
||||
proto string
|
||||
statsRefresh = time.Minute
|
||||
requestQueueLen = 64
|
||||
requestProcessors = 8
|
||||
geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY")
|
||||
geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
|
||||
|
||||
requests chan request
|
||||
|
||||
mut = sync.NewRWMutex()
|
||||
knownRelays = make([]*relay, 0)
|
||||
permanentRelays = make([]*relay, 0)
|
||||
evictionTimers = make(map[string]*time.Timer)
|
||||
globalBlocklist = newErrorTracker(1000)
|
||||
)
|
||||
|
||||
const (
|
||||
httpStatusEnhanceYourCalm = 429
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(log.Lshortfile)
|
||||
|
||||
flag.StringVar(&listen, "listen", listen, "Listen address")
|
||||
flag.StringVar(&metricsListen, "metrics-listen", metricsListen, "Metrics listen address")
|
||||
flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
|
||||
flag.BoolVar(&debug, "debug", debug, "Enable debug output")
|
||||
flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
|
||||
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
|
||||
flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
|
||||
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
|
||||
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
|
||||
flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
|
||||
flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
|
||||
flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
|
||||
flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
requests = make(chan request, requestQueueLen)
|
||||
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir())
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to create GeoIP provider:", err)
|
||||
}
|
||||
go geoip.Serve(context.TODO())
|
||||
|
||||
var listener net.Listener
|
||||
|
||||
if permRelaysFile != "" {
|
||||
permanentRelays = loadRelays(permRelaysFile, geoip)
|
||||
}
|
||||
|
||||
testCert = createTestCertificate()
|
||||
|
||||
for i := 0; i < requestProcessors; i++ {
|
||||
go requestProcessor(geoip)
|
||||
}
|
||||
|
||||
// Load relays from cache in the background.
|
||||
// Load them in a serial fashion to make sure any genuine requests
|
||||
// are not dropped.
|
||||
go func() {
|
||||
for _, relay := range loadRelays(knownRelaysFile, geoip) {
|
||||
resultChan := make(chan result)
|
||||
requests <- request{relay, resultChan, nil}
|
||||
result := <-resultChan
|
||||
if result.err != nil {
|
||||
relayTestsTotal.WithLabelValues("failed").Inc()
|
||||
} else {
|
||||
relayTestsTotal.WithLabelValues("success").Inc()
|
||||
}
|
||||
}
|
||||
// Run the the stats refresher once the relays are loaded.
|
||||
statsRefresher(statsRefresh)
|
||||
}()
|
||||
|
||||
if dir != "" {
|
||||
if debug {
|
||||
log.Println("Starting TLS listener on", listen)
|
||||
}
|
||||
certFile, keyFile := filepath.Join(dir, "http-cert.pem"), filepath.Join(dir, "http-key.pem")
|
||||
var cert tls.Certificate
|
||||
cert, err = tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to load HTTP X509 key pair:", err)
|
||||
}
|
||||
|
||||
tlsCfg := &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
MinVersion: tls.VersionTLS10, // No SSLv3
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
CipherSuites: []uint16{
|
||||
// No RC4
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
|
||||
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
|
||||
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
|
||||
tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
|
||||
tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
|
||||
},
|
||||
}
|
||||
|
||||
listener, err = tls.Listen(proto, listen, tlsCfg)
|
||||
} else {
|
||||
if debug {
|
||||
log.Println("Starting plain listener on", listen)
|
||||
}
|
||||
listener, err = net.Listen(proto, listen)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Fatalln("listen:", err)
|
||||
}
|
||||
|
||||
if metricsListen != "" {
|
||||
mmux := http.NewServeMux()
|
||||
mmux.HandleFunc("/metrics", handleMetrics)
|
||||
go func() {
|
||||
if err := http.ListenAndServe(metricsListen, mmux); err != nil {
|
||||
log.Fatalln("HTTP serve metrics:", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
getMux := http.NewServeMux()
|
||||
getMux.HandleFunc("/", handleAssets)
|
||||
getMux.HandleFunc("/endpoint", withAPIMetrics(handleEndpointShort))
|
||||
getMux.HandleFunc("/endpoint/full", withAPIMetrics(handleEndpointFull))
|
||||
|
||||
postMux := http.NewServeMux()
|
||||
postMux.HandleFunc("/endpoint", withAPIMetrics(handleRegister))
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodOptions:
|
||||
getMux.ServeHTTP(w, r)
|
||||
case http.MethodPost:
|
||||
postMux.ServeHTTP(w, r)
|
||||
default:
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
})
|
||||
|
||||
srv := http.Server{
|
||||
Handler: handler,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
}
|
||||
srv.SetKeepAlivesEnabled(false)
|
||||
|
||||
err = srv.Serve(listener)
|
||||
if err != nil {
|
||||
log.Fatalln("serve:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
timer := prometheus.NewTimer(metricsRequestsSeconds)
|
||||
// Acquire the mutex just to make sure we're not caught mid-way stats collection
|
||||
mut.RLock()
|
||||
promhttp.Handler().ServeHTTP(w, r)
|
||||
mut.RUnlock()
|
||||
timer.ObserveDuration()
|
||||
}
|
||||
|
||||
func handleAssets(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Cache-Control", "no-cache, must-revalidate")
|
||||
|
||||
path := r.URL.Path[1:]
|
||||
if path == "" {
|
||||
path = "index.html"
|
||||
}
|
||||
|
||||
as, ok := auto.Assets()[path]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
assets.Serve(w, r, as)
|
||||
}
|
||||
|
||||
func withAPIMetrics(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
|
||||
w = NewLoggingResponseWriter(w)
|
||||
defer func() {
|
||||
timer.ObserveDuration()
|
||||
lw := w.(*loggingResponseWriter)
|
||||
apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
|
||||
}()
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// handleEndpointFull returns the relay list with full metadata and
|
||||
// statistics. Large, and expensive.
|
||||
func handleEndpointFull(rw http.ResponseWriter, r *http.Request) {
|
||||
rw.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
rw.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
mut.RLock()
|
||||
relays := make([]*relay, len(permanentRelays)+len(knownRelays))
|
||||
n := copy(relays, permanentRelays)
|
||||
copy(relays[n:], knownRelays)
|
||||
mut.RUnlock()
|
||||
|
||||
_ = json.NewEncoder(rw).Encode(map[string][]*relay{
|
||||
"relays": relays,
|
||||
})
|
||||
}
|
||||
|
||||
// handleEndpointShort returns the relay list with only the URL.
|
||||
func handleEndpointShort(rw http.ResponseWriter, r *http.Request) {
|
||||
rw.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
rw.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
mut.RLock()
|
||||
relays := make([]relayShort, 0, len(permanentRelays)+len(knownRelays))
|
||||
for _, r := range append(permanentRelays, knownRelays...) {
|
||||
relays = append(relays, relayShort{URL: slimURL(r.URL)})
|
||||
}
|
||||
mut.RUnlock()
|
||||
|
||||
_ = json.NewEncoder(rw).Encode(map[string][]relayShort{
|
||||
"relays": relays,
|
||||
})
|
||||
}
|
||||
|
||||
func handleRegister(w http.ResponseWriter, r *http.Request) {
|
||||
// Get the IP address of the client
|
||||
rhost := r.RemoteAddr
|
||||
if ipHeader != "" {
|
||||
hdr := r.Header.Get(ipHeader)
|
||||
fields := strings.Split(hdr, ",")
|
||||
if len(fields) > 0 {
|
||||
rhost = strings.TrimSpace(fields[len(fields)-1])
|
||||
}
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(rhost); err == nil {
|
||||
rhost = host
|
||||
}
|
||||
|
||||
// Check the black list. A client is blacklisted if their last 10
|
||||
// attempts to join have all failed. The "Unauthorized" status return
|
||||
// causes strelaysrv to cease attempting to join.
|
||||
if globalBlocklist.IsBlocked(rhost) {
|
||||
log.Println("Rejected blocked client", rhost)
|
||||
http.Error(w, "Too many errors", http.StatusUnauthorized)
|
||||
globalBlocklist.ClearErrors(rhost)
|
||||
return
|
||||
}
|
||||
|
||||
var relayCert *x509.Certificate
|
||||
if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
|
||||
relayCert = r.TLS.PeerCertificates[0]
|
||||
log.Printf("Got TLS cert from relay server")
|
||||
}
|
||||
|
||||
var newRelay relay
|
||||
err := json.NewDecoder(r.Body).Decode(&newRelay)
|
||||
r.Body.Close()
|
||||
|
||||
if err != nil {
|
||||
if debug {
|
||||
log.Println("Failed to parse payload")
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
uri, err := url.Parse(newRelay.URL)
|
||||
if err != nil {
|
||||
if debug {
|
||||
log.Println("Failed to parse URI", newRelay.URL)
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Canonicalize the URL. In particular, parse and re-encode the query
|
||||
// string so that it's guaranteed to be valid.
|
||||
uri.RawQuery = uri.Query().Encode()
|
||||
newRelay.URL = uri.String()
|
||||
|
||||
if relayCert != nil {
|
||||
advertisedId := uri.Query().Get("id")
|
||||
idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
|
||||
if advertisedId != idFromCert {
|
||||
log.Println("Warning: Relay server requested to join with an ID different from the join request, rejecting")
|
||||
http.Error(w, "mismatched advertised id and join request cert", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(uri.Host)
|
||||
if err != nil {
|
||||
if debug {
|
||||
log.Println("Failed to split URI", newRelay.URL)
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
ip := net.ParseIP(host)
|
||||
// The client did not provide an IP address, use the IP address of the client.
|
||||
if ip == nil || ip.IsUnspecified() {
|
||||
uri.Host = net.JoinHostPort(rhost, port)
|
||||
newRelay.URL = uri.String()
|
||||
} else if host != rhost && relayCert == nil {
|
||||
if debug {
|
||||
log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("IP advertised %s does not match client IP %s", host, rhost), http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
newRelay.uri = uri
|
||||
|
||||
for _, current := range permanentRelays {
|
||||
if current.uri.Host == newRelay.uri.Host {
|
||||
if debug {
|
||||
log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
|
||||
}
|
||||
http.Error(w, "Invalid request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
reschan := make(chan result)
|
||||
|
||||
select {
|
||||
case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
|
||||
result := <-reschan
|
||||
if result.err != nil {
|
||||
log.Println("Join from", r.RemoteAddr, "failed:", result.err)
|
||||
globalBlocklist.AddError(rhost)
|
||||
relayTestsTotal.WithLabelValues("failed").Inc()
|
||||
http.Error(w, result.err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
log.Println("Join from", r.RemoteAddr, "succeeded")
|
||||
globalBlocklist.ClearErrors(rhost)
|
||||
relayTestsTotal.WithLabelValues("success").Inc()
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
json.NewEncoder(w).Encode(map[string]time.Duration{
|
||||
"evictionIn": result.eviction,
|
||||
})
|
||||
|
||||
default:
|
||||
relayTestsTotal.WithLabelValues("dropped").Inc()
|
||||
if debug {
|
||||
log.Println("Dropping request")
|
||||
}
|
||||
w.WriteHeader(httpStatusEnhanceYourCalm)
|
||||
}
|
||||
}
|
||||
|
||||
func requestProcessor(geoip *geoip.Provider) {
|
||||
for request := range requests {
|
||||
if request.queueTimer != nil {
|
||||
request.queueTimer.ObserveDuration()
|
||||
}
|
||||
|
||||
timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
|
||||
handleRelayTest(request, geoip)
|
||||
timer.ObserveDuration()
|
||||
}
|
||||
}
|
||||
|
||||
func handleRelayTest(request request, geoip *geoip.Provider) {
|
||||
if debug {
|
||||
log.Println("Request for", request.relay)
|
||||
}
|
||||
if err := client.TestRelay(context.TODO(), request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3); err != nil {
|
||||
if debug {
|
||||
log.Println("Test for relay", request.relay, "failed:", err)
|
||||
}
|
||||
request.result <- result{err, 0}
|
||||
return
|
||||
}
|
||||
|
||||
stats := fetchStats(request.relay)
|
||||
location := getLocation(request.relay.uri.Host, geoip)
|
||||
|
||||
mut.Lock()
|
||||
if stats != nil {
|
||||
updateMetrics(request.relay.uri.Host, *stats, location)
|
||||
}
|
||||
request.relay.Stats = stats
|
||||
request.relay.StatsRetrieved = time.Now().Truncate(time.Second)
|
||||
request.relay.Location = location
|
||||
|
||||
timer, ok := evictionTimers[request.relay.uri.Host]
|
||||
if ok {
|
||||
if debug {
|
||||
log.Println("Stopping existing timer for", request.relay)
|
||||
}
|
||||
timer.Stop()
|
||||
}
|
||||
|
||||
for i, current := range knownRelays {
|
||||
if current.uri.Host == request.relay.uri.Host {
|
||||
if debug {
|
||||
log.Println("Relay", request.relay, "already exists")
|
||||
}
|
||||
|
||||
// Evict the old entry anyway, as configuration might have changed.
|
||||
last := len(knownRelays) - 1
|
||||
knownRelays[i] = knownRelays[last]
|
||||
knownRelays = knownRelays[:last]
|
||||
|
||||
goto found
|
||||
}
|
||||
}
|
||||
|
||||
if debug {
|
||||
log.Println("Adding new relay", request.relay)
|
||||
}
|
||||
|
||||
found:
|
||||
|
||||
knownRelays = append(knownRelays, request.relay)
|
||||
evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
|
||||
|
||||
mut.Unlock()
|
||||
|
||||
if err := saveRelays(knownRelaysFile, knownRelays); err != nil {
|
||||
log.Println("Failed to write known relays: " + err.Error())
|
||||
}
|
||||
|
||||
request.result <- result{nil, evictionTime}
|
||||
}
|
||||
|
||||
func evict(relay *relay) func() {
|
||||
return func() {
|
||||
mut.Lock()
|
||||
defer mut.Unlock()
|
||||
if debug {
|
||||
log.Println("Evicting", relay)
|
||||
}
|
||||
for i, current := range knownRelays {
|
||||
if current.uri.Host == relay.uri.Host {
|
||||
if debug {
|
||||
log.Println("Evicted", relay)
|
||||
}
|
||||
last := len(knownRelays) - 1
|
||||
knownRelays[i] = knownRelays[last]
|
||||
knownRelays = knownRelays[:last]
|
||||
deleteMetrics(current.uri.Host)
|
||||
}
|
||||
}
|
||||
delete(evictionTimers, relay.uri.Host)
|
||||
}
|
||||
}
|
||||
|
||||
func loadRelays(file string, geoip *geoip.Provider) []*relay {
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
log.Println("Failed to load relays: " + err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
var relays []*relay
|
||||
for _, line := range strings.Split(string(content), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
uri, err := url.Parse(line)
|
||||
if err != nil {
|
||||
if debug {
|
||||
log.Println("Skipping relay", line, "due to parse error", err)
|
||||
}
|
||||
continue
|
||||
|
||||
}
|
||||
|
||||
relays = append(relays, &relay{
|
||||
URL: line,
|
||||
Location: getLocation(uri.Host, geoip),
|
||||
uri: uri,
|
||||
})
|
||||
if debug {
|
||||
log.Println("Adding relay", line)
|
||||
}
|
||||
}
|
||||
return relays
|
||||
}
|
||||
|
||||
func saveRelays(file string, relays []*relay) error {
|
||||
var content string
|
||||
for _, relay := range relays {
|
||||
content += relay.uri.String() + "\n"
|
||||
}
|
||||
return os.WriteFile(file, []byte(content), 0o777)
|
||||
}
|
||||
|
||||
func createTestCertificate() tls.Certificate {
|
||||
tmpDir, err := os.MkdirTemp("", "relaypoolsrv")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
|
||||
cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to create test X509 key pair:", err)
|
||||
}
|
||||
|
||||
return cert
|
||||
}
|
||||
|
||||
func getLocation(host string, geoip *geoip.Provider) location {
|
||||
timer := prometheus.NewTimer(locationLookupSeconds)
|
||||
defer timer.ObserveDuration()
|
||||
|
||||
addr, err := net.ResolveTCPAddr("tcp", host)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
|
||||
city, err := geoip.City(addr.IP)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
|
||||
return location{
|
||||
Longitude: city.Location.Longitude,
|
||||
Latitude: city.Location.Latitude,
|
||||
City: city.City.Names["en"],
|
||||
Country: city.Country.IsoCode,
|
||||
Continent: city.Continent.Code,
|
||||
}
|
||||
}
|
||||
|
||||
type loggingResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
statusCode int
|
||||
}
|
||||
|
||||
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
|
||||
return &loggingResponseWriter{w, http.StatusOK}
|
||||
}
|
||||
|
||||
func (lrw *loggingResponseWriter) WriteHeader(code int) {
|
||||
lrw.statusCode = code
|
||||
lrw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
type errorTracker struct {
|
||||
errors *lru.TwoQueueCache[string, *errorCounter]
|
||||
}
|
||||
|
||||
type errorCounter struct {
|
||||
count atomic.Int32
|
||||
}
|
||||
|
||||
func newErrorTracker(size int) *errorTracker {
|
||||
cache, err := lru.New2Q[string, *errorCounter](size)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &errorTracker{
|
||||
errors: cache,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *errorTracker) AddError(host string) {
|
||||
entry, ok := b.errors.Get(host)
|
||||
if !ok {
|
||||
entry = &errorCounter{}
|
||||
b.errors.Add(host, entry)
|
||||
}
|
||||
c := entry.count.Add(1)
|
||||
log.Printf("Error count for %s is now %d", host, c)
|
||||
}
|
||||
|
||||
func (b *errorTracker) ClearErrors(host string) {
|
||||
b.errors.Remove(host)
|
||||
}
|
||||
|
||||
func (b *errorTracker) IsBlocked(host string) bool {
|
||||
if be, ok := b.errors.Get(host); ok {
|
||||
return be.count.Load() > 10
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func slimURL(u string) string {
|
||||
p, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return u
|
||||
}
|
||||
newQuery := url.Values{}
|
||||
if id := p.Query().Get("id"); id != "" {
|
||||
newQuery.Set("id", id)
|
||||
}
|
||||
p.RawQuery = newQuery.Encode()
|
||||
return p.String()
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright © 2020 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func init() {
|
||||
for i := 0; i < 10; i++ {
|
||||
u := fmt.Sprintf("permanent%d", i)
|
||||
permanentRelays = append(permanentRelays, &relay{URL: u})
|
||||
}
|
||||
|
||||
knownRelays = []*relay{
|
||||
{URL: "known1"},
|
||||
{URL: "known2"},
|
||||
{URL: "known3"},
|
||||
}
|
||||
|
||||
mut = new(sync.RWMutex)
|
||||
}
|
||||
|
||||
// Regression test: handleGetRequest should not modify permanentRelays.
|
||||
func TestHandleGetRequest(t *testing.T) {
|
||||
needcap := len(permanentRelays) + len(knownRelays)
|
||||
if needcap > cap(permanentRelays) {
|
||||
t.Fatalf("test setup failed: need cap(permanentRelays) >= %d, have %d",
|
||||
needcap, cap(permanentRelays))
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
w.Body = new(bytes.Buffer)
|
||||
handleEndpointFull(w, httptest.NewRequest("GET", "/", nil))
|
||||
|
||||
result := make(map[string][]*relay)
|
||||
err := json.NewDecoder(w.Body).Decode(&result)
|
||||
if err != nil {
|
||||
t.Fatalf("invalid JSON: %v", err)
|
||||
}
|
||||
|
||||
relays := result["relays"]
|
||||
expect, actual := len(knownRelays)+len(permanentRelays), len(relays)
|
||||
if actual != expect {
|
||||
t.Errorf("expected %d relays, got %d", expect, actual)
|
||||
}
|
||||
|
||||
// Check for changes in permanentRelays.
|
||||
for i, r := range permanentRelays {
|
||||
switch {
|
||||
case !strings.HasPrefix(r.URL, "permanent"):
|
||||
t.Errorf("relay %q among permanent relays", r.URL)
|
||||
case r.URL != fmt.Sprintf("permanent%d", i):
|
||||
t.Error("order of permanent relays changed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalizeQueryValues(t *testing.T) {
|
||||
// This just demonstrates and validates the uri.Parse/String stuff in
|
||||
// regards to query strings.
|
||||
|
||||
in := "http://example.com/?some weird= query^value"
|
||||
exp := "http://example.com/?some+weird=+query%5Evalue"
|
||||
|
||||
uri, err := url.Parse(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
str := uri.String()
|
||||
if str != in {
|
||||
// Just re-encoding the URL doesn't sanitize the query string.
|
||||
t.Errorf("expected %q, got %q", in, str)
|
||||
}
|
||||
|
||||
uri.RawQuery = uri.Query().Encode()
|
||||
str = uri.String()
|
||||
if str != exp {
|
||||
// The query string is now in correct format.
|
||||
t.Errorf("expected %q, got %q", exp, str)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlimURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, out string
|
||||
}{
|
||||
{"http://example.com/", "http://example.com/"},
|
||||
{"relay://192.0.2.42:22067/?globalLimitBps=0&id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M&networkTimeout=2m0s&pingInterval=1m0s&providedBy=Test&sessionLimitBps=0&statusAddr=%3A22070", "relay://192.0.2.42:22067/?id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
if got := slimURL(c.in); got != c.out {
|
||||
t.Errorf("expected %q, got %q", c.out, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
// Copyright (C) 2018 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
)
|
||||
|
||||
var (
|
||||
statusClient = http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
apiRequestsTotal = makeCounter("api_requests_total", "Number of API requests.", "type", "result")
|
||||
apiRequestsSeconds = makeSummary("api_requests_seconds", "Latency of API requests.", "type")
|
||||
|
||||
relayTestsTotal = makeCounter("tests_total", "Number of relay tests.", "result")
|
||||
relayTestActionsSeconds = makeSummary("test_actions_seconds", "Latency of relay test actions.", "type")
|
||||
|
||||
locationLookupSeconds = makeSummary("location_lookup_seconds", "Latency of location lookups.").WithLabelValues()
|
||||
|
||||
metricsRequestsSeconds = makeSummary("metrics_requests_seconds", "Latency of metric requests.").WithLabelValues()
|
||||
scrapeSeconds = makeSummary("relay_scrape_seconds", "Latency of metric scrapes from remote relays.", "result")
|
||||
|
||||
relayUptime = makeGauge("relay_uptime", "Uptime of relay", "relay")
|
||||
relayPendingSessionKeys = makeGauge("relay_pending_session_keys", "Number of pending session keys (two keys per session, one per each side of the connection)", "relay")
|
||||
relayActiveSessions = makeGauge("relay_active_sessions", "Number of sessions that are happening, a session contains two parties", "relay")
|
||||
relayConnections = makeGauge("relay_connections", "Number of devices connected to the relay", "relay")
|
||||
relayProxies = makeGauge("relay_proxies", "Number of active proxy routines sending data between peers (two proxies per session, one for each way)", "relay")
|
||||
relayBytesProxied = makeGauge("relay_bytes_proxied", "Number of bytes proxied by the relay", "relay")
|
||||
relayGoRoutines = makeGauge("relay_go_routines", "Number of Go routines in the process", "relay")
|
||||
relaySessionRate = makeGauge("relay_session_rate", "Rate applied per session", "relay")
|
||||
relayGlobalRate = makeGauge("relay_global_rate", "Global rate applied on the whole relay", "relay")
|
||||
relayBuildInfo = makeGauge("relay_build_info", "Build information about a relay", "relay", "go_version", "go_os", "go_arch")
|
||||
relayLocationInfo = makeGauge("relay_location_info", "Location information about a relay", "relay", "city", "country", "continent")
|
||||
|
||||
lastStats = make(map[string]stats)
|
||||
)
|
||||
|
||||
func makeGauge(name string, help string, labels ...string) *prometheus.GaugeVec {
|
||||
gauge := prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "relaypoolsrv",
|
||||
Name: name,
|
||||
Help: help,
|
||||
},
|
||||
labels,
|
||||
)
|
||||
prometheus.MustRegister(gauge)
|
||||
return gauge
|
||||
}
|
||||
|
||||
func makeSummary(name string, help string, labels ...string) *prometheus.SummaryVec {
|
||||
summary := prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "relaypoolsrv",
|
||||
Name: name,
|
||||
Help: help,
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
},
|
||||
labels,
|
||||
)
|
||||
prometheus.MustRegister(summary)
|
||||
return summary
|
||||
}
|
||||
|
||||
func makeCounter(name string, help string, labels ...string) *prometheus.CounterVec {
|
||||
counter := prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "relaypoolsrv",
|
||||
Name: name,
|
||||
Help: help,
|
||||
},
|
||||
labels,
|
||||
)
|
||||
prometheus.MustRegister(counter)
|
||||
return counter
|
||||
}
|
||||
|
||||
func statsRefresher(interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
for range ticker.C {
|
||||
refreshStats()
|
||||
}
|
||||
}
|
||||
|
||||
type statsFetchResult struct {
|
||||
relay *relay
|
||||
stats *stats
|
||||
}
|
||||
|
||||
func refreshStats() {
|
||||
mut.RLock()
|
||||
relays := append(permanentRelays, knownRelays...)
|
||||
mut.RUnlock()
|
||||
|
||||
now := time.Now()
|
||||
wg := sync.NewWaitGroup()
|
||||
|
||||
results := make(chan statsFetchResult, len(relays))
|
||||
for _, rel := range relays {
|
||||
wg.Add(1)
|
||||
go func(rel *relay) {
|
||||
t0 := time.Now()
|
||||
stats := fetchStats(rel)
|
||||
duration := time.Since(t0).Seconds()
|
||||
result := "success"
|
||||
if stats == nil {
|
||||
result = "failed"
|
||||
}
|
||||
scrapeSeconds.WithLabelValues(result).Observe(duration)
|
||||
|
||||
results <- statsFetchResult{
|
||||
relay: rel,
|
||||
stats: fetchStats(rel),
|
||||
}
|
||||
wg.Done()
|
||||
}(rel)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
mut.Lock()
|
||||
relayBuildInfo.Reset()
|
||||
relayLocationInfo.Reset()
|
||||
for result := range results {
|
||||
result.relay.StatsRetrieved = now
|
||||
result.relay.Stats = result.stats
|
||||
if result.stats == nil {
|
||||
deleteMetrics(result.relay.uri.Host)
|
||||
} else {
|
||||
updateMetrics(result.relay.uri.Host, *result.stats, result.relay.Location)
|
||||
}
|
||||
}
|
||||
mut.Unlock()
|
||||
}
|
||||
|
||||
func fetchStats(relay *relay) *stats {
|
||||
statusAddr := relay.uri.Query().Get("statusAddr")
|
||||
if statusAddr == "" {
|
||||
statusAddr = ":22070"
|
||||
}
|
||||
|
||||
statusHost, statusPort, err := net.SplitHostPort(statusAddr)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if statusHost == "" {
|
||||
if host, _, err := net.SplitHostPort(relay.uri.Host); err != nil {
|
||||
return nil
|
||||
} else {
|
||||
statusHost = host
|
||||
}
|
||||
}
|
||||
|
||||
url := "http://" + net.JoinHostPort(statusHost, statusPort) + "/status"
|
||||
|
||||
response, err := statusClient.Get(url)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var stats stats
|
||||
|
||||
if json.NewDecoder(response.Body).Decode(&stats); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &stats
|
||||
}
|
||||
|
||||
func updateMetrics(host string, stats stats, location location) {
|
||||
if stats.GoVersion != "" || stats.GoOS != "" || stats.GoArch != "" {
|
||||
relayBuildInfo.WithLabelValues(host, stats.GoVersion, stats.GoOS, stats.GoArch).Add(1)
|
||||
}
|
||||
if location.City != "" || location.Country != "" || location.Continent != "" {
|
||||
relayLocationInfo.WithLabelValues(host, location.City, location.Country, location.Continent).Add(1)
|
||||
}
|
||||
|
||||
if lastStat, ok := lastStats[host]; ok {
|
||||
stats = mergeStats(stats, lastStat)
|
||||
}
|
||||
|
||||
relayUptime.WithLabelValues(host).Set(float64(stats.UptimeSeconds))
|
||||
relayPendingSessionKeys.WithLabelValues(host).Set(float64(stats.PendingSessionKeys))
|
||||
relayActiveSessions.WithLabelValues(host).Set(float64(stats.ActiveSessions))
|
||||
relayConnections.WithLabelValues(host).Set(float64(stats.Connections))
|
||||
relayProxies.WithLabelValues(host).Set(float64(stats.Proxies))
|
||||
relayBytesProxied.WithLabelValues(host).Set(float64(stats.BytesProxied))
|
||||
relayGoRoutines.WithLabelValues(host).Set(float64(stats.GoRoutines))
|
||||
relaySessionRate.WithLabelValues(host).Set(float64(stats.Options.SessionRate))
|
||||
relayGlobalRate.WithLabelValues(host).Set(float64(stats.Options.GlobalRate))
|
||||
lastStats[host] = stats
|
||||
}
|
||||
|
||||
func deleteMetrics(host string) {
|
||||
relayUptime.DeleteLabelValues(host)
|
||||
relayPendingSessionKeys.DeleteLabelValues(host)
|
||||
relayActiveSessions.DeleteLabelValues(host)
|
||||
relayConnections.DeleteLabelValues(host)
|
||||
relayProxies.DeleteLabelValues(host)
|
||||
relayBytesProxied.DeleteLabelValues(host)
|
||||
relayGoRoutines.DeleteLabelValues(host)
|
||||
relaySessionRate.DeleteLabelValues(host)
|
||||
relayGlobalRate.DeleteLabelValues(host)
|
||||
delete(lastStats, host)
|
||||
}
|
||||
|
||||
// Due to some unexplainable behaviour, some of the numbers sometimes travel slightly backwards (by less than 1%)
|
||||
// This happens between scrapes, which is 30s, so this can't be a race.
|
||||
// This causes prometheus to assume a "rate reset", hence causes phenomenal spikes.
|
||||
// One of the number that moves backwards is BytesProxied, which atomically increments a counter with numeric value
|
||||
// returned by net.Conn.Read(). I don't think that can return a negative value, so I have no idea what's going on.
|
||||
func mergeStats(new stats, old stats) stats {
|
||||
new.UptimeSeconds = mergeValue(new.UptimeSeconds, old.UptimeSeconds)
|
||||
new.PendingSessionKeys = mergeValue(new.PendingSessionKeys, old.PendingSessionKeys)
|
||||
new.ActiveSessions = mergeValue(new.ActiveSessions, old.ActiveSessions)
|
||||
new.Connections = mergeValue(new.Connections, old.Connections)
|
||||
new.Proxies = mergeValue(new.Proxies, old.Proxies)
|
||||
new.BytesProxied = mergeValue(new.BytesProxied, old.BytesProxied)
|
||||
new.GoRoutines = mergeValue(new.GoRoutines, old.GoRoutines)
|
||||
new.Options.SessionRate = mergeValue(new.Options.SessionRate, old.Options.SessionRate)
|
||||
new.Options.GlobalRate = mergeValue(new.Options.GlobalRate, old.Options.GlobalRate)
|
||||
return new
|
||||
}
|
||||
|
||||
func mergeValue(new, old int) int {
|
||||
if new >= old {
|
||||
return new // normal increase
|
||||
}
|
||||
if float64(new) > 0.99*float64(old) {
|
||||
return old // slight backward movement
|
||||
}
|
||||
return new // reset (relay restart)
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMerge(t *testing.T) {
|
||||
if mergeValue(1001, 1000) != 1001 {
|
||||
t.Error("the computer says no")
|
||||
}
|
||||
|
||||
if mergeValue(999, 1000) != 1000 {
|
||||
t.Error("the computer says no")
|
||||
}
|
||||
|
||||
if mergeValue(1, 1000) != 1 {
|
||||
t.Error("the computer says no")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user