felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/ur: Normalise contract between syncthing and ursrv (#6770)

Browse Source 
* Fix ui, hide report date

* Undo Goland madness

* UR now web scale

* Fix migration

* Fix marshaling, force tick on start

* Fix tests

* Darwin build

* Split "all" build target, add package name as a tag

* Remove pq and sql dep from syncthing, split build targets

* Empty line

* Revert "Empty line"

This reverts commit f74af2b067.

* Revert "Remove pq and sql dep from syncthing, split build targets"

This reverts commit 8fc295ad00.

* Revert "Split "all" build target, add package name as a tag"

This reverts commit f4dc889951.

* Normalise contract types

* Fix build add more logging
This commit is contained in:
Audrius Butkevicius
2020-06-23 09:47:15 +01:00
committed by GitHub
parent 72f954dcab
commit 689cf2a5ee
18 changed files with 976 additions and 855 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/uraggregate/main.go
+26 -26
View File
@@ -175,13 +175,13 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
res, err := db.Exec(`INSERT INTO VersionSummary (
SELECT
DATE_TRUNC('day', Received) AS Day,
SUBSTRING(Version FROM '^v\d.\d+') AS Ver,
SUBSTRING(Report->>'version' FROM '^v\d.\d+') AS Ver,
COUNT(*) AS Count
FROM Reports
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
AND Report->>'version' like 'v_.%'
GROUP BY Day, Ver
);
`, since)
@@ -195,11 +195,11 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
func aggregateUserMovement(db *sql.DB) (int64, error) {
rows, err := db.Query(`SELECT
DATE_TRUNC('day', Received) AS Day,
UniqueID
FROM Reports
Report->>'uniqueID'
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
AND Report->>'version' like 'v_.%'
ORDER BY Day
`)
if err != nil {
@@ -276,16 +276,16 @@ func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
res, err := db.Exec(`INSERT INTO Performance (
SELECT
DATE_TRUNC('day', Received) AS Day,
AVG(TotFiles) As TotFiles,
AVG(TotMiB) As TotMiB,
AVG(SHA256Perf) As SHA256Perf,
AVG(MemorySize) As MemorySize,
AVG(MemoryUsageMiB) As MemoryUsageMiB
FROM Reports
AVG((Report->>'totFiles')::numeric) As TotFiles,
AVG((Report->>'totMiB')::numeric) As TotMiB,
AVG((Report->>'sha256Perf')::numeric) As SHA256Perf,
AVG((Report->>'memorySize')::numeric) As MemorySize,
AVG((Report->>'memoryUsageMiB')::numeric) As MemoryUsageMiB
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
AND Report->>'version' like 'v_.%'
GROUP BY Day
);
`, since)
@@ -303,22 +303,22 @@ func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
SELECT
DATE_TRUNC('day', Received) AS Day,
COUNT(1) As Reports,
SUM(BlocksTotal) AS Total,
SUM(BlocksRenamed) AS Renamed,
SUM(BlocksReused) AS Reused,
SUM(BlocksPulled) AS Pulled,
SUM(BlocksCopyOrigin) AS CopyOrigin,
SUM(BlocksCopyOriginShifted) AS CopyOriginShifted,
SUM(BlocksCopyElsewhere) AS CopyElsewhere
FROM Reports
SUM((Report->'blockStats'->>'total')::numeric) AS Total,
SUM((Report->'blockStats'->>'renamed')::numeric) AS Renamed,
SUM((Report->'blockStats'->>'reused')::numeric) AS Reused,
SUM((Report->'blockStats'->>'pulled')::numeric) AS Pulled,
SUM((Report->'blockStats'->>'copyOrigin')::numeric) AS CopyOrigin,
SUM((Report->'blockStats'->>'copyOriginShifted')::numeric) AS CopyOriginShifted,
SUM((Report->'blockStats'->>'copyElsewhere')::numeric) AS CopyElsewhere
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND ReportVersion = 3
AND Version like 'v_.%'
AND Version NOT LIKE 'v0.14.40%'
AND Version NOT LIKE 'v0.14.39%'
AND Version NOT LIKE 'v0.14.38%'
AND (Report->>'urVersion')::numeric >= 3
AND Report->>'version' like 'v_.%'
AND Report->>'version' NOT LIKE 'v0.14.40%'
AND Report->>'version' NOT LIKE 'v0.14.39%'
AND Report->>'version' NOT LIKE 'v0.14.38%'
GROUP BY Day
);
`, since)
cmd/ursrv/analytics.go
+17
View File
@@ -114,6 +114,23 @@ func statsForInts(data []int) [4]float64 {
return res
}
func statsForInt64s(data []int64) [4]float64 {
var res [4]float64
if len(data) == 0 {
return res
}
sort.Slice(data, func(a, b int) bool {
return data[a] < data[b]
})
res[0] = float64(data[int(float64(len(data))*0.05)])
res[1] = float64(data[len(data)/2])
res[2] = float64(data[int(float64(len(data))*0.95)])
res[3] = float64(data[len(data)-1])
return res
}
func statsForFloats(data []float64) [4]float64 {
var res [4]float64
if len(data) == 0 {
cmd/ursrv/main.go
+35 -578
View File
@@ -10,10 +10,7 @@ import (
"bytes"
"crypto/tls"
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
"io/ioutil"
@@ -29,8 +26,9 @@ import (
"time"
"unicode"
"github.com/lib/pq"
geoip2 "github.com/oschwald/geoip2-golang"
"github.com/oschwald/geoip2-golang"
"github.com/syncthing/syncthing/lib/ur/contract"
)
var (
@@ -103,585 +101,44 @@ func getEnvDefault(key, def string) string {
return def
}
type IntMap map[string]int
func (p IntMap) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *IntMap) Scan(src interface{}) error {
source, ok := src.([]byte)
if !ok {
return errors.New("Type assertion .([]byte) failed.")
}
var i map[string]int
err := json.Unmarshal(source, &i)
if err != nil {
return err
}
*p = i
return nil
}
type report struct {
Received time.Time // Only from DB
UniqueID string
Version string
LongVersion string
Platform string
NumFolders int
NumDevices int
TotFiles int
FolderMaxFiles int
TotMiB int
FolderMaxMiB int
MemoryUsageMiB int
SHA256Perf float64
MemorySize int
// v2 fields
URVersion int
NumCPU int
FolderUses struct {
SendOnly int
ReceiveOnly int
IgnorePerms int
IgnoreDelete int
AutoNormalize int
SimpleVersioning int
ExternalVersioning int
StaggeredVersioning int
TrashcanVersioning int
}
DeviceUses struct {
Introducer int
CustomCertName int
CompressAlways int
CompressMetadata int
CompressNever int
DynamicAddr int
StaticAddr int
}
Announce struct {
GlobalEnabled bool
LocalEnabled bool
DefaultServersDNS int
DefaultServersIP int
OtherServers int
}
Relays struct {
Enabled bool
DefaultServers int
OtherServers int
}
UsesRateLimit bool
UpgradeAllowedManual bool
UpgradeAllowedAuto bool
// V2.5 fields (fields that were in v2 but never added to the database
UpgradeAllowedPre bool
RescanIntvs pq.Int64Array
// v3 fields
Uptime int
NATType string
AlwaysLocalNets bool
CacheIgnoredFiles bool
OverwriteRemoteDeviceNames bool
ProgressEmitterEnabled bool
CustomDefaultFolderPath bool
WeakHashSelection string
CustomTrafficClass bool
CustomTempIndexMinBlocks bool
TemporariesDisabled bool
TemporariesCustom bool
LimitBandwidthInLan bool
CustomReleaseURL bool
RestartOnWakeup bool
CustomStunServers bool
FolderUsesV3 struct {
ScanProgressDisabled int
ConflictsDisabled int
ConflictsUnlimited int
ConflictsOther int
DisableSparseFiles int
DisableTempIndexes int
AlwaysWeakHash int
CustomWeakHashThreshold int
FsWatcherEnabled int
PullOrder IntMap
FilesystemType IntMap
FsWatcherDelays pq.Int64Array
}
GUIStats struct {
Enabled int
UseTLS int
UseAuth int
InsecureAdminAccess int
Debugging int
InsecureSkipHostCheck int
InsecureAllowFrameLoading int
ListenLocal int
ListenUnspecified int
Theme IntMap
}
BlockStats struct {
Total int
Renamed int
Reused int
Pulled int
CopyOrigin int
CopyOriginShifted int
CopyElsewhere int
}
TransportStats IntMap
IgnoreStats struct {
Lines int
Inverts int
Folded int
Deletable int
Rooted int
Includes int
EscapedIncludes int
DoubleStars int
Stars int
}
// V3 fields added late in the RC
WeakHashEnabled bool
// Generated
Date string
Address string
}
func (r *report) Validate() error {
if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
return errors.New("missing required field")
}
if len(r.Date) != 8 {
return errors.New("date not initialized")
}
// Some fields may not be null.
if r.RescanIntvs == nil {
r.RescanIntvs = []int64{}
}
if r.FolderUsesV3.FsWatcherDelays == nil {
r.FolderUsesV3.FsWatcherDelays = []int64{}
}
return nil
}
func (r *report) FieldPointers() []interface{} {
// All the fields of the report, in the same order as the database fields.
return []interface{}{
&r.Received, &r.UniqueID, &r.Version, &r.LongVersion, &r.Platform,
&r.NumFolders, &r.NumDevices, &r.TotFiles, &r.FolderMaxFiles,
&r.TotMiB, &r.FolderMaxMiB, &r.MemoryUsageMiB, &r.SHA256Perf,
&r.MemorySize, &r.Date,
// V2
&r.URVersion, &r.NumCPU, &r.FolderUses.SendOnly, &r.FolderUses.IgnorePerms,
&r.FolderUses.IgnoreDelete, &r.FolderUses.AutoNormalize, &r.DeviceUses.Introducer,
&r.DeviceUses.CustomCertName, &r.DeviceUses.CompressAlways,
&r.DeviceUses.CompressMetadata, &r.DeviceUses.CompressNever,
&r.DeviceUses.DynamicAddr, &r.DeviceUses.StaticAddr,
&r.Announce.GlobalEnabled, &r.Announce.LocalEnabled,
&r.Announce.DefaultServersDNS, &r.Announce.DefaultServersIP,
&r.Announce.OtherServers, &r.Relays.Enabled, &r.Relays.DefaultServers,
&r.Relays.OtherServers, &r.UsesRateLimit, &r.UpgradeAllowedManual,
&r.UpgradeAllowedAuto, &r.FolderUses.SimpleVersioning,
&r.FolderUses.ExternalVersioning, &r.FolderUses.StaggeredVersioning,
&r.FolderUses.TrashcanVersioning,
// V2.5
&r.UpgradeAllowedPre, &r.RescanIntvs,
// V3
&r.Uptime, &r.NATType, &r.AlwaysLocalNets, &r.CacheIgnoredFiles,
&r.OverwriteRemoteDeviceNames, &r.ProgressEmitterEnabled, &r.CustomDefaultFolderPath,
&r.WeakHashSelection, &r.CustomTrafficClass, &r.CustomTempIndexMinBlocks,
&r.TemporariesDisabled, &r.TemporariesCustom, &r.LimitBandwidthInLan,
&r.CustomReleaseURL, &r.RestartOnWakeup, &r.CustomStunServers,
&r.FolderUsesV3.ScanProgressDisabled, &r.FolderUsesV3.ConflictsDisabled,
&r.FolderUsesV3.ConflictsUnlimited, &r.FolderUsesV3.ConflictsOther,
&r.FolderUsesV3.DisableSparseFiles, &r.FolderUsesV3.DisableTempIndexes,
&r.FolderUsesV3.AlwaysWeakHash, &r.FolderUsesV3.CustomWeakHashThreshold,
&r.FolderUsesV3.FsWatcherEnabled,
&r.FolderUsesV3.PullOrder, &r.FolderUsesV3.FilesystemType,
&r.FolderUsesV3.FsWatcherDelays,
&r.GUIStats.Enabled, &r.GUIStats.UseTLS, &r.GUIStats.UseAuth,
&r.GUIStats.InsecureAdminAccess,
&r.GUIStats.Debugging, &r.GUIStats.InsecureSkipHostCheck,
&r.GUIStats.InsecureAllowFrameLoading, &r.GUIStats.ListenLocal,
&r.GUIStats.ListenUnspecified, &r.GUIStats.Theme,
&r.BlockStats.Total, &r.BlockStats.Renamed,
&r.BlockStats.Reused, &r.BlockStats.Pulled, &r.BlockStats.CopyOrigin,
&r.BlockStats.CopyOriginShifted, &r.BlockStats.CopyElsewhere,
&r.TransportStats,
&r.IgnoreStats.Lines, &r.IgnoreStats.Inverts, &r.IgnoreStats.Folded,
&r.IgnoreStats.Deletable, &r.IgnoreStats.Rooted, &r.IgnoreStats.Includes,
&r.IgnoreStats.EscapedIncludes, &r.IgnoreStats.DoubleStars, &r.IgnoreStats.Stars,
// V3 added late in the RC
&r.WeakHashEnabled,
&r.Address,
// Receive only folders
&r.FolderUses.ReceiveOnly,
}
}
func (r *report) FieldNames() []string {
// The database fields that back this struct in PostgreSQL
return []string{
// V1
"Received",
"UniqueID",
"Version",
"LongVersion",
"Platform",
"NumFolders",
"NumDevices",
"TotFiles",
"FolderMaxFiles",
"TotMiB",
"FolderMaxMiB",
"MemoryUsageMiB",
"SHA256Perf",
"MemorySize",
"Date",
// V2
"ReportVersion",
"NumCPU",
"FolderRO",
"FolderIgnorePerms",
"FolderIgnoreDelete",
"FolderAutoNormalize",
"DeviceIntroducer",
"DeviceCustomCertName",
"DeviceCompressAlways",
"DeviceCompressMetadata",
"DeviceCompressNever",
"DeviceDynamicAddr",
"DeviceStaticAddr",
"AnnounceGlobalEnabled",
"AnnounceLocalEnabled",
"AnnounceDefaultServersDNS",
"AnnounceDefaultServersIP",
"AnnounceOtherServers",
"RelayEnabled",
"RelayDefaultServers",
"RelayOtherServers",
"RateLimitEnabled",
"UpgradeAllowedManual",
"UpgradeAllowedAuto",
// v0.12.19+
"FolderSimpleVersioning",
"FolderExternalVersioning",
"FolderStaggeredVersioning",
"FolderTrashcanVersioning",
// V2.5
"UpgradeAllowedPre",
"RescanIntvs",
// V3
"Uptime",
"NATType",
"AlwaysLocalNets",
"CacheIgnoredFiles",
"OverwriteRemoteDeviceNames",
"ProgressEmitterEnabled",
"CustomDefaultFolderPath",
"WeakHashSelection",
"CustomTrafficClass",
"CustomTempIndexMinBlocks",
"TemporariesDisabled",
"TemporariesCustom",
"LimitBandwidthInLan",
"CustomReleaseURL",
"RestartOnWakeup",
"CustomStunServers",
"FolderScanProgressDisabled",
"FolderConflictsDisabled",
"FolderConflictsUnlimited",
"FolderConflictsOther",
"FolderDisableSparseFiles",
"FolderDisableTempIndexes",
"FolderAlwaysWeakHash",
"FolderCustomWeakHashThreshold",
"FolderFsWatcherEnabled",
"FolderPullOrder",
"FolderFilesystemType",
"FolderFsWatcherDelays",
"GUIEnabled",
"GUIUseTLS",
"GUIUseAuth",
"GUIInsecureAdminAccess",
"GUIDebugging",
"GUIInsecureSkipHostCheck",
"GUIInsecureAllowFrameLoading",
"GUIListenLocal",
"GUIListenUnspecified",
"GUITheme",
"BlocksTotal",
"BlocksRenamed",
"BlocksReused",
"BlocksPulled",
"BlocksCopyOrigin",
"BlocksCopyOriginShifted",
"BlocksCopyElsewhere",
"Transport",
"IgnoreLines",
"IgnoreInverts",
"IgnoreFolded",
"IgnoreDeletable",
"IgnoreRooted",
"IgnoreIncludes",
"IgnoreEscapedIncludes",
"IgnoreDoubleStars",
"IgnoreStars",
// V3 added late in the RC
"WeakHashEnabled",
"Address",
// Receive only folders
"FolderRecvOnly",
}
}
func setupDB(db *sql.DB) error {
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS Reports (
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS ReportsJson (
Received TIMESTAMP NOT NULL,
UniqueID VARCHAR(32) NOT NULL,
Version VARCHAR(32) NOT NULL,
LongVersion VARCHAR(256) NOT NULL,
Platform VARCHAR(32) NOT NULL,
NumFolders INTEGER NOT NULL,
NumDevices INTEGER NOT NULL,
TotFiles INTEGER NOT NULL,
FolderMaxFiles INTEGER NOT NULL,
TotMiB INTEGER NOT NULL,
FolderMaxMiB INTEGER NOT NULL,
MemoryUsageMiB INTEGER NOT NULL,
SHA256Perf DOUBLE PRECISION NOT NULL,
MemorySize INTEGER NOT NULL,
Date VARCHAR(8) NOT NULL
Report JSONB NOT NULL
)`)
if err != nil {
return err
}
var t string
row := db.QueryRow(`SELECT 'UniqueIDIndex'::regclass`)
if err := row.Scan(&t); err != nil {
if _, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDIndex ON Reports (Date, UniqueID)`); err != nil {
if err := db.QueryRow(`SELECT 'UniqueIDJsonIndex'::regclass`).Scan(&t); err != nil {
if _, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDJsonIndex ON ReportsJson ((Report->>'date'), (Report->>'uniqueID'))`); err != nil {
return err
}
}
row = db.QueryRow(`SELECT 'ReceivedIndex'::regclass`)
if err := row.Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReceivedIndex ON Reports (Received)`); err != nil {
if err := db.QueryRow(`SELECT 'ReceivedJsonIndex'::regclass`).Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReceivedJsonIndex ON ReportsJson (Received)`); err != nil {
return err
}
}
// V2
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'reportversion'`)
if err := row.Scan(&t); err != nil {
// The ReportVersion column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN ReportVersion INTEGER NOT NULL DEFAULT 0,
ADD COLUMN NumCPU INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderRO INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderIgnorePerms INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderIgnoreDelete INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderAutoNormalize INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceIntroducer INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCustomCertName INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCompressAlways INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCompressMetadata INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCompressNever INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceDynamicAddr INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceStaticAddr INTEGER NOT NULL DEFAULT 0,
ADD COLUMN AnnounceGlobalEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN AnnounceLocalEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN AnnounceDefaultServersDNS INTEGER NOT NULL DEFAULT 0,
ADD COLUMN AnnounceDefaultServersIP INTEGER NOT NULL DEFAULT 0,
ADD COLUMN AnnounceOtherServers INTEGER NOT NULL DEFAULT 0,
ADD COLUMN RelayEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN RelayDefaultServers INTEGER NOT NULL DEFAULT 0,
ADD COLUMN RelayOtherServers INTEGER NOT NULL DEFAULT 0,
ADD COLUMN RateLimitEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN UpgradeAllowedManual BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN UpgradeAllowedAuto BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN FolderSimpleVersioning INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderExternalVersioning INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderStaggeredVersioning INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderTrashcanVersioning INTEGER NOT NULL DEFAULT 0
`)
if err != nil {
if err := db.QueryRow(`SELECT 'ReportVersionJsonIndex'::regclass`).Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReportVersionJsonIndex ON ReportsJson (cast((Report->>'urVersion') as numeric))`); err != nil {
return err
}
}
row = db.QueryRow(`SELECT 'ReportVersionIndex'::regclass`)
if err := row.Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReportVersionIndex ON Reports (ReportVersion)`); err != nil {
return err
}
}
// V2.5
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'upgradeallowedpre'`)
if err := row.Scan(&t); err != nil {
// The ReportVersion column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN UpgradeAllowedPre BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN RescanIntvs INT[] NOT NULL DEFAULT '{}'
`)
if err != nil {
return err
}
}
// V3
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'uptime'`)
if err := row.Scan(&t); err != nil {
// The Uptime column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN Uptime INTEGER NOT NULL DEFAULT 0,
ADD COLUMN NATType VARCHAR(32) NOT NULL DEFAULT '',
ADD COLUMN AlwaysLocalNets BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CacheIgnoredFiles BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN OverwriteRemoteDeviceNames BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN ProgressEmitterEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomDefaultFolderPath BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN WeakHashSelection VARCHAR(32) NOT NULL DEFAULT '',
ADD COLUMN CustomTrafficClass BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomTempIndexMinBlocks BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN TemporariesDisabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN TemporariesCustom BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN LimitBandwidthInLan BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomReleaseURL BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN RestartOnWakeup BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomStunServers BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN FolderScanProgressDisabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderConflictsDisabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderConflictsUnlimited INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderConflictsOther INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderDisableSparseFiles INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderDisableTempIndexes INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderAlwaysWeakHash INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderCustomWeakHashThreshold INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderFsWatcherEnabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderPullOrder JSONB NOT NULL DEFAULT '{}',
ADD COLUMN FolderFilesystemType JSONB NOT NULL DEFAULT '{}',
ADD COLUMN FolderFsWatcherDelays INT[] NOT NULL DEFAULT '{}',
ADD COLUMN GUIEnabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIUseTLS INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIUseAuth INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIInsecureAdminAccess INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIDebugging INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIInsecureSkipHostCheck INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIInsecureAllowFrameLoading INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIListenLocal INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIListenUnspecified INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUITheme JSONB NOT NULL DEFAULT '{}',
ADD COLUMN BlocksTotal INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksRenamed INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksReused INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksPulled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksCopyOrigin INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksCopyOriginShifted INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksCopyElsewhere INTEGER NOT NULL DEFAULT 0,
ADD COLUMN Transport JSONB NOT NULL DEFAULT '{}',
ADD COLUMN IgnoreLines INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreInverts INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreFolded INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreDeletable INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreRooted INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreIncludes INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreEscapedIncludes INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreDoubleStars INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreStars INTEGER NOT NULL DEFAULT 0
`)
if err != nil {
return err
}
}
// V3 added late in the RC
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'weakhashenabled'`)
if err := row.Scan(&t); err != nil {
// The WeakHashEnabled column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN WeakHashEnabled BOOLEAN NOT NULL DEFAULT FALSE
ADD COLUMN Address VARCHAR(45) NOT NULL DEFAULT ''
`)
if err != nil {
return err
}
}
// Receive only added ad-hoc
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'folderrecvonly'`)
if err := row.Scan(&t); err != nil {
// The RecvOnly column doesn't exist; add it.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN FolderRecvOnly INTEGER NOT NULL DEFAULT 0
`)
if err != nil {
return err
}
// Migrate from old schema to new schema if the table exists.
if err := migrate(db); err != nil {
return err
}
return nil
}
func insertReport(db *sql.DB, r report) error {
r.Received = time.Now().UTC()
fields := r.FieldPointers()
params := make([]string, len(fields))
for i := range params {
params[i] = fmt.Sprintf("$%d", i+1)
}
query := "INSERT INTO Reports (" + strings.Join(r.FieldNames(), ", ") + ") VALUES (" + strings.Join(params, ", ") + ")"
_, err := db.Exec(query, fields...)
func insertReport(db *sql.DB, r contract.Report) error {
_, err := db.Exec("INSERT INTO ReportsJson (Report, Received) VALUES ($1, $2)", r, time.Now().UTC())
return err
}
@@ -689,9 +146,9 @@ func insertReport(db *sql.DB, r report) error {
type withDBFunc func(*sql.DB, http.ResponseWriter, *http.Request)
func withDB(db *sql.DB, f withDBFunc) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
f(db, w, r)
})
}
}
func main() {
@@ -778,7 +235,7 @@ const maxCacheTime = 15 * time.Minute
func cacheRefresher(db *sql.DB) {
ticker := time.NewTicker(maxCacheTime - time.Minute)
defer ticker.Stop()
for range ticker.C {
for ; true; <-ticker.C {
cacheMut.Lock()
if err := refreshCacheLocked(db); err != nil {
log.Println(err)
@@ -861,7 +318,7 @@ func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
addr = ""
}
var rep report
var rep contract.Report
rep.Date = time.Now().UTC().Format("20060102")
rep.Address = addr
@@ -1069,11 +526,11 @@ func getReport(db *sql.DB) map[string]interface{} {
var numDevices []int
var totFiles []int
var maxFiles []int
var totMiB []int
var maxMiB []int
var memoryUsage []int
var totMiB []int64
var maxMiB []int64
var memoryUsage []int64
var sha256Perf []float64
var memorySize []int
var memorySize []int64
var uptime []int
var compilers []string
var builders []string
@@ -1112,9 +569,9 @@ func getReport(db *sql.DB) map[string]interface{} {
var numCPU []int
var rep report
var rep contract.Report
rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ",") + ` FROM Reports WHERE Received > now() - '1 day'::INTERVAL`)
rows, err := db.Query(`SELECT Received, Report FROM ReportsJson WHERE Received > now() - '1 day'::INTERVAL`)
if err != nil {
log.Println("sql:", err)
return nil
@@ -1122,7 +579,7 @@ func getReport(db *sql.DB) map[string]interface{} {
defer rows.Close()
for rows.Next() {
err := rows.Scan(rep.FieldPointers()...)
err := rows.Scan(&rep.Received, &rep)
if err != nil {
log.Println("sql:", err)
@@ -1173,19 +630,19 @@ func getReport(db *sql.DB) map[string]interface{} {
maxFiles = append(maxFiles, rep.FolderMaxFiles)
}
if rep.TotMiB > 0 {
totMiB = append(totMiB, rep.TotMiB*(1<<20))
totMiB = append(totMiB, int64(rep.TotMiB)*(1<<20))
}
if rep.FolderMaxMiB > 0 {
maxMiB = append(maxMiB, rep.FolderMaxMiB*(1<<20))
maxMiB = append(maxMiB, int64(rep.FolderMaxMiB)*(1<<20))
}
if rep.MemoryUsageMiB > 0 {
memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20))
memoryUsage = append(memoryUsage, int64(rep.MemoryUsageMiB)*(1<<20))
}
if rep.SHA256Perf > 0 {
sha256Perf = append(sha256Perf, rep.SHA256Perf*(1<<20))
}
if rep.MemorySize > 0 {
memorySize = append(memorySize, rep.MemorySize*(1<<20))
memorySize = append(memorySize, int64(rep.MemorySize)*(1<<20))
}
if rep.Uptime > 0 {
uptime = append(uptime, rep.Uptime)
@@ -1336,14 +793,14 @@ func getReport(db *sql.DB) map[string]interface{} {
})
categories = append(categories, category{
Values: statsForInts(totMiB),
Values: statsForInt64s(totMiB),
Descr: "Data Managed per Device",
Unit: "B",
Type: NumberBinary,
})
categories = append(categories, category{
Values: statsForInts(maxMiB),
Values: statsForInt64s(maxMiB),
Descr: "Data in Largest Folder",
Unit: "B",
Type: NumberBinary,
@@ -1360,14 +817,14 @@ func getReport(db *sql.DB) map[string]interface{} {
})
categories = append(categories, category{
Values: statsForInts(memoryUsage),
Values: statsForInt64s(memoryUsage),
Descr: "Memory Usage",
Unit: "B",
Type: NumberBinary,
})
categories = append(categories, category{
Values: statsForInts(memorySize),
Values: statsForInt64s(memorySize),
Descr: "System Memory",
Unit: "B",
Type: NumberBinary,
cmd/ursrv/migration.go
+143
View File
@@ -0,0 +1,143 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"log"
"strings"
"github.com/lib/pq"
"github.com/syncthing/syncthing/lib/ur/contract"
)
func migrate(db *sql.DB) error {
var count uint64
log.Println("Checking old table row count, this might take a while...")
if err := db.QueryRow(`SELECT COUNT(1) FROM Reports`).Scan(&count); err != nil || count == 0 {
// err != nil most likely means table does not exist.
return nil
}
log.Printf("Found %d records, will perform migration.", count)
tx, err := db.Begin()
if err != nil {
log.Println("sql:", err)
return err
}
defer tx.Rollback()
// These must be lower case, because we don't quote them when creating, so postgres creates them lower case.
// Yet pg.CopyIn quotes them, which makes them case sensitive.
stmt, err := tx.Prepare(pq.CopyIn("reportsjson", "received", "report"))
if err != nil {
log.Println("sql:", err)
return err
}
// Custom types used in the old struct.
var rep contract.Report
var rescanIntvs pq.Int64Array
var fsWatcherDelay pq.Int64Array
pullOrder := make(IntMap)
fileSystemType := make(IntMap)
themes := make(IntMap)
transportStats := make(IntMap)
rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ", ") + `, FolderFsWatcherDelays, RescanIntvs, FolderPullOrder, FolderFilesystemType, GUITheme, Transport FROM Reports`)
if err != nil {
log.Println("sql:", err)
return err
}
defer rows.Close()
var done uint64
pct := count / 100
for rows.Next() {
err := rows.Scan(append(rep.FieldPointers(), &fsWatcherDelay, &rescanIntvs, &pullOrder, &fileSystemType, &themes, &transportStats)...)
if err != nil {
log.Println("sql scan:", err)
return err
}
// Patch up parts that used to use custom types
rep.RescanIntvs = make([]int, len(rescanIntvs))
for i := range rescanIntvs {
rep.RescanIntvs[i] = int(rescanIntvs[i])
}
rep.FolderUsesV3.FsWatcherDelays = make([]int, len(fsWatcherDelay))
for i := range fsWatcherDelay {
rep.FolderUsesV3.FsWatcherDelays[i] = int(fsWatcherDelay[i])
}
rep.FolderUsesV3.PullOrder = pullOrder
rep.FolderUsesV3.FilesystemType = fileSystemType
rep.GUIStats.Theme = themes
rep.TransportStats = transportStats
_, err = stmt.Exec(rep.Received, rep)
if err != nil {
log.Println("sql insert:", err)
return err
}
done++
if done%pct == 0 {
log.Printf("Migration progress %d/%d (%d%%)", done, count, (100*done)/count)
}
}
// Tell the driver bulk copy is finished
_, err = stmt.Exec()
if err != nil {
log.Println("sql stmt exec:", err)
return err
}
err = stmt.Close()
if err != nil {
log.Println("sql stmt close:", err)
return err
}
_, err = tx.Exec("DROP TABLE Reports")
if err != nil {
log.Println("sql drop:", err)
return err
}
err = tx.Commit()
if err != nil {
log.Println("sql commit:", err)
return err
}
return nil
}
type IntMap map[string]int
func (p IntMap) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *IntMap) Scan(src interface{}) error {
source, ok := src.([]byte)
if !ok {
return errors.New("Type assertion .([]byte) failed.")
}
var i map[string]int
err := json.Unmarshal(source, &i)
if err != nil {
return err
}
*p = i
return nil
}