From 689cf2a5ee255f1e1f53d31bff5047d96f08290d Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Tue, 23 Jun 2020 09:47:15 +0100 Subject: [PATCH] lib/ur: Normalise contract between syncthing and ursrv (#6770) * Fix ui, hide report date * Undo Goland madness * UR now web scale * Fix migration * Fix marshaling, force tick on start * Fix tests * Darwin build * Split "all" build target, add package name as a tag * Remove pq and sql dep from syncthing, split build targets * Empty line * Revert "Empty line" This reverts commit f74af2b067dadda8a343714123512bd545a643c3. * Revert "Remove pq and sql dep from syncthing, split build targets" This reverts commit 8fc295ad007c5bb7886c557f492dacf51be307ad. * Revert "Split "all" build target, add package name as a tag" This reverts commit f4dc88995106d2b06042f30bea781a0feb08e55f. * Normalise contract types * Fix build add more logging --- cmd/uraggregate/main.go | 52 +- cmd/ursrv/analytics.go | 17 + cmd/ursrv/main.go | 613 +----------------- cmd/ursrv/migration.go | 143 ++++ .../syncthing/core/syncthingController.js | 10 +- .../usageReportPreviewModalView.html | 6 +- lib/api/api.go | 19 +- lib/api/mocked_model_test.go | 4 +- lib/model/model.go | 63 +- lib/ur/contract/contract.go | 425 ++++++++++++ lib/ur/contract/contract_test.go | 130 ++++ lib/ur/memsize_darwin.go | 7 +- lib/ur/memsize_linux.go | 13 +- lib/ur/memsize_netbsd.go | 11 +- lib/ur/memsize_solaris.go | 8 +- lib/ur/memsize_unimpl.go | 6 +- lib/ur/memsize_windows.go | 8 +- lib/ur/usage_report.go | 296 ++++----- 18 files changed, 976 insertions(+), 855 deletions(-) create mode 100644 cmd/ursrv/migration.go create mode 100644 lib/ur/contract/contract.go create mode 100644 lib/ur/contract/contract_test.go diff --git a/cmd/uraggregate/main.go b/cmd/uraggregate/main.go index f2fef605c..adcdf465e 100644 --- a/cmd/uraggregate/main.go +++ b/cmd/uraggregate/main.go @@ -175,13 +175,13 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) { res, err := db.Exec(`INSERT INTO VersionSummary ( SELECT DATE_TRUNC('day', Received) AS Day, - SUBSTRING(Version FROM '^v\d.\d+') AS Ver, + SUBSTRING(Report->>'version' FROM '^v\d.\d+') AS Ver, COUNT(*) AS Count - FROM Reports + FROM ReportsJson WHERE DATE_TRUNC('day', Received) > $1 AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW()) - AND Version like 'v_.%' + AND Report->>'version' like 'v_.%' GROUP BY Day, Ver ); `, since) @@ -195,11 +195,11 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) { func aggregateUserMovement(db *sql.DB) (int64, error) { rows, err := db.Query(`SELECT DATE_TRUNC('day', Received) AS Day, - UniqueID - FROM Reports + Report->>'uniqueID' + FROM ReportsJson WHERE DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW()) - AND Version like 'v_.%' + AND Report->>'version' like 'v_.%' ORDER BY Day `) if err != nil { @@ -276,16 +276,16 @@ func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) { res, err := db.Exec(`INSERT INTO Performance ( SELECT DATE_TRUNC('day', Received) AS Day, - AVG(TotFiles) As TotFiles, - AVG(TotMiB) As TotMiB, - AVG(SHA256Perf) As SHA256Perf, - AVG(MemorySize) As MemorySize, - AVG(MemoryUsageMiB) As MemoryUsageMiB - FROM Reports + AVG((Report->>'totFiles')::numeric) As TotFiles, + AVG((Report->>'totMiB')::numeric) As TotMiB, + AVG((Report->>'sha256Perf')::numeric) As SHA256Perf, + AVG((Report->>'memorySize')::numeric) As MemorySize, + AVG((Report->>'memoryUsageMiB')::numeric) As MemoryUsageMiB + FROM ReportsJson WHERE DATE_TRUNC('day', Received) > $1 AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW()) - AND Version like 'v_.%' + AND Report->>'version' like 'v_.%' GROUP BY Day ); `, since) @@ -303,22 +303,22 @@ func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) { SELECT DATE_TRUNC('day', Received) AS Day, COUNT(1) As Reports, - SUM(BlocksTotal) AS Total, - SUM(BlocksRenamed) AS Renamed, - SUM(BlocksReused) AS Reused, - SUM(BlocksPulled) AS Pulled, - SUM(BlocksCopyOrigin) AS CopyOrigin, - SUM(BlocksCopyOriginShifted) AS CopyOriginShifted, - SUM(BlocksCopyElsewhere) AS CopyElsewhere - FROM Reports + SUM((Report->'blockStats'->>'total')::numeric) AS Total, + SUM((Report->'blockStats'->>'renamed')::numeric) AS Renamed, + SUM((Report->'blockStats'->>'reused')::numeric) AS Reused, + SUM((Report->'blockStats'->>'pulled')::numeric) AS Pulled, + SUM((Report->'blockStats'->>'copyOrigin')::numeric) AS CopyOrigin, + SUM((Report->'blockStats'->>'copyOriginShifted')::numeric) AS CopyOriginShifted, + SUM((Report->'blockStats'->>'copyElsewhere')::numeric) AS CopyElsewhere + FROM ReportsJson WHERE DATE_TRUNC('day', Received) > $1 AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW()) - AND ReportVersion = 3 - AND Version like 'v_.%' - AND Version NOT LIKE 'v0.14.40%' - AND Version NOT LIKE 'v0.14.39%' - AND Version NOT LIKE 'v0.14.38%' + AND (Report->>'urVersion')::numeric >= 3 + AND Report->>'version' like 'v_.%' + AND Report->>'version' NOT LIKE 'v0.14.40%' + AND Report->>'version' NOT LIKE 'v0.14.39%' + AND Report->>'version' NOT LIKE 'v0.14.38%' GROUP BY Day ); `, since) diff --git a/cmd/ursrv/analytics.go b/cmd/ursrv/analytics.go index d52c6c96c..0e802a06c 100644 --- a/cmd/ursrv/analytics.go +++ b/cmd/ursrv/analytics.go @@ -114,6 +114,23 @@ func statsForInts(data []int) [4]float64 { return res } +func statsForInt64s(data []int64) [4]float64 { + var res [4]float64 + if len(data) == 0 { + return res + } + + sort.Slice(data, func(a, b int) bool { + return data[a] < data[b] + }) + + res[0] = float64(data[int(float64(len(data))*0.05)]) + res[1] = float64(data[len(data)/2]) + res[2] = float64(data[int(float64(len(data))*0.95)]) + res[3] = float64(data[len(data)-1]) + return res +} + func statsForFloats(data []float64) [4]float64 { var res [4]float64 if len(data) == 0 { diff --git a/cmd/ursrv/main.go b/cmd/ursrv/main.go index 6a6a99be8..c7c16b583 100644 --- a/cmd/ursrv/main.go +++ b/cmd/ursrv/main.go @@ -10,10 +10,7 @@ import ( "bytes" "crypto/tls" "database/sql" - "database/sql/driver" "encoding/json" - "errors" - "fmt" "html/template" "io" "io/ioutil" @@ -29,8 +26,9 @@ import ( "time" "unicode" - "github.com/lib/pq" - geoip2 "github.com/oschwald/geoip2-golang" + "github.com/oschwald/geoip2-golang" + + "github.com/syncthing/syncthing/lib/ur/contract" ) var ( @@ -103,585 +101,44 @@ func getEnvDefault(key, def string) string { return def } -type IntMap map[string]int - -func (p IntMap) Value() (driver.Value, error) { - return json.Marshal(p) -} - -func (p *IntMap) Scan(src interface{}) error { - source, ok := src.([]byte) - if !ok { - return errors.New("Type assertion .([]byte) failed.") - } - - var i map[string]int - err := json.Unmarshal(source, &i) - if err != nil { - return err - } - - *p = i - return nil -} - -type report struct { - Received time.Time // Only from DB - - UniqueID string - Version string - LongVersion string - Platform string - NumFolders int - NumDevices int - TotFiles int - FolderMaxFiles int - TotMiB int - FolderMaxMiB int - MemoryUsageMiB int - SHA256Perf float64 - MemorySize int - - // v2 fields - - URVersion int - NumCPU int - FolderUses struct { - SendOnly int - ReceiveOnly int - IgnorePerms int - IgnoreDelete int - AutoNormalize int - SimpleVersioning int - ExternalVersioning int - StaggeredVersioning int - TrashcanVersioning int - } - DeviceUses struct { - Introducer int - CustomCertName int - CompressAlways int - CompressMetadata int - CompressNever int - DynamicAddr int - StaticAddr int - } - Announce struct { - GlobalEnabled bool - LocalEnabled bool - DefaultServersDNS int - DefaultServersIP int - OtherServers int - } - Relays struct { - Enabled bool - DefaultServers int - OtherServers int - } - UsesRateLimit bool - UpgradeAllowedManual bool - UpgradeAllowedAuto bool - - // V2.5 fields (fields that were in v2 but never added to the database - UpgradeAllowedPre bool - RescanIntvs pq.Int64Array - - // v3 fields - - Uptime int - NATType string - AlwaysLocalNets bool - CacheIgnoredFiles bool - OverwriteRemoteDeviceNames bool - ProgressEmitterEnabled bool - CustomDefaultFolderPath bool - WeakHashSelection string - CustomTrafficClass bool - CustomTempIndexMinBlocks bool - TemporariesDisabled bool - TemporariesCustom bool - LimitBandwidthInLan bool - CustomReleaseURL bool - RestartOnWakeup bool - CustomStunServers bool - - FolderUsesV3 struct { - ScanProgressDisabled int - ConflictsDisabled int - ConflictsUnlimited int - ConflictsOther int - DisableSparseFiles int - DisableTempIndexes int - AlwaysWeakHash int - CustomWeakHashThreshold int - FsWatcherEnabled int - PullOrder IntMap - FilesystemType IntMap - FsWatcherDelays pq.Int64Array - } - - GUIStats struct { - Enabled int - UseTLS int - UseAuth int - InsecureAdminAccess int - Debugging int - InsecureSkipHostCheck int - InsecureAllowFrameLoading int - ListenLocal int - ListenUnspecified int - Theme IntMap - } - - BlockStats struct { - Total int - Renamed int - Reused int - Pulled int - CopyOrigin int - CopyOriginShifted int - CopyElsewhere int - } - - TransportStats IntMap - - IgnoreStats struct { - Lines int - Inverts int - Folded int - Deletable int - Rooted int - Includes int - EscapedIncludes int - DoubleStars int - Stars int - } - - // V3 fields added late in the RC - WeakHashEnabled bool - - // Generated - - Date string - Address string -} - -func (r *report) Validate() error { - if r.UniqueID == "" || r.Version == "" || r.Platform == "" { - return errors.New("missing required field") - } - if len(r.Date) != 8 { - return errors.New("date not initialized") - } - - // Some fields may not be null. - if r.RescanIntvs == nil { - r.RescanIntvs = []int64{} - } - if r.FolderUsesV3.FsWatcherDelays == nil { - r.FolderUsesV3.FsWatcherDelays = []int64{} - } - - return nil -} - -func (r *report) FieldPointers() []interface{} { - // All the fields of the report, in the same order as the database fields. - return []interface{}{ - &r.Received, &r.UniqueID, &r.Version, &r.LongVersion, &r.Platform, - &r.NumFolders, &r.NumDevices, &r.TotFiles, &r.FolderMaxFiles, - &r.TotMiB, &r.FolderMaxMiB, &r.MemoryUsageMiB, &r.SHA256Perf, - &r.MemorySize, &r.Date, - // V2 - &r.URVersion, &r.NumCPU, &r.FolderUses.SendOnly, &r.FolderUses.IgnorePerms, - &r.FolderUses.IgnoreDelete, &r.FolderUses.AutoNormalize, &r.DeviceUses.Introducer, - &r.DeviceUses.CustomCertName, &r.DeviceUses.CompressAlways, - &r.DeviceUses.CompressMetadata, &r.DeviceUses.CompressNever, - &r.DeviceUses.DynamicAddr, &r.DeviceUses.StaticAddr, - &r.Announce.GlobalEnabled, &r.Announce.LocalEnabled, - &r.Announce.DefaultServersDNS, &r.Announce.DefaultServersIP, - &r.Announce.OtherServers, &r.Relays.Enabled, &r.Relays.DefaultServers, - &r.Relays.OtherServers, &r.UsesRateLimit, &r.UpgradeAllowedManual, - &r.UpgradeAllowedAuto, &r.FolderUses.SimpleVersioning, - &r.FolderUses.ExternalVersioning, &r.FolderUses.StaggeredVersioning, - &r.FolderUses.TrashcanVersioning, - - // V2.5 - &r.UpgradeAllowedPre, &r.RescanIntvs, - - // V3 - &r.Uptime, &r.NATType, &r.AlwaysLocalNets, &r.CacheIgnoredFiles, - &r.OverwriteRemoteDeviceNames, &r.ProgressEmitterEnabled, &r.CustomDefaultFolderPath, - &r.WeakHashSelection, &r.CustomTrafficClass, &r.CustomTempIndexMinBlocks, - &r.TemporariesDisabled, &r.TemporariesCustom, &r.LimitBandwidthInLan, - &r.CustomReleaseURL, &r.RestartOnWakeup, &r.CustomStunServers, - - &r.FolderUsesV3.ScanProgressDisabled, &r.FolderUsesV3.ConflictsDisabled, - &r.FolderUsesV3.ConflictsUnlimited, &r.FolderUsesV3.ConflictsOther, - &r.FolderUsesV3.DisableSparseFiles, &r.FolderUsesV3.DisableTempIndexes, - &r.FolderUsesV3.AlwaysWeakHash, &r.FolderUsesV3.CustomWeakHashThreshold, - &r.FolderUsesV3.FsWatcherEnabled, - - &r.FolderUsesV3.PullOrder, &r.FolderUsesV3.FilesystemType, - &r.FolderUsesV3.FsWatcherDelays, - - &r.GUIStats.Enabled, &r.GUIStats.UseTLS, &r.GUIStats.UseAuth, - &r.GUIStats.InsecureAdminAccess, - &r.GUIStats.Debugging, &r.GUIStats.InsecureSkipHostCheck, - &r.GUIStats.InsecureAllowFrameLoading, &r.GUIStats.ListenLocal, - &r.GUIStats.ListenUnspecified, &r.GUIStats.Theme, - - &r.BlockStats.Total, &r.BlockStats.Renamed, - &r.BlockStats.Reused, &r.BlockStats.Pulled, &r.BlockStats.CopyOrigin, - &r.BlockStats.CopyOriginShifted, &r.BlockStats.CopyElsewhere, - - &r.TransportStats, - - &r.IgnoreStats.Lines, &r.IgnoreStats.Inverts, &r.IgnoreStats.Folded, - &r.IgnoreStats.Deletable, &r.IgnoreStats.Rooted, &r.IgnoreStats.Includes, - &r.IgnoreStats.EscapedIncludes, &r.IgnoreStats.DoubleStars, &r.IgnoreStats.Stars, - - // V3 added late in the RC - &r.WeakHashEnabled, - &r.Address, - - // Receive only folders - &r.FolderUses.ReceiveOnly, - } -} - -func (r *report) FieldNames() []string { - // The database fields that back this struct in PostgreSQL - return []string{ - // V1 - "Received", - "UniqueID", - "Version", - "LongVersion", - "Platform", - "NumFolders", - "NumDevices", - "TotFiles", - "FolderMaxFiles", - "TotMiB", - "FolderMaxMiB", - "MemoryUsageMiB", - "SHA256Perf", - "MemorySize", - "Date", - // V2 - "ReportVersion", - "NumCPU", - "FolderRO", - "FolderIgnorePerms", - "FolderIgnoreDelete", - "FolderAutoNormalize", - "DeviceIntroducer", - "DeviceCustomCertName", - "DeviceCompressAlways", - "DeviceCompressMetadata", - "DeviceCompressNever", - "DeviceDynamicAddr", - "DeviceStaticAddr", - "AnnounceGlobalEnabled", - "AnnounceLocalEnabled", - "AnnounceDefaultServersDNS", - "AnnounceDefaultServersIP", - "AnnounceOtherServers", - "RelayEnabled", - "RelayDefaultServers", - "RelayOtherServers", - "RateLimitEnabled", - "UpgradeAllowedManual", - "UpgradeAllowedAuto", - // v0.12.19+ - "FolderSimpleVersioning", - "FolderExternalVersioning", - "FolderStaggeredVersioning", - "FolderTrashcanVersioning", - // V2.5 - "UpgradeAllowedPre", - "RescanIntvs", - // V3 - "Uptime", - "NATType", - "AlwaysLocalNets", - "CacheIgnoredFiles", - "OverwriteRemoteDeviceNames", - "ProgressEmitterEnabled", - "CustomDefaultFolderPath", - "WeakHashSelection", - "CustomTrafficClass", - "CustomTempIndexMinBlocks", - "TemporariesDisabled", - "TemporariesCustom", - "LimitBandwidthInLan", - "CustomReleaseURL", - "RestartOnWakeup", - "CustomStunServers", - - "FolderScanProgressDisabled", - "FolderConflictsDisabled", - "FolderConflictsUnlimited", - "FolderConflictsOther", - "FolderDisableSparseFiles", - "FolderDisableTempIndexes", - "FolderAlwaysWeakHash", - "FolderCustomWeakHashThreshold", - "FolderFsWatcherEnabled", - "FolderPullOrder", - "FolderFilesystemType", - "FolderFsWatcherDelays", - - "GUIEnabled", - "GUIUseTLS", - "GUIUseAuth", - "GUIInsecureAdminAccess", - "GUIDebugging", - "GUIInsecureSkipHostCheck", - "GUIInsecureAllowFrameLoading", - "GUIListenLocal", - "GUIListenUnspecified", - "GUITheme", - - "BlocksTotal", - "BlocksRenamed", - "BlocksReused", - "BlocksPulled", - "BlocksCopyOrigin", - "BlocksCopyOriginShifted", - "BlocksCopyElsewhere", - - "Transport", - - "IgnoreLines", - "IgnoreInverts", - "IgnoreFolded", - "IgnoreDeletable", - "IgnoreRooted", - "IgnoreIncludes", - "IgnoreEscapedIncludes", - "IgnoreDoubleStars", - "IgnoreStars", - - // V3 added late in the RC - "WeakHashEnabled", - "Address", - - // Receive only folders - "FolderRecvOnly", - } -} - func setupDB(db *sql.DB) error { - _, err := db.Exec(`CREATE TABLE IF NOT EXISTS Reports ( + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS ReportsJson ( Received TIMESTAMP NOT NULL, - UniqueID VARCHAR(32) NOT NULL, - Version VARCHAR(32) NOT NULL, - LongVersion VARCHAR(256) NOT NULL, - Platform VARCHAR(32) NOT NULL, - NumFolders INTEGER NOT NULL, - NumDevices INTEGER NOT NULL, - TotFiles INTEGER NOT NULL, - FolderMaxFiles INTEGER NOT NULL, - TotMiB INTEGER NOT NULL, - FolderMaxMiB INTEGER NOT NULL, - MemoryUsageMiB INTEGER NOT NULL, - SHA256Perf DOUBLE PRECISION NOT NULL, - MemorySize INTEGER NOT NULL, - Date VARCHAR(8) NOT NULL + Report JSONB NOT NULL )`) if err != nil { return err } var t string - row := db.QueryRow(`SELECT 'UniqueIDIndex'::regclass`) - if err := row.Scan(&t); err != nil { - if _, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDIndex ON Reports (Date, UniqueID)`); err != nil { + if err := db.QueryRow(`SELECT 'UniqueIDJsonIndex'::regclass`).Scan(&t); err != nil { + if _, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDJsonIndex ON ReportsJson ((Report->>'date'), (Report->>'uniqueID'))`); err != nil { return err } } - row = db.QueryRow(`SELECT 'ReceivedIndex'::regclass`) - if err := row.Scan(&t); err != nil { - if _, err = db.Exec(`CREATE INDEX ReceivedIndex ON Reports (Received)`); err != nil { + if err := db.QueryRow(`SELECT 'ReceivedJsonIndex'::regclass`).Scan(&t); err != nil { + if _, err = db.Exec(`CREATE INDEX ReceivedJsonIndex ON ReportsJson (Received)`); err != nil { return err } } - // V2 - - row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'reportversion'`) - if err := row.Scan(&t); err != nil { - // The ReportVersion column doesn't exist; add the new columns. - _, err = db.Exec(`ALTER TABLE Reports - ADD COLUMN ReportVersion INTEGER NOT NULL DEFAULT 0, - ADD COLUMN NumCPU INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderRO INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderIgnorePerms INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderIgnoreDelete INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderAutoNormalize INTEGER NOT NULL DEFAULT 0, - ADD COLUMN DeviceIntroducer INTEGER NOT NULL DEFAULT 0, - ADD COLUMN DeviceCustomCertName INTEGER NOT NULL DEFAULT 0, - ADD COLUMN DeviceCompressAlways INTEGER NOT NULL DEFAULT 0, - ADD COLUMN DeviceCompressMetadata INTEGER NOT NULL DEFAULT 0, - ADD COLUMN DeviceCompressNever INTEGER NOT NULL DEFAULT 0, - ADD COLUMN DeviceDynamicAddr INTEGER NOT NULL DEFAULT 0, - ADD COLUMN DeviceStaticAddr INTEGER NOT NULL DEFAULT 0, - ADD COLUMN AnnounceGlobalEnabled BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN AnnounceLocalEnabled BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN AnnounceDefaultServersDNS INTEGER NOT NULL DEFAULT 0, - ADD COLUMN AnnounceDefaultServersIP INTEGER NOT NULL DEFAULT 0, - ADD COLUMN AnnounceOtherServers INTEGER NOT NULL DEFAULT 0, - ADD COLUMN RelayEnabled BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN RelayDefaultServers INTEGER NOT NULL DEFAULT 0, - ADD COLUMN RelayOtherServers INTEGER NOT NULL DEFAULT 0, - ADD COLUMN RateLimitEnabled BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN UpgradeAllowedManual BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN UpgradeAllowedAuto BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN FolderSimpleVersioning INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderExternalVersioning INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderStaggeredVersioning INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderTrashcanVersioning INTEGER NOT NULL DEFAULT 0 - `) - if err != nil { + if err := db.QueryRow(`SELECT 'ReportVersionJsonIndex'::regclass`).Scan(&t); err != nil { + if _, err = db.Exec(`CREATE INDEX ReportVersionJsonIndex ON ReportsJson (cast((Report->>'urVersion') as numeric))`); err != nil { return err } } - row = db.QueryRow(`SELECT 'ReportVersionIndex'::regclass`) - if err := row.Scan(&t); err != nil { - if _, err = db.Exec(`CREATE INDEX ReportVersionIndex ON Reports (ReportVersion)`); err != nil { - return err - } - } - - // V2.5 - - row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'upgradeallowedpre'`) - if err := row.Scan(&t); err != nil { - // The ReportVersion column doesn't exist; add the new columns. - _, err = db.Exec(`ALTER TABLE Reports - ADD COLUMN UpgradeAllowedPre BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN RescanIntvs INT[] NOT NULL DEFAULT '{}' - `) - if err != nil { - return err - } - } - - // V3 - - row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'uptime'`) - if err := row.Scan(&t); err != nil { - // The Uptime column doesn't exist; add the new columns. - _, err = db.Exec(`ALTER TABLE Reports - ADD COLUMN Uptime INTEGER NOT NULL DEFAULT 0, - ADD COLUMN NATType VARCHAR(32) NOT NULL DEFAULT '', - ADD COLUMN AlwaysLocalNets BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN CacheIgnoredFiles BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN OverwriteRemoteDeviceNames BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN ProgressEmitterEnabled BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN CustomDefaultFolderPath BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN WeakHashSelection VARCHAR(32) NOT NULL DEFAULT '', - ADD COLUMN CustomTrafficClass BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN CustomTempIndexMinBlocks BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN TemporariesDisabled BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN TemporariesCustom BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN LimitBandwidthInLan BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN CustomReleaseURL BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN RestartOnWakeup BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN CustomStunServers BOOLEAN NOT NULL DEFAULT FALSE, - - ADD COLUMN FolderScanProgressDisabled INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderConflictsDisabled INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderConflictsUnlimited INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderConflictsOther INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderDisableSparseFiles INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderDisableTempIndexes INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderAlwaysWeakHash INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderCustomWeakHashThreshold INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderFsWatcherEnabled INTEGER NOT NULL DEFAULT 0, - ADD COLUMN FolderPullOrder JSONB NOT NULL DEFAULT '{}', - ADD COLUMN FolderFilesystemType JSONB NOT NULL DEFAULT '{}', - ADD COLUMN FolderFsWatcherDelays INT[] NOT NULL DEFAULT '{}', - - ADD COLUMN GUIEnabled INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIUseTLS INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIUseAuth INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIInsecureAdminAccess INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIDebugging INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIInsecureSkipHostCheck INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIInsecureAllowFrameLoading INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIListenLocal INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUIListenUnspecified INTEGER NOT NULL DEFAULT 0, - ADD COLUMN GUITheme JSONB NOT NULL DEFAULT '{}', - - ADD COLUMN BlocksTotal INTEGER NOT NULL DEFAULT 0, - ADD COLUMN BlocksRenamed INTEGER NOT NULL DEFAULT 0, - ADD COLUMN BlocksReused INTEGER NOT NULL DEFAULT 0, - ADD COLUMN BlocksPulled INTEGER NOT NULL DEFAULT 0, - ADD COLUMN BlocksCopyOrigin INTEGER NOT NULL DEFAULT 0, - ADD COLUMN BlocksCopyOriginShifted INTEGER NOT NULL DEFAULT 0, - ADD COLUMN BlocksCopyElsewhere INTEGER NOT NULL DEFAULT 0, - - ADD COLUMN Transport JSONB NOT NULL DEFAULT '{}', - - ADD COLUMN IgnoreLines INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreInverts INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreFolded INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreDeletable INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreRooted INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreIncludes INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreEscapedIncludes INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreDoubleStars INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IgnoreStars INTEGER NOT NULL DEFAULT 0 - `) - if err != nil { - return err - } - } - - // V3 added late in the RC - - row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'weakhashenabled'`) - if err := row.Scan(&t); err != nil { - // The WeakHashEnabled column doesn't exist; add the new columns. - _, err = db.Exec(`ALTER TABLE Reports - ADD COLUMN WeakHashEnabled BOOLEAN NOT NULL DEFAULT FALSE - ADD COLUMN Address VARCHAR(45) NOT NULL DEFAULT '' - `) - if err != nil { - return err - } - } - - // Receive only added ad-hoc - - row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'folderrecvonly'`) - if err := row.Scan(&t); err != nil { - // The RecvOnly column doesn't exist; add it. - _, err = db.Exec(`ALTER TABLE Reports - ADD COLUMN FolderRecvOnly INTEGER NOT NULL DEFAULT 0 - `) - if err != nil { - return err - } + // Migrate from old schema to new schema if the table exists. + if err := migrate(db); err != nil { + return err } return nil } -func insertReport(db *sql.DB, r report) error { - r.Received = time.Now().UTC() - fields := r.FieldPointers() - params := make([]string, len(fields)) - for i := range params { - params[i] = fmt.Sprintf("$%d", i+1) - } - query := "INSERT INTO Reports (" + strings.Join(r.FieldNames(), ", ") + ") VALUES (" + strings.Join(params, ", ") + ")" - _, err := db.Exec(query, fields...) +func insertReport(db *sql.DB, r contract.Report) error { + _, err := db.Exec("INSERT INTO ReportsJson (Report, Received) VALUES ($1, $2)", r, time.Now().UTC()) return err } @@ -689,9 +146,9 @@ func insertReport(db *sql.DB, r report) error { type withDBFunc func(*sql.DB, http.ResponseWriter, *http.Request) func withDB(db *sql.DB, f withDBFunc) http.HandlerFunc { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { f(db, w, r) - }) + } } func main() { @@ -778,7 +235,7 @@ const maxCacheTime = 15 * time.Minute func cacheRefresher(db *sql.DB) { ticker := time.NewTicker(maxCacheTime - time.Minute) defer ticker.Stop() - for range ticker.C { + for ; true; <-ticker.C { cacheMut.Lock() if err := refreshCacheLocked(db); err != nil { log.Println(err) @@ -861,7 +318,7 @@ func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) { addr = "" } - var rep report + var rep contract.Report rep.Date = time.Now().UTC().Format("20060102") rep.Address = addr @@ -1069,11 +526,11 @@ func getReport(db *sql.DB) map[string]interface{} { var numDevices []int var totFiles []int var maxFiles []int - var totMiB []int - var maxMiB []int - var memoryUsage []int + var totMiB []int64 + var maxMiB []int64 + var memoryUsage []int64 var sha256Perf []float64 - var memorySize []int + var memorySize []int64 var uptime []int var compilers []string var builders []string @@ -1112,9 +569,9 @@ func getReport(db *sql.DB) map[string]interface{} { var numCPU []int - var rep report + var rep contract.Report - rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ",") + ` FROM Reports WHERE Received > now() - '1 day'::INTERVAL`) + rows, err := db.Query(`SELECT Received, Report FROM ReportsJson WHERE Received > now() - '1 day'::INTERVAL`) if err != nil { log.Println("sql:", err) return nil @@ -1122,7 +579,7 @@ func getReport(db *sql.DB) map[string]interface{} { defer rows.Close() for rows.Next() { - err := rows.Scan(rep.FieldPointers()...) + err := rows.Scan(&rep.Received, &rep) if err != nil { log.Println("sql:", err) @@ -1173,19 +630,19 @@ func getReport(db *sql.DB) map[string]interface{} { maxFiles = append(maxFiles, rep.FolderMaxFiles) } if rep.TotMiB > 0 { - totMiB = append(totMiB, rep.TotMiB*(1<<20)) + totMiB = append(totMiB, int64(rep.TotMiB)*(1<<20)) } if rep.FolderMaxMiB > 0 { - maxMiB = append(maxMiB, rep.FolderMaxMiB*(1<<20)) + maxMiB = append(maxMiB, int64(rep.FolderMaxMiB)*(1<<20)) } if rep.MemoryUsageMiB > 0 { - memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20)) + memoryUsage = append(memoryUsage, int64(rep.MemoryUsageMiB)*(1<<20)) } if rep.SHA256Perf > 0 { sha256Perf = append(sha256Perf, rep.SHA256Perf*(1<<20)) } if rep.MemorySize > 0 { - memorySize = append(memorySize, rep.MemorySize*(1<<20)) + memorySize = append(memorySize, int64(rep.MemorySize)*(1<<20)) } if rep.Uptime > 0 { uptime = append(uptime, rep.Uptime) @@ -1336,14 +793,14 @@ func getReport(db *sql.DB) map[string]interface{} { }) categories = append(categories, category{ - Values: statsForInts(totMiB), + Values: statsForInt64s(totMiB), Descr: "Data Managed per Device", Unit: "B", Type: NumberBinary, }) categories = append(categories, category{ - Values: statsForInts(maxMiB), + Values: statsForInt64s(maxMiB), Descr: "Data in Largest Folder", Unit: "B", Type: NumberBinary, @@ -1360,14 +817,14 @@ func getReport(db *sql.DB) map[string]interface{} { }) categories = append(categories, category{ - Values: statsForInts(memoryUsage), + Values: statsForInt64s(memoryUsage), Descr: "Memory Usage", Unit: "B", Type: NumberBinary, }) categories = append(categories, category{ - Values: statsForInts(memorySize), + Values: statsForInt64s(memorySize), Descr: "System Memory", Unit: "B", Type: NumberBinary, diff --git a/cmd/ursrv/migration.go b/cmd/ursrv/migration.go new file mode 100644 index 000000000..9b92d6ef3 --- /dev/null +++ b/cmd/ursrv/migration.go @@ -0,0 +1,143 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package main + +import ( + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "log" + "strings" + + "github.com/lib/pq" + + "github.com/syncthing/syncthing/lib/ur/contract" +) + +func migrate(db *sql.DB) error { + var count uint64 + log.Println("Checking old table row count, this might take a while...") + if err := db.QueryRow(`SELECT COUNT(1) FROM Reports`).Scan(&count); err != nil || count == 0 { + // err != nil most likely means table does not exist. + return nil + } + log.Printf("Found %d records, will perform migration.", count) + + tx, err := db.Begin() + if err != nil { + log.Println("sql:", err) + return err + } + defer tx.Rollback() + + // These must be lower case, because we don't quote them when creating, so postgres creates them lower case. + // Yet pg.CopyIn quotes them, which makes them case sensitive. + stmt, err := tx.Prepare(pq.CopyIn("reportsjson", "received", "report")) + if err != nil { + log.Println("sql:", err) + return err + } + + // Custom types used in the old struct. + var rep contract.Report + var rescanIntvs pq.Int64Array + var fsWatcherDelay pq.Int64Array + pullOrder := make(IntMap) + fileSystemType := make(IntMap) + themes := make(IntMap) + transportStats := make(IntMap) + + rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ", ") + `, FolderFsWatcherDelays, RescanIntvs, FolderPullOrder, FolderFilesystemType, GUITheme, Transport FROM Reports`) + if err != nil { + log.Println("sql:", err) + return err + } + defer rows.Close() + + var done uint64 + pct := count / 100 + + for rows.Next() { + err := rows.Scan(append(rep.FieldPointers(), &fsWatcherDelay, &rescanIntvs, &pullOrder, &fileSystemType, &themes, &transportStats)...) + if err != nil { + log.Println("sql scan:", err) + return err + } + // Patch up parts that used to use custom types + rep.RescanIntvs = make([]int, len(rescanIntvs)) + for i := range rescanIntvs { + rep.RescanIntvs[i] = int(rescanIntvs[i]) + } + rep.FolderUsesV3.FsWatcherDelays = make([]int, len(fsWatcherDelay)) + for i := range fsWatcherDelay { + rep.FolderUsesV3.FsWatcherDelays[i] = int(fsWatcherDelay[i]) + } + rep.FolderUsesV3.PullOrder = pullOrder + rep.FolderUsesV3.FilesystemType = fileSystemType + rep.GUIStats.Theme = themes + rep.TransportStats = transportStats + + _, err = stmt.Exec(rep.Received, rep) + if err != nil { + log.Println("sql insert:", err) + return err + } + done++ + if done%pct == 0 { + log.Printf("Migration progress %d/%d (%d%%)", done, count, (100*done)/count) + } + } + + // Tell the driver bulk copy is finished + _, err = stmt.Exec() + if err != nil { + log.Println("sql stmt exec:", err) + return err + } + + err = stmt.Close() + if err != nil { + log.Println("sql stmt close:", err) + return err + } + + _, err = tx.Exec("DROP TABLE Reports") + if err != nil { + log.Println("sql drop:", err) + return err + } + + err = tx.Commit() + if err != nil { + log.Println("sql commit:", err) + return err + } + return nil +} + +type IntMap map[string]int + +func (p IntMap) Value() (driver.Value, error) { + return json.Marshal(p) +} + +func (p *IntMap) Scan(src interface{}) error { + source, ok := src.([]byte) + if !ok { + return errors.New("Type assertion .([]byte) failed.") + } + + var i map[string]int + err := json.Unmarshal(source, &i) + if err != nil { + return err + } + + *p = i + return nil +} diff --git a/gui/default/syncthing/core/syncthingController.js b/gui/default/syncthing/core/syncthingController.js index 7d92be1a2..161dd7fdd 100755 --- a/gui/default/syncthing/core/syncthingController.js +++ b/gui/default/syncthing/core/syncthingController.js @@ -32,8 +32,6 @@ angular.module('syncthing.core') $scope.protocolChanged = false; $scope.reportData = {}; $scope.reportDataPreview = ''; - $scope.reportDataPreviewVersion = ''; - $scope.reportDataPreviewDiff = false; $scope.reportPreview = false; $scope.folders = {}; $scope.seenError = ''; @@ -2322,13 +2320,13 @@ angular.module('syncthing.core') $scope.reportPreview = true; }; - $scope.refreshReportDataPreview = function () { + $scope.refreshReportDataPreview = function (ver, diff) { $scope.reportDataPreview = ''; - if (!$scope.reportDataPreviewVersion) { + if (!ver) { return; } - var version = parseInt($scope.reportDataPreviewVersion); - if ($scope.reportDataPreviewDiff && version > 2) { + var version = parseInt(ver); + if (diff && version > 2) { $q.all([ $http.get(urlbase + '/svc/report?version=' + version), $http.get(urlbase + '/svc/report?version=' + (version - 1)), diff --git a/gui/default/syncthing/usagereport/usageReportPreviewModalView.html b/gui/default/syncthing/usagereport/usageReportPreviewModalView.html index 95d5ef156..1064be349 100644 --- a/gui/default/syncthing/usagereport/usageReportPreviewModalView.html +++ b/gui/default/syncthing/usagereport/usageReportPreviewModalView.html @@ -6,13 +6,13 @@

The aggregated statistics are publicly available at the URL below.

https://data.syncthing.net/

- -
+
diff --git a/lib/api/api.go b/lib/api/api.go index 279e7a56b..bcd1ca693 100644 --- a/lib/api/api.go +++ b/lib/api/api.go @@ -1063,10 +1063,15 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) { } // Report Data as a JSON - if usageReportingData, err := json.MarshalIndent(s.urService.ReportData(context.TODO()), "", " "); err != nil { - l.Warnln("Support bundle: failed to create versionPlatform.json:", err) + if r, err := s.urService.ReportData(context.TODO()); err != nil { + l.Warnln("Support bundle: failed to create usage-reporting.json.txt:", err) } else { - files = append(files, fileEntry{name: "usage-reporting.json.txt", data: usageReportingData}) + if usageReportingData, err := json.MarshalIndent(r, "", " "); err != nil { + l.Warnln("Support bundle: failed to serialize usage-reporting.json.txt", err) + } else { + files = append(files, fileEntry{name: "usage-reporting.json.txt", data: usageReportingData}) + + } } // Heap and CPU Proofs as a pprof extension @@ -1148,7 +1153,13 @@ func (s *service) getReport(w http.ResponseWriter, r *http.Request) { if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 { version = val } - sendJSON(w, s.urService.ReportDataPreview(context.TODO(), version)) + if r, err := s.urService.ReportDataPreview(context.TODO(), version); err != nil { + http.Error(w, err.Error(), 500) + return + } else { + sendJSON(w, r) + } + } func (s *service) getRandomString(w http.ResponseWriter, r *http.Request) { diff --git a/lib/api/mocked_model_test.go b/lib/api/mocked_model_test.go index 24eba2533..553507c4c 100644 --- a/lib/api/mocked_model_test.go +++ b/lib/api/mocked_model_test.go @@ -15,6 +15,7 @@ import ( "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/stats" + "github.com/syncthing/syncthing/lib/ur/contract" "github.com/syncthing/syncthing/lib/versioner" ) @@ -112,8 +113,7 @@ func (m *mockedModel) State(folder string) (string, time.Time, error) { return "", time.Time{}, nil } -func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} { - return nil +func (m *mockedModel) UsageReportingStats(r *contract.Report, version int, preview bool) { } func (m *mockedModel) FolderErrors(folder string) ([]model.FileError, error) { diff --git a/lib/model/model.go b/lib/model/model.go index a8aa77926..3e95dd937 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -34,6 +34,7 @@ import ( "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/stats" "github.com/syncthing/syncthing/lib/sync" + "github.com/syncthing/syncthing/lib/ur/contract" "github.com/syncthing/syncthing/lib/util" "github.com/syncthing/syncthing/lib/versioner" ) @@ -101,7 +102,7 @@ type Model interface { ConnectionStats() map[string]interface{} DeviceStatistics() (map[string]stats.DeviceStatistics, error) FolderStatistics() (map[string]stats.FolderStatistics, error) - UsageReportingStats(version int, preview bool) map[string]interface{} + UsageReportingStats(report *contract.Report, version int, preview bool) StartDeadlockDetector(timeout time.Duration) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} @@ -443,7 +444,7 @@ func (m *model) stopFolder(cfg config.FolderConfiguration, err error) { // Need to hold lock on m.fmut when calling this. func (m *model) cleanupFolderLocked(cfg config.FolderConfiguration) { - // Clean up our config maps + // clear up our config maps delete(m.folderCfgs, cfg.ID) delete(m.folderFiles, cfg.ID) delete(m.folderIgnores, cfg.ID) @@ -520,49 +521,49 @@ func (m *model) newFolder(cfg config.FolderConfiguration) { m.addAndStartFolderLocked(cfg, fset) } -func (m *model) UsageReportingStats(version int, preview bool) map[string]interface{} { - stats := make(map[string]interface{}) +func (m *model) UsageReportingStats(report *contract.Report, version int, preview bool) { if version >= 3 { // Block stats blockStatsMut.Lock() - copyBlockStats := make(map[string]int) for k, v := range blockStats { - copyBlockStats[k] = v + switch k { + case "total": + report.BlockStats.Total = v + case "renamed": + report.BlockStats.Renamed = v + case "reused": + report.BlockStats.Reused = v + case "pulled": + report.BlockStats.Pulled = v + case "copyOrigin": + report.BlockStats.CopyOrigin = v + case "copyOriginShifted": + report.BlockStats.CopyOriginShifted = v + case "copyElsewhere": + report.BlockStats.CopyElsewhere = v + } + // Reset counts, as these are incremental if !preview { blockStats[k] = 0 } } blockStatsMut.Unlock() - stats["blockStats"] = copyBlockStats // Transport stats m.pmut.RLock() - transportStats := make(map[string]int) for _, conn := range m.conn { - transportStats[conn.Transport()]++ + report.TransportStats[conn.Transport()]++ } m.pmut.RUnlock() - stats["transportStats"] = transportStats // Ignore stats - ignoreStats := map[string]int{ - "lines": 0, - "inverts": 0, - "folded": 0, - "deletable": 0, - "rooted": 0, - "includes": 0, - "escapedIncludes": 0, - "doubleStars": 0, - "stars": 0, - } var seenPrefix [3]bool for folder := range m.cfg.Folders() { lines, _, err := m.GetIgnores(folder) if err != nil { continue } - ignoreStats["lines"] += len(lines) + report.IgnoreStats.Lines += len(lines) for _, line := range lines { // Allow prefixes to be specified in any order, but only once. @@ -570,15 +571,15 @@ func (m *model) UsageReportingStats(version int, preview bool) map[string]interf if strings.HasPrefix(line, "!") && !seenPrefix[0] { seenPrefix[0] = true line = line[1:] - ignoreStats["inverts"] += 1 + report.IgnoreStats.Inverts++ } else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] { seenPrefix[1] = true line = line[4:] - ignoreStats["folded"] += 1 + report.IgnoreStats.Folded++ } else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] { seenPrefix[2] = true line = line[4:] - ignoreStats["deletable"] += 1 + report.IgnoreStats.Deletable++ } else { seenPrefix[0] = false seenPrefix[1] = false @@ -592,28 +593,26 @@ func (m *model) UsageReportingStats(version int, preview bool) map[string]interf line = strings.TrimPrefix(line, "**/") if strings.HasPrefix(line, "/") { - ignoreStats["rooted"] += 1 + report.IgnoreStats.Rooted++ } else if strings.HasPrefix(line, "#include ") { - ignoreStats["includes"] += 1 + report.IgnoreStats.Includes++ if strings.Contains(line, "..") { - ignoreStats["escapedIncludes"] += 1 + report.IgnoreStats.EscapedIncludes++ } } if strings.Contains(line, "**") { - ignoreStats["doubleStars"] += 1 + report.IgnoreStats.DoubleStars++ // Remove not to trip up star checks. line = strings.Replace(line, "**", "", -1) } if strings.Contains(line, "*") { - ignoreStats["stars"] += 1 + report.IgnoreStats.Stars++ } } } - stats["ignoreStats"] = ignoreStats } - return stats } type ConnectionInfo struct { diff --git a/lib/ur/contract/contract.go b/lib/ur/contract/contract.go new file mode 100644 index 000000000..a2eb6454e --- /dev/null +++ b/lib/ur/contract/contract.go @@ -0,0 +1,425 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package contract + +import ( + "database/sql/driver" + "encoding/json" + "errors" + "reflect" + "strconv" + "time" +) + +type Report struct { + // Generated + Received time.Time `json:"-"` // Only from DB + Date string `json:"date,omitempty"` + Address string `json:"address,omitempty"` + + // v1 fields + + UniqueID string `json:"uniqueID,omitempty" since:"1"` + Version string `json:"version,omitempty" since:"1"` + LongVersion string `json:"longVersion,omitempty" since:"1"` + Platform string `json:"platform,omitempty" since:"1"` + NumFolders int `json:"numFolders,omitempty" since:"1"` + NumDevices int `json:"numDevices,omitempty" since:"1"` + TotFiles int `json:"totFiles,omitempty" since:"1"` + FolderMaxFiles int `json:"folderMaxFiles,omitempty" since:"1"` + TotMiB int `json:"totMiB,omitempty" since:"1"` + FolderMaxMiB int `json:"folderMaxMiB,omitempty" since:"1"` + MemoryUsageMiB int `json:"memoryUsageMiB,omitempty" since:"1"` + SHA256Perf float64 `json:"sha256Perf,omitempty" since:"1"` + HashPerf float64 `json:"hashPerf,omitempty" since:"1"` // Was previously not stored server-side + MemorySize int `json:"memorySize,omitempty" since:"1"` + + // v2 fields + + URVersion int `json:"urVersion,omitempty" since:"2"` + NumCPU int `json:"numCPU,omitempty" since:"2"` + FolderUses struct { + SendOnly int `json:"sendonly,omitempty" since:"2"` + SendReceive int `json:"sendreceive,omitempty" since:"2"` // Was previously not stored server-side + ReceiveOnly int `json:"receiveonly,omitempty" since:"2"` + IgnorePerms int `json:"ignorePerms,omitempty" since:"2"` + IgnoreDelete int `json:"ignoreDelete,omitempty" since:"2"` + AutoNormalize int `json:"autoNormalize,omitempty" since:"2"` + SimpleVersioning int `json:"simpleVersioning,omitempty" since:"2"` + ExternalVersioning int `json:"externalVersioning,omitempty" since:"2"` + StaggeredVersioning int `json:"staggeredVersioning,omitempty" since:"2"` + TrashcanVersioning int `json:"trashcanVersioning,omitempty" since:"2"` + } `json:"folderUses,omitempty" since:"2"` + + DeviceUses struct { + Introducer int `json:"introducer,omitempty" since:"2"` + CustomCertName int `json:"customCertName,omitempty" since:"2"` + CompressAlways int `json:"compressAlways,omitempty" since:"2"` + CompressMetadata int `json:"compressMetadata,omitempty" since:"2"` + CompressNever int `json:"compressNever,omitempty" since:"2"` + DynamicAddr int `json:"dynamicAddr,omitempty" since:"2"` + StaticAddr int `json:"staticAddr,omitempty" since:"2"` + } `json:"deviceUses,omitempty" since:"2"` + + Announce struct { + GlobalEnabled bool `json:"globalEnabled,omitempty" since:"2"` + LocalEnabled bool `json:"localEnabled,omitempty" since:"2"` + DefaultServersDNS int `json:"defaultServersDNS,omitempty" since:"2"` + DefaultServersIP int `json:"defaultServersIP,omitempty" since:"2"` // Deprecated and not provided client-side anymore + OtherServers int `json:"otherServers,omitempty" since:"2"` + } `json:"announce,omitempty" since:"2"` + + Relays struct { + Enabled bool `json:"enabled,omitempty" since:"2"` + DefaultServers int `json:"defaultServers,omitempty" since:"2"` + OtherServers int `json:"otherServers,omitempty" since:"2"` + } `json:"relays,omitempty" since:"2"` + + UsesRateLimit bool `json:"usesRateLimit,omitempty" since:"2"` + UpgradeAllowedManual bool `json:"upgradeAllowedManual,omitempty" since:"2"` + UpgradeAllowedAuto bool `json:"upgradeAllowedAuto,omitempty" since:"2"` + + // V2.5 fields (fields that were in v2 but never added to the database + UpgradeAllowedPre bool `json:"upgradeAllowedPre,omitempty" since:"2"` + RescanIntvs []int `json:"rescanIntvs,omitempty" since:"2"` + + // v3 fields + + Uptime int `json:"uptime,omitempty" since:"3"` + NATType string `json:"natType,omitempty" since:"3"` + AlwaysLocalNets bool `json:"alwaysLocalNets,omitempty" since:"3"` + CacheIgnoredFiles bool `json:"cacheIgnoredFiles,omitempty" since:"3"` + OverwriteRemoteDeviceNames bool `json:"overwriteRemoteDeviceNames,omitempty" since:"3"` + ProgressEmitterEnabled bool `json:"progressEmitterEnabled,omitempty" since:"3"` + CustomDefaultFolderPath bool `json:"customDefaultFolderPath,omitempty" since:"3"` + WeakHashSelection string `json:"weakHashSelection,omitempty" since:"3"` // Deprecated and not provided client-side anymore + CustomTrafficClass bool `json:"customTrafficClass,omitempty" since:"3"` + CustomTempIndexMinBlocks bool `json:"customTempIndexMinBlocks,omitempty" since:"3"` + TemporariesDisabled bool `json:"temporariesDisabled,omitempty" since:"3"` + TemporariesCustom bool `json:"temporariesCustom,omitempty" since:"3"` + LimitBandwidthInLan bool `json:"limitBandwidthInLan,omitempty" since:"3"` + CustomReleaseURL bool `json:"customReleaseURL,omitempty" since:"3"` + RestartOnWakeup bool `json:"restartOnWakeup,omitempty" since:"3"` + CustomStunServers bool `json:"customStunServers,omitempty" since:"3"` + + FolderUsesV3 struct { + ScanProgressDisabled int `json:"scanProgressDisabled,omitempty" since:"3"` + ConflictsDisabled int `json:"conflictsDisabled,omitempty" since:"3"` + ConflictsUnlimited int `json:"conflictsUnlimited,omitempty" since:"3"` + ConflictsOther int `json:"conflictsOther,omitempty" since:"3"` + DisableSparseFiles int `json:"disableSparseFiles,omitempty" since:"3"` + DisableTempIndexes int `json:"disableTempIndexes,omitempty" since:"3"` + AlwaysWeakHash int `json:"alwaysWeakHash,omitempty" since:"3"` + CustomWeakHashThreshold int `json:"customWeakHashThreshold,omitempty" since:"3"` + FsWatcherEnabled int `json:"fsWatcherEnabled,omitempty" since:"3"` + PullOrder map[string]int `json:"pullOrder,omitempty" since:"3"` + FilesystemType map[string]int `json:"filesystemType,omitempty" since:"3"` + FsWatcherDelays []int `json:"fsWatcherDelays,omitempty" since:"3"` + } `json:"folderUsesV3,omitempty" since:"3"` + + GUIStats struct { + Enabled int `json:"enabled,omitempty" since:"3"` + UseTLS int `json:"useTLS,omitempty" since:"3"` + UseAuth int `json:"useAuth,omitempty" since:"3"` + InsecureAdminAccess int `json:"insecureAdminAccess,omitempty" since:"3"` + Debugging int `json:"debugging,omitempty" since:"3"` + InsecureSkipHostCheck int `json:"insecureSkipHostCheck,omitempty" since:"3"` + InsecureAllowFrameLoading int `json:"insecureAllowFrameLoading,omitempty" since:"3"` + ListenLocal int `json:"listenLocal,omitempty" since:"3"` + ListenUnspecified int `json:"listenUnspecified,omitempty" since:"3"` + Theme map[string]int `json:"theme,omitempty" since:"3"` + } `json:"guiStats,omitempty" since:"3"` + + BlockStats struct { + Total int `json:"total,omitempty" since:"3"` + Renamed int `json:"renamed,omitempty" since:"3"` + Reused int `json:"reused,omitempty" since:"3"` + Pulled int `json:"pulled,omitempty" since:"3"` + CopyOrigin int `json:"copyOrigin,omitempty" since:"3"` + CopyOriginShifted int `json:"copyOriginShifted,omitempty" since:"3"` + CopyElsewhere int `json:"copyElsewhere,omitempty" since:"3"` + } `json:"blockStats,omitempty" since:"3"` + + TransportStats map[string]int `json:"transportStats,omitempty" since:"3"` + + IgnoreStats struct { + Lines int `json:"lines,omitempty" since:"3"` + Inverts int `json:"inverts,omitempty" since:"3"` + Folded int `json:"folded,omitempty" since:"3"` + Deletable int `json:"deletable,omitempty" since:"3"` + Rooted int `json:"rooted,omitempty" since:"3"` + Includes int `json:"includes,omitempty" since:"3"` + EscapedIncludes int `json:"escapedIncludes,omitempty" since:"3"` + DoubleStars int `json:"doubleStars,omitempty" since:"3"` + Stars int `json:"stars,omitempty" since:"3"` + } `json:"ignoreStats,omitempty" since:"3"` + + // V3 fields added late in the RC + WeakHashEnabled bool `json:"weakHashEnabled,omitempty" since:"3"` // Deprecated and not provided client-side anymore +} + +func New() *Report { + r := &Report{} + r.FolderUsesV3.PullOrder = make(map[string]int) + r.FolderUsesV3.FilesystemType = make(map[string]int) + r.GUIStats.Theme = make(map[string]int) + r.TransportStats = make(map[string]int) + r.RescanIntvs = make([]int, 0) + r.FolderUsesV3.FsWatcherDelays = make([]int, 0) + return r +} + +func (r *Report) Validate() error { + if r.UniqueID == "" || r.Version == "" || r.Platform == "" { + return errors.New("missing required field") + } + if len(r.Date) != 8 { + return errors.New("date not initialized") + } + + // Some fields may not be null. + if r.RescanIntvs == nil { + r.RescanIntvs = []int{} + } + if r.FolderUsesV3.FsWatcherDelays == nil { + r.FolderUsesV3.FsWatcherDelays = []int{} + } + + return nil +} + +func (r *Report) ClearForVersion(version int) error { + return clear(r, version) +} + +func (r *Report) FieldPointers() []interface{} { + // All the fields of the Report, in the same order as the database fields. + return []interface{}{ + &r.Received, &r.UniqueID, &r.Version, &r.LongVersion, &r.Platform, + &r.NumFolders, &r.NumDevices, &r.TotFiles, &r.FolderMaxFiles, + &r.TotMiB, &r.FolderMaxMiB, &r.MemoryUsageMiB, &r.SHA256Perf, + &r.MemorySize, &r.Date, + // V2 + &r.URVersion, &r.NumCPU, &r.FolderUses.SendOnly, &r.FolderUses.IgnorePerms, + &r.FolderUses.IgnoreDelete, &r.FolderUses.AutoNormalize, &r.DeviceUses.Introducer, + &r.DeviceUses.CustomCertName, &r.DeviceUses.CompressAlways, + &r.DeviceUses.CompressMetadata, &r.DeviceUses.CompressNever, + &r.DeviceUses.DynamicAddr, &r.DeviceUses.StaticAddr, + &r.Announce.GlobalEnabled, &r.Announce.LocalEnabled, + &r.Announce.DefaultServersDNS, &r.Announce.DefaultServersIP, + &r.Announce.OtherServers, &r.Relays.Enabled, &r.Relays.DefaultServers, + &r.Relays.OtherServers, &r.UsesRateLimit, &r.UpgradeAllowedManual, + &r.UpgradeAllowedAuto, &r.FolderUses.SimpleVersioning, + &r.FolderUses.ExternalVersioning, &r.FolderUses.StaggeredVersioning, + &r.FolderUses.TrashcanVersioning, + + // V2.5 + &r.UpgradeAllowedPre, + + // V3 + &r.Uptime, &r.NATType, &r.AlwaysLocalNets, &r.CacheIgnoredFiles, + &r.OverwriteRemoteDeviceNames, &r.ProgressEmitterEnabled, &r.CustomDefaultFolderPath, + &r.WeakHashSelection, &r.CustomTrafficClass, &r.CustomTempIndexMinBlocks, + &r.TemporariesDisabled, &r.TemporariesCustom, &r.LimitBandwidthInLan, + &r.CustomReleaseURL, &r.RestartOnWakeup, &r.CustomStunServers, + + &r.FolderUsesV3.ScanProgressDisabled, &r.FolderUsesV3.ConflictsDisabled, + &r.FolderUsesV3.ConflictsUnlimited, &r.FolderUsesV3.ConflictsOther, + &r.FolderUsesV3.DisableSparseFiles, &r.FolderUsesV3.DisableTempIndexes, + &r.FolderUsesV3.AlwaysWeakHash, &r.FolderUsesV3.CustomWeakHashThreshold, + &r.FolderUsesV3.FsWatcherEnabled, + + &r.GUIStats.Enabled, &r.GUIStats.UseTLS, &r.GUIStats.UseAuth, + &r.GUIStats.InsecureAdminAccess, + &r.GUIStats.Debugging, &r.GUIStats.InsecureSkipHostCheck, + &r.GUIStats.InsecureAllowFrameLoading, &r.GUIStats.ListenLocal, + &r.GUIStats.ListenUnspecified, + + &r.BlockStats.Total, &r.BlockStats.Renamed, + &r.BlockStats.Reused, &r.BlockStats.Pulled, &r.BlockStats.CopyOrigin, + &r.BlockStats.CopyOriginShifted, &r.BlockStats.CopyElsewhere, + + &r.IgnoreStats.Lines, &r.IgnoreStats.Inverts, &r.IgnoreStats.Folded, + &r.IgnoreStats.Deletable, &r.IgnoreStats.Rooted, &r.IgnoreStats.Includes, + &r.IgnoreStats.EscapedIncludes, &r.IgnoreStats.DoubleStars, &r.IgnoreStats.Stars, + + // V3 added late in the RC + &r.WeakHashEnabled, + &r.Address, + + // Receive only folders + &r.FolderUses.ReceiveOnly, + } +} + +func (r *Report) FieldNames() []string { + // The database fields that back this struct in PostgreSQL + return []string{ + // V1 + "Received", + "UniqueID", + "Version", + "LongVersion", + "Platform", + "NumFolders", + "NumDevices", + "TotFiles", + "FolderMaxFiles", + "TotMiB", + "FolderMaxMiB", + "MemoryUsageMiB", + "SHA256Perf", + "MemorySize", + "Date", + // V2 + "ReportVersion", + "NumCPU", + "FolderRO", + "FolderIgnorePerms", + "FolderIgnoreDelete", + "FolderAutoNormalize", + "DeviceIntroducer", + "DeviceCustomCertName", + "DeviceCompressAlways", + "DeviceCompressMetadata", + "DeviceCompressNever", + "DeviceDynamicAddr", + "DeviceStaticAddr", + "AnnounceGlobalEnabled", + "AnnounceLocalEnabled", + "AnnounceDefaultServersDNS", + "AnnounceDefaultServersIP", + "AnnounceOtherServers", + "RelayEnabled", + "RelayDefaultServers", + "RelayOtherServers", + "RateLimitEnabled", + "UpgradeAllowedManual", + "UpgradeAllowedAuto", + // v0.12.19+ + "FolderSimpleVersioning", + "FolderExternalVersioning", + "FolderStaggeredVersioning", + "FolderTrashcanVersioning", + // V2.5 + "UpgradeAllowedPre", + // V3 + "Uptime", + "NATType", + "AlwaysLocalNets", + "CacheIgnoredFiles", + "OverwriteRemoteDeviceNames", + "ProgressEmitterEnabled", + "CustomDefaultFolderPath", + "WeakHashSelection", + "CustomTrafficClass", + "CustomTempIndexMinBlocks", + "TemporariesDisabled", + "TemporariesCustom", + "LimitBandwidthInLan", + "CustomReleaseURL", + "RestartOnWakeup", + "CustomStunServers", + + "FolderScanProgressDisabled", + "FolderConflictsDisabled", + "FolderConflictsUnlimited", + "FolderConflictsOther", + "FolderDisableSparseFiles", + "FolderDisableTempIndexes", + "FolderAlwaysWeakHash", + "FolderCustomWeakHashThreshold", + "FolderFsWatcherEnabled", + + "GUIEnabled", + "GUIUseTLS", + "GUIUseAuth", + "GUIInsecureAdminAccess", + "GUIDebugging", + "GUIInsecureSkipHostCheck", + "GUIInsecureAllowFrameLoading", + "GUIListenLocal", + "GUIListenUnspecified", + + "BlocksTotal", + "BlocksRenamed", + "BlocksReused", + "BlocksPulled", + "BlocksCopyOrigin", + "BlocksCopyOriginShifted", + "BlocksCopyElsewhere", + + "IgnoreLines", + "IgnoreInverts", + "IgnoreFolded", + "IgnoreDeletable", + "IgnoreRooted", + "IgnoreIncludes", + "IgnoreEscapedIncludes", + "IgnoreDoubleStars", + "IgnoreStars", + + // V3 added late in the RC + "WeakHashEnabled", + "Address", + + // Receive only folders + "FolderRecvOnly", + } +} + +func (r Report) Value() (driver.Value, error) { + // This needs to be string, yet we read back bytes.. + bs, err := json.Marshal(r) + return string(bs), err +} + +func (r *Report) Scan(value interface{}) error { + b, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + + return json.Unmarshal(b, &r) +} + +func clear(v interface{}, since int) error { + s := reflect.ValueOf(v).Elem() + t := s.Type() + + for i := 0; i < s.NumField(); i++ { + f := s.Field(i) + tag := t.Field(i).Tag + + v := tag.Get("since") + if len(v) == 0 { + f.Set(reflect.Zero(f.Type())) + continue + } + + vn, err := strconv.Atoi(v) + if err != nil { + return err + } + if vn > since { + f.Set(reflect.Zero(f.Type())) + continue + } + + // Dive deeper + if f.Kind() == reflect.Ptr { + f = f.Elem() + } + + if f.Kind() == reflect.Struct { + if err := clear(f.Addr().Interface(), since); err != nil { + return err + } + } + } + return nil +} diff --git a/lib/ur/contract/contract_test.go b/lib/ur/contract/contract_test.go new file mode 100644 index 000000000..0f676af87 --- /dev/null +++ b/lib/ur/contract/contract_test.go @@ -0,0 +1,130 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package contract + +import ( + "reflect" + "testing" +) + +type PtrStruct struct { + A string `since:"2"` + B map[string]int `since:"3"` +} + +type Nested struct { + A float32 `since:"4"` + B [4]int `since:"5"` + C bool `since:"1"` +} + +type TestStruct struct { + A int + B map[string]string `since:"1"` + C []string `since:"2"` + Nested Nested `since:"3"` + + Ptr *PtrStruct `since:"2"` +} + +func testValue() TestStruct { + return TestStruct{ + A: 1, + B: map[string]string{ + "foo": "bar", + }, + C: []string{"a", "b"}, + Nested: Nested{ + A: 0.10, + B: [4]int{1, 2, 3, 4}, + C: true, + }, + Ptr: &PtrStruct{ + A: "value", + B: map[string]int{ + "x": 1, + "b": 2, + }, + }, + } +} + +func TestClean(t *testing.T) { + expect(t, 0, TestStruct{}) + + expect(t, 1, TestStruct{ + // A unset, since it does not have "since" + B: map[string]string{ + "foo": "bar", + }, + }) + + expect(t, 2, TestStruct{ + // A unset, since it does not have "since" + B: map[string]string{ + "foo": "bar", + }, + C: []string{"a", "b"}, + Ptr: &PtrStruct{ + A: "value", + }, + }) + + expect(t, 3, TestStruct{ + // A unset, since it does not have "since" + B: map[string]string{ + "foo": "bar", + }, + C: []string{"a", "b"}, + Nested: Nested{ + C: true, + }, + Ptr: &PtrStruct{ + A: "value", + B: map[string]int{ + "x": 1, + "b": 2, + }, + }, + }) + + expect(t, 4, TestStruct{ + // A unset, since it does not have "since" + B: map[string]string{ + "foo": "bar", + }, + C: []string{"a", "b"}, + Nested: Nested{ + A: 0.10, + C: true, + }, + Ptr: &PtrStruct{ + A: "value", + B: map[string]int{ + "x": 1, + "b": 2, + }, + }, + }) + + x := testValue() + x.A = 0 + + expect(t, 5, x) + expect(t, 6, x) +} + +func expect(t *testing.T, since int, b interface{}) { + t.Helper() + x := testValue() + if err := clear(&x, since); err != nil { + t.Fatal(err.Error()) + } + if !reflect.DeepEqual(x, b) { + t.Errorf("%#v != %#v", x, b) + } +} diff --git a/lib/ur/memsize_darwin.go b/lib/ur/memsize_darwin.go index b88d3fdb9..0d19921e0 100644 --- a/lib/ur/memsize_darwin.go +++ b/lib/ur/memsize_darwin.go @@ -8,7 +8,10 @@ package ur import "golang.org/x/sys/unix" -func memorySize() (int64, error) { +func memorySize() int64 { mem, err := unix.SysctlUint64("hw.memsize") - return int64(mem), err + if err != nil { + return 0 + } + return int64(mem) } diff --git a/lib/ur/memsize_linux.go b/lib/ur/memsize_linux.go index ca6d9edb7..2be03aa95 100644 --- a/lib/ur/memsize_linux.go +++ b/lib/ur/memsize_linux.go @@ -8,32 +8,31 @@ package ur import ( "bufio" - "errors" "os" "strconv" "strings" ) -func memorySize() (int64, error) { +func memorySize() int64 { f, err := os.Open("/proc/meminfo") if err != nil { - return 0, err + return 0 } s := bufio.NewScanner(f) if !s.Scan() { - return 0, errors.New("/proc/meminfo parse error 1") + return 0 } l := s.Text() fs := strings.Fields(l) if len(fs) != 3 || fs[2] != "kB" { - return 0, errors.New("/proc/meminfo parse error 2") + return 0 } kb, err := strconv.ParseInt(fs[1], 10, 64) if err != nil { - return 0, err + return 0 } - return kb * 1024, nil + return kb * 1024 } diff --git a/lib/ur/memsize_netbsd.go b/lib/ur/memsize_netbsd.go index 3fd4a77a6..7a0568bca 100644 --- a/lib/ur/memsize_netbsd.go +++ b/lib/ur/memsize_netbsd.go @@ -7,25 +7,24 @@ package ur import ( - "errors" "os/exec" "strconv" "strings" ) -func memorySize() (int64, error) { +func memorySize() int64 { cmd := exec.Command("/sbin/sysctl", "hw.physmem64") out, err := cmd.Output() if err != nil { - return 0, err + return 0 } fs := strings.Fields(string(out)) if len(fs) != 3 { - return 0, errors.New("sysctl parse error") + return 0 } bytes, err := strconv.ParseInt(fs[2], 10, 64) if err != nil { - return 0, err + return 0 } - return bytes, nil + return bytes } diff --git a/lib/ur/memsize_solaris.go b/lib/ur/memsize_solaris.go index 23965fd05..cd805b736 100644 --- a/lib/ur/memsize_solaris.go +++ b/lib/ur/memsize_solaris.go @@ -13,16 +13,16 @@ import ( "strconv" ) -func memorySize() (int64, error) { +func memorySize() int64 { cmd := exec.Command("prtconf", "-m") out, err := cmd.CombinedOutput() if err != nil { - return 0, err + return 0 } mb, err := strconv.ParseInt(string(out), 10, 64) if err != nil { - return 0, err + return 0 } - return mb * 1024 * 1024, nil + return mb * 1024 * 1024 } diff --git a/lib/ur/memsize_unimpl.go b/lib/ur/memsize_unimpl.go index 0b4d792f7..5a80b1a27 100644 --- a/lib/ur/memsize_unimpl.go +++ b/lib/ur/memsize_unimpl.go @@ -8,8 +8,6 @@ package ur -import "errors" - -func memorySize() (int64, error) { - return 0, errors.New("not implemented") +func memorySize() int64 { + return 0 } diff --git a/lib/ur/memsize_windows.go b/lib/ur/memsize_windows.go index 92c780829..a6a70419e 100644 --- a/lib/ur/memsize_windows.go +++ b/lib/ur/memsize_windows.go @@ -17,14 +17,14 @@ var ( globalMemoryStatusEx, _ = syscall.GetProcAddress(kernel32, "GlobalMemoryStatusEx") ) -func memorySize() (int64, error) { +func memorySize() int64 { var memoryStatusEx [64]byte binary.LittleEndian.PutUint32(memoryStatusEx[:], 64) - ret, _, callErr := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, uintptr(unsafe.Pointer(&memoryStatusEx[0])), 0, 0) + ret, _, _ := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, uintptr(unsafe.Pointer(&memoryStatusEx[0])), 0, 0) if ret == 0 { - return 0, callErr + return 0 } - return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:])), nil + return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:])) } diff --git a/lib/ur/usage_report.go b/lib/ur/usage_report.go index 30282d199..e56130084 100644 --- a/lib/ur/usage_report.go +++ b/lib/ur/usage_report.go @@ -28,6 +28,7 @@ import ( "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/upgrade" + "github.com/syncthing/syncthing/lib/ur/contract" "github.com/syncthing/syncthing/lib/util" "github.com/thejerf/suture" @@ -63,27 +64,19 @@ func New(cfg config.Wrapper, m model.Model, connectionsService connections.Servi // ReportData returns the data to be sent in a usage report with the currently // configured usage reporting version. -func (s *Service) ReportData(ctx context.Context) map[string]interface{} { +func (s *Service) ReportData(ctx context.Context) (*contract.Report, error) { urVersion := s.cfg.Options().URAccepted return s.reportData(ctx, urVersion, false) } // ReportDataPreview returns a preview of the data to be sent in a usage report // with the given version. -func (s *Service) ReportDataPreview(ctx context.Context, urVersion int) map[string]interface{} { +func (s *Service) ReportDataPreview(ctx context.Context, urVersion int) (*contract.Report, error) { return s.reportData(ctx, urVersion, true) } -func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) map[string]interface{} { +func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (*contract.Report, error) { opts := s.cfg.Options() - res := make(map[string]interface{}) - res["urVersion"] = urVersion - res["uniqueID"] = opts.URUniqueID - res["version"] = build.Version - res["longVersion"] = build.LongVersion - res["platform"] = runtime.GOOS + "-" + runtime.GOARCH - res["numFolders"] = len(s.cfg.Folders()) - res["numDevices"] = len(s.cfg.Devices()) var totFiles, maxFiles int var totBytes, maxBytes int64 @@ -104,264 +97,211 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) m } } - res["totFiles"] = totFiles - res["folderMaxFiles"] = maxFiles - res["totMiB"] = totBytes / 1024 / 1024 - res["folderMaxMiB"] = maxBytes / 1024 / 1024 - var mem runtime.MemStats runtime.ReadMemStats(&mem) - res["memoryUsageMiB"] = (mem.Sys - mem.HeapReleased) / 1024 / 1024 - res["sha256Perf"] = CpuBench(ctx, 5, 125*time.Millisecond, false) - res["hashPerf"] = CpuBench(ctx, 5, 125*time.Millisecond, true) - bytes, err := memorySize() - if err == nil { - res["memorySize"] = bytes / 1024 / 1024 - } - res["numCPU"] = runtime.NumCPU() + report := contract.New() + + report.URVersion = urVersion + report.UniqueID = opts.URUniqueID + report.Version = build.Version + report.LongVersion = build.LongVersion + report.Platform = runtime.GOOS + "-" + runtime.GOARCH + report.NumFolders = len(s.cfg.Folders()) + report.NumDevices = len(s.cfg.Devices()) + report.TotFiles = totFiles + report.FolderMaxFiles = maxFiles + report.TotMiB = int(totBytes / 1024 / 1024) + report.FolderMaxMiB = int(maxBytes / 1024 / 1024) + report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024) + report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false) + report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true) + report.MemorySize = int(memorySize() / 1024 / 1024) + report.NumCPU = runtime.NumCPU() - var rescanIntvs []int - folderUses := map[string]int{ - "sendonly": 0, - "sendreceive": 0, - "receiveonly": 0, - "ignorePerms": 0, - "ignoreDelete": 0, - "autoNormalize": 0, - "simpleVersioning": 0, - "externalVersioning": 0, - "staggeredVersioning": 0, - "trashcanVersioning": 0, - } for _, cfg := range s.cfg.Folders() { - rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS) + report.RescanIntvs = append(report.RescanIntvs, cfg.RescanIntervalS) switch cfg.Type { case config.FolderTypeSendOnly: - folderUses["sendonly"]++ + report.FolderUses.SendOnly++ case config.FolderTypeSendReceive: - folderUses["sendreceive"]++ + report.FolderUses.SendReceive++ case config.FolderTypeReceiveOnly: - folderUses["receiveonly"]++ + report.FolderUses.ReceiveOnly++ } if cfg.IgnorePerms { - folderUses["ignorePerms"]++ + report.FolderUses.IgnorePerms++ } if cfg.IgnoreDelete { - folderUses["ignoreDelete"]++ + report.FolderUses.IgnoreDelete++ } if cfg.AutoNormalize { - folderUses["autoNormalize"]++ + report.FolderUses.AutoNormalize++ } - if cfg.Versioning.Type != "" { - folderUses[cfg.Versioning.Type+"Versioning"]++ + switch cfg.Versioning.Type { + case "": + // None + case "simple": + report.FolderUses.SimpleVersioning++ + case "staggered": + report.FolderUses.StaggeredVersioning++ + case "external": + report.FolderUses.ExternalVersioning++ + case "trashcan": + report.FolderUses.TrashcanVersioning++ + default: + l.Warnf("Unhandled versioning type for usage reports: %s", cfg.Versioning.Type) } } - sort.Ints(rescanIntvs) - res["rescanIntvs"] = rescanIntvs - res["folderUses"] = folderUses + sort.Ints(report.RescanIntvs) - deviceUses := map[string]int{ - "introducer": 0, - "customCertName": 0, - "compressAlways": 0, - "compressMetadata": 0, - "compressNever": 0, - "dynamicAddr": 0, - "staticAddr": 0, - } for _, cfg := range s.cfg.Devices() { if cfg.Introducer { - deviceUses["introducer"]++ + report.DeviceUses.Introducer++ } if cfg.CertName != "" && cfg.CertName != "syncthing" { - deviceUses["customCertName"]++ + report.DeviceUses.CustomCertName++ } - if cfg.Compression == protocol.CompressAlways { - deviceUses["compressAlways"]++ - } else if cfg.Compression == protocol.CompressMetadata { - deviceUses["compressMetadata"]++ - } else if cfg.Compression == protocol.CompressNever { - deviceUses["compressNever"]++ + switch cfg.Compression { + case protocol.CompressAlways: + report.DeviceUses.CompressAlways++ + case protocol.CompressMetadata: + report.DeviceUses.CompressMetadata++ + case protocol.CompressNever: + report.DeviceUses.CompressNever++ + default: + l.Warnf("Unhandled versioning type for usage reports: %s", cfg.Compression) } + for _, addr := range cfg.Addresses { if addr == "dynamic" { - deviceUses["dynamicAddr"]++ + report.DeviceUses.DynamicAddr++ } else { - deviceUses["staticAddr"]++ + report.DeviceUses.StaticAddr++ } } } - res["deviceUses"] = deviceUses - defaultAnnounceServersDNS, defaultAnnounceServersIP, otherAnnounceServers := 0, 0, 0 + report.Announce.GlobalEnabled = opts.GlobalAnnEnabled + report.Announce.LocalEnabled = opts.LocalAnnEnabled for _, addr := range opts.RawGlobalAnnServers { if addr == "default" || addr == "default-v4" || addr == "default-v6" { - defaultAnnounceServersDNS++ + report.Announce.DefaultServersDNS++ } else { - otherAnnounceServers++ + report.Announce.OtherServers++ } } - res["announce"] = map[string]interface{}{ - "globalEnabled": opts.GlobalAnnEnabled, - "localEnabled": opts.LocalAnnEnabled, - "defaultServersDNS": defaultAnnounceServersDNS, - "defaultServersIP": defaultAnnounceServersIP, - "otherServers": otherAnnounceServers, - } - defaultRelayServers, otherRelayServers := 0, 0 + report.Relays.Enabled = opts.RelaysEnabled for _, addr := range s.cfg.Options().ListenAddresses() { switch { case addr == "dynamic+https://relays.syncthing.net/endpoint": - defaultRelayServers++ + report.Relays.DefaultServers++ case strings.HasPrefix(addr, "relay://") || strings.HasPrefix(addr, "dynamic+http"): - otherRelayServers++ + report.Relays.OtherServers++ + } } - res["relays"] = map[string]interface{}{ - "enabled": defaultRelayServers+otherAnnounceServers > 0, - "defaultServers": defaultRelayServers, - "otherServers": otherRelayServers, - } - res["usesRateLimit"] = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0 + report.UsesRateLimit = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0 + report.UpgradeAllowedManual = !(upgrade.DisabledByCompilation || s.noUpgrade) + report.UpgradeAllowedAuto = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 + report.UpgradeAllowedPre = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases - res["upgradeAllowedManual"] = !(upgrade.DisabledByCompilation || s.noUpgrade) - res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 - res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases + // V3 if urVersion >= 3 { - res["uptime"] = s.UptimeS() - res["natType"] = s.connectionsService.NATType() - res["alwaysLocalNets"] = len(opts.AlwaysLocalNets) > 0 - res["cacheIgnoredFiles"] = opts.CacheIgnoredFiles - res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames - res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1 - res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~" - res["customTrafficClass"] = opts.TrafficClass != 0 - res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10 - res["temporariesDisabled"] = opts.KeepTemporariesH == 0 - res["temporariesCustom"] = opts.KeepTemporariesH != 24 - res["limitBandwidthInLan"] = opts.LimitBandwidthInLan - res["customReleaseURL"] = opts.ReleasesURL != "https://upgrades.syncthing.net/meta.json" - res["restartOnWakeup"] = opts.RestartOnWakeup + report.Uptime = s.UptimeS() + report.NATType = s.connectionsService.NATType() + report.AlwaysLocalNets = len(opts.AlwaysLocalNets) > 0 + report.CacheIgnoredFiles = opts.CacheIgnoredFiles + report.OverwriteRemoteDeviceNames = opts.OverwriteRemoteDevNames + report.ProgressEmitterEnabled = opts.ProgressUpdateIntervalS > -1 + report.CustomDefaultFolderPath = opts.DefaultFolderPath != "~" + report.CustomTrafficClass = opts.TrafficClass != 0 + report.CustomTempIndexMinBlocks = opts.TempIndexMinBlocks != 10 + report.TemporariesDisabled = opts.KeepTemporariesH == 0 + report.TemporariesCustom = opts.KeepTemporariesH != 24 + report.LimitBandwidthInLan = opts.LimitBandwidthInLan + report.CustomReleaseURL = opts.ReleasesURL != "https=//upgrades.syncthing.net/meta.json" + report.RestartOnWakeup = opts.RestartOnWakeup + report.CustomStunServers = len(opts.RawStunServers) != 1 || opts.RawStunServers[0] != "default" - folderUsesV3 := map[string]int{ - "scanProgressDisabled": 0, - "conflictsDisabled": 0, - "conflictsUnlimited": 0, - "conflictsOther": 0, - "disableSparseFiles": 0, - "disableTempIndexes": 0, - "alwaysWeakHash": 0, - "customWeakHashThreshold": 0, - "fsWatcherEnabled": 0, - } - pullOrder := make(map[string]int) - filesystemType := make(map[string]int) - var fsWatcherDelays []int for _, cfg := range s.cfg.Folders() { if cfg.ScanProgressIntervalS < 0 { - folderUsesV3["scanProgressDisabled"]++ + report.FolderUsesV3.ScanProgressDisabled++ } if cfg.MaxConflicts == 0 { - folderUsesV3["conflictsDisabled"]++ + report.FolderUsesV3.ConflictsDisabled++ } else if cfg.MaxConflicts < 0 { - folderUsesV3["conflictsUnlimited"]++ + report.FolderUsesV3.ConflictsUnlimited++ } else { - folderUsesV3["conflictsOther"]++ + report.FolderUsesV3.ConflictsOther++ } if cfg.DisableSparseFiles { - folderUsesV3["disableSparseFiles"]++ + report.FolderUsesV3.DisableSparseFiles++ } if cfg.DisableTempIndexes { - folderUsesV3["disableTempIndexes"]++ + report.FolderUsesV3.DisableTempIndexes++ } if cfg.WeakHashThresholdPct < 0 { - folderUsesV3["alwaysWeakHash"]++ + report.FolderUsesV3.AlwaysWeakHash++ } else if cfg.WeakHashThresholdPct != 25 { - folderUsesV3["customWeakHashThreshold"]++ + report.FolderUsesV3.CustomWeakHashThreshold++ } if cfg.FSWatcherEnabled { - folderUsesV3["fsWatcherEnabled"]++ + report.FolderUsesV3.FsWatcherEnabled++ } - pullOrder[cfg.Order.String()]++ - filesystemType[cfg.FilesystemType.String()]++ - fsWatcherDelays = append(fsWatcherDelays, cfg.FSWatcherDelayS) + report.FolderUsesV3.PullOrder[cfg.Order.String()]++ + report.FolderUsesV3.FilesystemType[cfg.FilesystemType.String()]++ + report.FolderUsesV3.FsWatcherDelays = append(report.FolderUsesV3.FsWatcherDelays, cfg.FSWatcherDelayS) } - sort.Ints(fsWatcherDelays) - folderUsesV3Interface := map[string]interface{}{ - "pullOrder": pullOrder, - "filesystemType": filesystemType, - "fsWatcherDelays": fsWatcherDelays, - } - for key, value := range folderUsesV3 { - folderUsesV3Interface[key] = value - } - res["folderUsesV3"] = folderUsesV3Interface + sort.Ints(report.FolderUsesV3.FsWatcherDelays) guiCfg := s.cfg.GUI() // Anticipate multiple GUI configs in the future, hence store counts. - guiStats := map[string]int{ - "enabled": 0, - "useTLS": 0, - "useAuth": 0, - "insecureAdminAccess": 0, - "debugging": 0, - "insecureSkipHostCheck": 0, - "insecureAllowFrameLoading": 0, - "listenLocal": 0, - "listenUnspecified": 0, - } - theme := make(map[string]int) if guiCfg.Enabled { - guiStats["enabled"]++ + report.GUIStats.Enabled++ if guiCfg.UseTLS() { - guiStats["useTLS"]++ + report.GUIStats.UseTLS++ } if len(guiCfg.User) > 0 && len(guiCfg.Password) > 0 { - guiStats["useAuth"]++ + report.GUIStats.UseAuth++ } if guiCfg.InsecureAdminAccess { - guiStats["insecureAdminAccess"]++ + report.GUIStats.InsecureAdminAccess++ } if guiCfg.Debugging { - guiStats["debugging"]++ + report.GUIStats.Debugging++ } if guiCfg.InsecureSkipHostCheck { - guiStats["insecureSkipHostCheck"]++ + report.GUIStats.InsecureSkipHostCheck++ } if guiCfg.InsecureAllowFrameLoading { - guiStats["insecureAllowFrameLoading"]++ + report.GUIStats.InsecureAllowFrameLoading++ } addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address()) if err == nil { if addr.IP.IsLoopback() { - guiStats["listenLocal"]++ + report.GUIStats.ListenLocal++ + } else if addr.IP.IsUnspecified() { - guiStats["listenUnspecified"]++ + report.GUIStats.ListenUnspecified++ } } - - theme[guiCfg.Theme]++ + report.GUIStats.Theme[guiCfg.Theme]++ } - guiStatsInterface := map[string]interface{}{ - "theme": theme, - } - for key, value := range guiStats { - guiStatsInterface[key] = value - } - res["guiStats"] = guiStatsInterface } - for key, value := range s.model.UsageReportingStats(urVersion, preview) { - res[key] = value + s.model.UsageReportingStats(report, urVersion, preview) + + if err := report.ClearForVersion(urVersion); err != nil { + return nil, err } - return res + return report, nil } func (s *Service) UptimeS() int { @@ -369,7 +309,10 @@ func (s *Service) UptimeS() int { } func (s *Service) sendUsageReport(ctx context.Context) error { - d := s.ReportData(ctx) + d, err := s.ReportData(ctx) + if err != nil { + return err + } var b bytes.Buffer if err := json.NewEncoder(&b).Encode(d); err != nil { return err @@ -384,12 +327,11 @@ func (s *Service) sendUsageReport(ctx context.Context) error { }, }, } - req, err := http.NewRequest("POST", s.cfg.Options().URURL, &b) + req, err := http.NewRequestWithContext(ctx, "POST", s.cfg.Options().URURL, &b) if err != nil { return err } req.Header.Set("Content-Type", "application/json") - req.Cancel = ctx.Done() resp, err := client.Do(req) if err != nil { return err