Compare commits
247
Commits
v1.26.1
...
v1.28.0-rc.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cba163a1fd | ||
|
|
a8e2c8edb6 | ||
|
|
3e501d9036 | ||
|
|
9ca101756d | ||
|
|
a873d12c65 | ||
|
|
8ff670c564 | ||
|
|
b1ed2802fb | ||
|
|
b70cb580c8 | ||
|
|
28be3ba788 | ||
|
|
d4770ddc77 | ||
|
|
cbe1220680 | ||
|
|
0b95c5fa76 | ||
|
|
0343bca257 | ||
|
|
878016db39 | ||
|
|
1f4fde9525 | ||
|
|
5b9d8a838f | ||
|
|
8b19cb1e11 | ||
|
|
ce1e259bb4 | ||
|
|
2238a288d9 | ||
|
|
c8ee2a5cf6 | ||
|
|
1704827d04 | ||
|
|
a156e88eef | ||
|
|
94d0195b63 | ||
|
|
1616edcee3 | ||
|
|
6505e123bb | ||
|
|
63e4659282 | ||
|
|
f3f5557c8e | ||
|
|
b794726e1f | ||
|
|
3d59740a0a | ||
|
|
66fb65b01f | ||
|
|
5c2fcbfd19 | ||
|
|
f9b72330a8 | ||
|
|
822b6ac36b | ||
|
|
77f7778292 | ||
|
|
aed2c66e52 | ||
|
|
68a1fd010f | ||
|
|
ac8b3342ac | ||
|
|
0ea90dd932 | ||
|
|
718b1ce2b7 | ||
|
|
29f7510f5a | ||
|
|
a7f9ed4a80 | ||
|
|
1baefea410 | ||
|
|
563cec8923 | ||
|
|
a3c340ece9 | ||
|
|
cb24638ec9 | ||
|
|
2fb24dc2cc | ||
|
|
9aa2d2c92f | ||
|
|
d1c5100c98 | ||
|
|
42e677c055 | ||
|
|
27bba2c0c2 | ||
|
|
feff334547 | ||
|
|
713cf357ce | ||
|
|
5342bec1b7 | ||
|
|
7df75e681d | ||
|
|
8dc826b234 | ||
|
|
9ef37e1485 | ||
|
|
7517d18fbb | ||
|
|
42d0fee536 | ||
|
|
2ca9d3b5c5 | ||
|
|
9cde068f2a | ||
|
|
1243083831 | ||
|
|
356c5055ad | ||
|
|
19693734a3 | ||
|
|
17e60b9e0c | ||
|
|
ac22b2d00a | ||
|
|
de0b4270df | ||
|
|
e738af7c56 | ||
|
|
a28441a9bf | ||
|
|
2e313716e5 | ||
|
|
0b5ff1f5f7 | ||
|
|
0fe6d97d3d | ||
|
|
0756e42a85 | ||
|
|
13ebe1c87f | ||
|
|
aea7fa5f22 | ||
|
|
403ce7e597 | ||
|
|
4704d3bc48 | ||
|
|
2794b04243 | ||
|
|
1ce64971fd | ||
|
|
eb6d80eac4 | ||
|
|
a8db3351ae | ||
|
|
23a900e096 | ||
|
|
5a304cf295 | ||
|
|
136b3742bf | ||
|
|
21e0f98fe2 | ||
|
|
2bb5b2244b | ||
|
|
2f281799c1 | ||
|
|
18a58a2ddc | ||
|
|
f283215fce | ||
|
|
495809ac9e | ||
|
|
9ca8addcf7 | ||
|
|
94181ade23 | ||
|
|
e50933433e | ||
|
|
a2b8f2361e | ||
|
|
4b60e86d02 | ||
|
|
d6b5676603 | ||
|
|
3821b6ceee | ||
|
|
973585e97d | ||
|
|
ba6ac2f604 | ||
|
|
57d399317e | ||
|
|
f2d6722348 | ||
|
|
7b1b77e50d | ||
|
|
06914b872b | ||
|
|
f6df8b40b4 | ||
|
|
e057f5ee9a | ||
|
|
a5bf110d90 | ||
|
|
debbe726e0 | ||
|
|
ec3e474a53 | ||
|
|
ebb1edc652 | ||
|
|
6204670c66 | ||
|
|
ff9b24f388 | ||
|
|
01b820dc78 | ||
|
|
79ae24df76 | ||
|
|
61b94b9ea5 | ||
|
|
6fb3c5ccf2 | ||
|
|
2e7c03420f | ||
|
|
faa56b4bb7 | ||
|
|
d7ba5316b8 | ||
|
|
bdfd0f0548 | ||
|
|
5d27185083 | ||
|
|
4dfb9d7c83 | ||
|
|
2f15670094 | ||
|
|
b49137ce36 | ||
|
|
7e4e65ebf5 | ||
|
|
8c8167a4ab | ||
|
|
73cc5553b6 | ||
|
|
2ab2488274 | ||
|
|
eb9cd363d0 | ||
|
|
7fe3906534 | ||
|
|
5fdab1bf11 | ||
|
|
13a6d43f0b | ||
|
|
ac942e2481 | ||
|
|
bbd2a7fbc5 | ||
|
|
07a9fa2dbd | ||
|
|
aa559bf496 | ||
|
|
2d968d46b7 | ||
|
|
86c4cafc96 | ||
|
|
c4dfb66d84 | ||
|
|
f4d160684b | ||
|
|
b76e6ce70d | ||
|
|
6b4028eede | ||
|
|
ad81ac8da7 | ||
|
|
7ebeaefe77 | ||
|
|
e1dd36561d | ||
|
|
96c30f8387 | ||
|
|
fc8b353011 | ||
|
|
416b9e8924 | ||
|
|
9f6d732587 | ||
|
|
f2f5786b33 | ||
|
|
eb617865d2 | ||
|
|
a49e318d25 | ||
|
|
e74674a019 | ||
|
|
d98fa474ae | ||
|
|
f16817632f | ||
|
|
bda4016109 | ||
|
|
8f5d07bd09 | ||
|
|
302b352d78 | ||
|
|
45beb28fa5 | ||
|
|
ee9b20e47a | ||
|
|
0f55d5fc3e | ||
|
|
35e153625c | ||
|
|
d5e1b99e6c | ||
|
|
3297624037 | ||
|
|
445e8cc532 | ||
|
|
e041877488 | ||
|
|
36e08f8eee | ||
|
|
8b321387c0 | ||
|
|
8edd67a569 | ||
|
|
e829a63295 | ||
|
|
6881a6897a | ||
|
|
9387e107b3 | ||
|
|
aa901790b9 | ||
|
|
17df4b8634 | ||
|
|
34ef30dd5c | ||
|
|
fc1c7a3c49 | ||
|
|
2abfefc18c | ||
|
|
86a08eb87d | ||
|
|
d330d65859 | ||
|
|
4e1831fa3a | ||
|
|
2f3eacdb6c | ||
|
|
1ce2af1238 | ||
|
|
683b48182c | ||
|
|
795aed306c | ||
|
|
cdefa535ed | ||
|
|
91084b83b4 | ||
|
|
5360e7153b | ||
|
|
5d0ca19350 | ||
|
|
e8d3529fed | ||
|
|
935a28c961 | ||
|
|
d21a2de055 | ||
|
|
6f1023665c | ||
|
|
c53a1f210c | ||
|
|
b71a930bfc | ||
|
|
a2cbc62521 | ||
|
|
768fd6bff8 | ||
|
|
48883e0e32 | ||
|
|
30fe2cf514 | ||
|
|
3850a08252 | ||
|
|
d0e407f3c3 | ||
|
|
16db6fcf3d | ||
|
|
4c5528bd0e | ||
|
|
d42fff1016 | ||
|
|
a28de73031 | ||
|
|
75310b58a0 | ||
|
|
8064957270 | ||
|
|
c1ec9a8826 | ||
|
|
1625b44892 | ||
|
|
d51760f410 | ||
|
|
7b1932d64e | ||
|
|
5bfc540c88 | ||
|
|
4cba99fcd4 | ||
|
|
2ae15aa454 | ||
|
|
47bcf4f8f4 | ||
|
|
a8b9096353 | ||
|
|
5328380691 | ||
|
|
6069cf39e5 | ||
|
|
1f7d236742 | ||
|
|
8e9ee3fbe8 | ||
|
|
35b0afc131 | ||
|
|
958ff67ccc | ||
|
|
13d9317a38 | ||
|
|
b184d46d8a | ||
|
|
3f32c5cb4b | ||
|
|
5c65a1bc83 | ||
|
|
d0a6dc5b13 | ||
|
|
439c6c5b7c | ||
|
|
aaee0c126b | ||
|
|
f3bd4d71de | ||
|
|
876d056705 | ||
|
|
e988978fa1 | ||
|
|
d5deede7a1 | ||
|
|
4f70f5c280 | ||
|
|
a1ad020b63 | ||
|
|
8f1b0df74b | ||
|
|
0f8dc6c1d3 | ||
|
|
8ae9db3b2d | ||
|
|
e477777f49 | ||
|
|
7a132bdf24 | ||
|
|
5e2b7825dc | ||
|
|
6d30c109e4 | ||
|
|
58bd931d90 | ||
|
|
854499382e | ||
|
|
cb4c1f9ad2 | ||
|
|
b452fb3ad2 | ||
|
|
c17a1fea77 | ||
|
|
d50511c5c6 | ||
|
|
bae6d5f375 | ||
|
|
b5082f6af8 |
@@ -1,13 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: If you're just not sure how to do something, see "ask a question".
|
||||
labels: enhancement, needs-triage
|
||||
---
|
||||
|
||||
### Include required information
|
||||
|
||||
Please be sure to include at least:
|
||||
|
||||
- what problem your new feature would solve
|
||||
- how or why you think it is generally useful (i.e., not just for you)
|
||||
- what alternatives or workarounds you considered
|
||||
@@ -0,0 +1,28 @@
|
||||
name: Feature request
|
||||
description: File a new feature request
|
||||
labels: ["enhancement", "needs-triage"]
|
||||
body:
|
||||
|
||||
- type: textarea
|
||||
id: feature
|
||||
attributes:
|
||||
label: Feature description
|
||||
description: Please describe the behavior you'd like to see.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: problem-usecase
|
||||
attributes:
|
||||
label: Problem or use case
|
||||
description: Please explain which problem this would solve, or what the use case is for the feature. Keep in mind that it's more likely to be implemented if it's generally useful for a larger number of users.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives or workarounds
|
||||
description: Please describe any alternatives or workarounds you have considered and, possibly, rejected.
|
||||
validations:
|
||||
required: true
|
||||
@@ -1,23 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: If you're actually looking for support, see "ask a question".
|
||||
labels: bug, needs-triage
|
||||
---
|
||||
|
||||
### Does your log mention database corruption?
|
||||
|
||||
If your Syncthing log reports panics because of database corruption it is
|
||||
most likely a fault with your system's storage or memory. Affected log
|
||||
entries will contain lines starting with `panic: leveldb`. You will need to
|
||||
delete the index database to clear this, by running `syncthing
|
||||
-reset-database`.
|
||||
|
||||
### Include required information
|
||||
|
||||
Please be sure to include at least:
|
||||
|
||||
- which version of Syncthing and what operating system you are using
|
||||
- browser and version, if applicable
|
||||
- what happened,
|
||||
- what you expected to happen instead, and
|
||||
- any steps to reproduce the problem.
|
||||
@@ -0,0 +1,51 @@
|
||||
name: Bug report
|
||||
description: If you're actually looking for support instead, see "I need help / I have a question".
|
||||
labels: ["bug", "needs-triage"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
:no_entry_sign: If you want to report a security issue, please see [our Security Policy](https://syncthing.net/security/) and do not report the issue here.
|
||||
|
||||
:interrobang: If you are not sure if there is a bug, but something isn't working right and you need help, please [use the forum](https://forum.syncthing.net/).
|
||||
|
||||
- type: textarea
|
||||
id: what-happened
|
||||
attributes:
|
||||
label: What happened?
|
||||
description: Also tell us, what did you expect to happen, and any steps we might use to reproduce the problem.
|
||||
placeholder: Tell us what you see!
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Syncthing version
|
||||
description: What version of Syncthing are you running?
|
||||
placeholder: v1.27.4
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: platform
|
||||
attributes:
|
||||
label: Platform & operating system
|
||||
description: On what platform(s) are you seeing the problem?
|
||||
placeholder: Linux arm64
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: browser
|
||||
attributes:
|
||||
label: Browser version
|
||||
description: If the problem is related to the GUI, describe your browser and version.
|
||||
placeholder: Safari 17.3.1
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Relevant log output
|
||||
description: Please copy and paste any relevant log output or crash backtrace. This will be automatically formatted into code, so no need for backticks.
|
||||
render: shell
|
||||
@@ -4,30 +4,37 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- infrastructure
|
||||
- infra-*
|
||||
|
||||
env:
|
||||
GO_VERSION: "~1.21.1"
|
||||
CGO_ENABLED: "0"
|
||||
BUILD_USER: docker
|
||||
BUILD_HOST: github.syncthing.net
|
||||
GO_VERSION: "~1.23.0"
|
||||
CGO_ENABLED: "0"
|
||||
BUILD_USER: docker
|
||||
BUILD_HOST: github.syncthing.net
|
||||
|
||||
jobs:
|
||||
|
||||
docker-syncthing:
|
||||
name: Build and push Docker images
|
||||
if: github.repository == 'syncthing/syncthing'
|
||||
runs-on: ubuntu-latest
|
||||
environment: docker
|
||||
strategy:
|
||||
matrix:
|
||||
pkg:
|
||||
- stcrashreceiver
|
||||
- strelaypoolsrv
|
||||
- stupgrades
|
||||
matrix:
|
||||
pkg:
|
||||
- stcrashreceiver
|
||||
- strelaypoolsrv
|
||||
- stupgrades
|
||||
- ursrv
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
check-latest: true
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -44,6 +51,17 @@ jobs:
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Set Docker tags (all branches)
|
||||
run: |
|
||||
tags=syncthing/${{ matrix.pkg }}:${{ github.sha }}
|
||||
echo "TAGS=$tags" >> $GITHUB_ENV
|
||||
|
||||
- name: Set Docker tags (latest)
|
||||
if: github.ref == 'refs/heads/infrastructure'
|
||||
run: |
|
||||
tags=syncthing/${{ matrix.pkg }}:latest,${{ env.TAGS }}
|
||||
echo "TAGS=$tags" >> $GITHUB_ENV
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
@@ -51,4 +69,6 @@ jobs:
|
||||
file: ./Dockerfile.${{ matrix.pkg }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: syncthing/${{ matrix.pkg }}:latest,syncthing/${{ matrix.pkg }}:${{ github.sha }}
|
||||
tags: ${{ env.TAGS }}
|
||||
labels: |
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
|
||||
@@ -12,7 +12,7 @@ env:
|
||||
# The go version to use for builds. We set check-latest to true when
|
||||
# installing, so we get the latest patch version that matches the
|
||||
# expression.
|
||||
GO_VERSION: "~1.21.1"
|
||||
GO_VERSION: "~1.23.0"
|
||||
|
||||
# Optimize compatibility on the slow archictures.
|
||||
GO386: softfloat
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
|
||||
# The oldest version in this list should match what we have in our go.mod.
|
||||
# Variables don't seem to be supported here, or we could have done something nice.
|
||||
go: ["1.20", "1.21"]
|
||||
go: ["~1.22.6", "~1.23.0"]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- name: Set git to use LF
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
cache: true
|
||||
@@ -79,6 +79,7 @@ jobs:
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
go version
|
||||
go run build.go test | go-test-json-to-loki
|
||||
env:
|
||||
GOFLAGS: "-json"
|
||||
@@ -97,7 +98,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
@@ -153,13 +154,18 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- uses: actions/cache@v3
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~\AppData\Local\go-build
|
||||
@@ -184,7 +190,7 @@ jobs:
|
||||
CODESIGN_TIMESTAMP_SERVER: ${{ secrets.CODESIGN_TIMESTAMP_SERVER }}
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-windows
|
||||
path: syncthing-windows-*.zip
|
||||
@@ -201,13 +207,18 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- uses: actions/cache@v3
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
@@ -224,10 +235,12 @@ jobs:
|
||||
CGO_ENABLED: "0"
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-linux
|
||||
path: syncthing-linux-*.tar.gz
|
||||
path: |
|
||||
syncthing-linux-*.tar.gz
|
||||
compat.json
|
||||
|
||||
#
|
||||
# macOS
|
||||
@@ -243,13 +256,18 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- uses: actions/cache@v3
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
@@ -318,7 +336,7 @@ jobs:
|
||||
zip -r "../$universal.zip" "$universal"
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-macos
|
||||
path: syncthing-*.zip
|
||||
@@ -333,7 +351,7 @@ jobs:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: packages-macos
|
||||
|
||||
@@ -365,13 +383,18 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- uses: actions/cache@v3
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
@@ -410,7 +433,7 @@ jobs:
|
||||
CGO_ENABLED: "0"
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-other
|
||||
path: syncthing-*.tar.gz
|
||||
@@ -427,7 +450,7 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
@@ -450,7 +473,7 @@ jobs:
|
||||
mv "syncthing-source-$version.tar.gz" syncthing
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-source
|
||||
path: syncthing-source-*.tar.gz
|
||||
@@ -483,11 +506,17 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Install signing tool
|
||||
run: |
|
||||
go install ./cmd/stsigtool
|
||||
go install ./cmd/dev/stsigtool
|
||||
|
||||
- name: Sign archives
|
||||
run: |
|
||||
@@ -522,7 +551,7 @@ jobs:
|
||||
GNUPG_SIGNING_KEY_BASE64: ${{ secrets.GNUPG_SIGNING_KEY_BASE64 }}
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packages-signed
|
||||
path: packages/*
|
||||
@@ -539,12 +568,17 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: '3.0'
|
||||
@@ -553,7 +587,7 @@ jobs:
|
||||
run: |
|
||||
gem install fpm
|
||||
|
||||
- uses: actions/cache@v3
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
@@ -569,7 +603,7 @@ jobs:
|
||||
BUILD_USER: debian
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-packages
|
||||
path: "*.deb"
|
||||
@@ -594,29 +628,36 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: packages-signed
|
||||
path: packages
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Create release json
|
||||
run: |
|
||||
cd packages
|
||||
"$GITHUB_WORKSPACE/tools/generate-release-json" "$BASE_URL" > nightly.json
|
||||
env:
|
||||
BASE_URL: https://syncthing.ams3.digitaloceanspaces.com/nightly/
|
||||
BASE_URL: ${{ secrets.NIGHTLY_BASE_URL }}
|
||||
|
||||
- name: Push artifacts
|
||||
uses: docker://docker.io/rclone/rclone:latest
|
||||
env:
|
||||
RCLONE_CONFIG_SPACES_TYPE: s3
|
||||
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
|
||||
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
|
||||
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
|
||||
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
|
||||
RCLONE_CONFIG_SPACES_ACL: public-read
|
||||
RCLONE_CONFIG_OBJSTORE_TYPE: s3
|
||||
RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
|
||||
RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
|
||||
RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
|
||||
RCLONE_CONFIG_OBJSTORE_ACL: public-read
|
||||
with:
|
||||
args: sync packages spaces:syncthing/nightly
|
||||
args: sync packages objstore:${{ secrets.S3_BUCKET }}/nightly
|
||||
|
||||
#
|
||||
# Push release artifacts to Spaces
|
||||
@@ -636,45 +677,53 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download signed packages
|
||||
uses: actions/download-artifact@v3
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: packages-signed
|
||||
path: packages
|
||||
|
||||
- name: Download debian packages
|
||||
uses: actions/download-artifact@v3
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: debian-packages
|
||||
path: packages
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Set version
|
||||
run: |
|
||||
version=$(go run build.go version)
|
||||
echo "VERSION=$version" >> $GITHUB_ENV
|
||||
|
||||
- name: Push to Spaces (${{ env.VERSION }})
|
||||
- name: Push to object store (${{ env.VERSION }})
|
||||
uses: docker://docker.io/rclone/rclone:latest
|
||||
env:
|
||||
RCLONE_CONFIG_SPACES_TYPE: s3
|
||||
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
|
||||
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
|
||||
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
|
||||
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
|
||||
RCLONE_CONFIG_SPACES_ACL: public-read
|
||||
RCLONE_CONFIG_OBJSTORE_TYPE: s3
|
||||
RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
|
||||
RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
|
||||
RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
|
||||
RCLONE_CONFIG_OBJSTORE_ACL: public-read
|
||||
with:
|
||||
args: sync packages spaces:syncthing/release/${{ env.VERSION }}
|
||||
args: sync packages objstore:${{ secrets.S3_BUCKET }}/release/${{ env.VERSION }}
|
||||
|
||||
- name: Push to Spaces (latest)
|
||||
- name: Push to object store (latest)
|
||||
uses: docker://docker.io/rclone/rclone:latest
|
||||
env:
|
||||
RCLONE_CONFIG_SPACES_TYPE: s3
|
||||
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
|
||||
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
|
||||
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
|
||||
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
|
||||
RCLONE_CONFIG_SPACES_ACL: public-read
|
||||
RCLONE_CONFIG_OBJSTORE_TYPE: s3
|
||||
RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
|
||||
RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
|
||||
RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
|
||||
RCLONE_CONFIG_OBJSTORE_ACL: public-read
|
||||
with:
|
||||
args: sync spaces:syncthing/release/${{ env.VERSION }} spaces:syncthing/release/latest
|
||||
args: sync objstore:${{ secrets.S3_BUCKET }}/release/${{ env.VERSION }} objstore:${{ secrets.S3_BUCKET }}/release/latest
|
||||
|
||||
#
|
||||
# Build and push to Docker Hub
|
||||
@@ -683,7 +732,7 @@ jobs:
|
||||
docker-syncthing:
|
||||
name: Build and push Docker images
|
||||
runs-on: ubuntu-latest
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-'))
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/infrastructure' || startsWith(github.ref, 'refs/heads/release-'))
|
||||
environment: docker
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -706,13 +755,18 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- uses: actions/cache@v3
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
@@ -762,6 +816,7 @@ jobs:
|
||||
tags=${{ matrix.image }}:edge
|
||||
fi
|
||||
echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
|
||||
echo "VERSION=$version" >> $GITHUB_ENV
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
@@ -771,6 +826,9 @@ jobs:
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/7
|
||||
push: ${{ env.DOCKER_PUSH == 'true' }}
|
||||
tags: ${{ env.DOCKER_TAGS }}
|
||||
labels: |
|
||||
org.opencontainers.image.version=${{ env.VERSION }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
|
||||
#
|
||||
# Check for known vulnerabilities in Go dependencies
|
||||
@@ -782,7 +840,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
|
||||
@@ -14,9 +14,9 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ^1.19.6
|
||||
go-version: stable
|
||||
- run: |
|
||||
set -euo pipefail
|
||||
git config --global user.name 'Syncthing Release Automation'
|
||||
|
||||
@@ -18,3 +18,4 @@ deb
|
||||
/repos
|
||||
/proto/scripts/protoc-gen-gosyncthing
|
||||
/gui/next-gen-gui
|
||||
/compat.json
|
||||
|
||||
@@ -27,6 +27,7 @@ Alexander Seiler <seileralex@gmail.com>
|
||||
Alexandre Alves <alexandrealvesdb.contact@gmail.com>
|
||||
Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
|
||||
Aman Gupta <aman@tmm1.net>
|
||||
Anatoli Babenia <anatoli@rainforce.org>
|
||||
Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
|
||||
Andreas Sommer <andreas.sommer87@googlemail.com>
|
||||
andresvia <andres.via@gmail.com>
|
||||
@@ -50,6 +51,7 @@ Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github
|
||||
Aurélien Rainone <476650+arl@users.noreply.github.com>
|
||||
BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
|
||||
Bart De Vries (mogwa1) <devriesb@gmail.com>
|
||||
Beat Reichenbach <44111292+beatreichenbach@users.noreply.github.com>
|
||||
Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
|
||||
Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
|
||||
Ben Shepherd (benshep) <bjashepherd@gmail.com>
|
||||
@@ -68,6 +70,7 @@ Brian R. Becker (brbecker) <brbecker@gmail.com>
|
||||
bt90 <btom1990@googlemail.com>
|
||||
Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
|
||||
Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
|
||||
Catfriend1 <16361913+Catfriend1@users.noreply.github.com>
|
||||
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
|
||||
Cedric Staniewski (xduugu) <cedric@gmx.ca>
|
||||
chenrui <rui@meetup.com>
|
||||
@@ -79,6 +82,7 @@ Chris Tonkinson <chris@masterbran.ch>
|
||||
Christian Kujau <ckujau@users.noreply.github.com>
|
||||
Christian Prescott <me@christianprescott.com>
|
||||
chucic <chucic@seznam.cz>
|
||||
cjc7373 <niuchangcun@gmail.com>
|
||||
Colin Kennedy (moshen) <moshen.colin@gmail.com>
|
||||
Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
|
||||
cui fliter <imcusg@gmail.com>
|
||||
@@ -90,6 +94,7 @@ Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
|
||||
Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
|
||||
Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
|
||||
Daniel Martí (mvdan) <mvdan@mvdan.cc>
|
||||
Daniel Padrta <64928366+danpadcz@users.noreply.github.com>
|
||||
Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
|
||||
David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
|
||||
deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
|
||||
@@ -99,8 +104,10 @@ Dennis Wilson (snnd) <dw@risu.io>
|
||||
dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
|
||||
dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
|
||||
derekriemer <derek.riemer@colorado.edu>
|
||||
DerRockWolf <50499906+DerRockWolf@users.noreply.github.com>
|
||||
desbma <desbma@users.noreply.github.com>
|
||||
Devon G. Redekopp <devon@redekopp.com>
|
||||
diemade <spamkill@posteo.ch>
|
||||
digital <didev@dinid.net>
|
||||
Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
|
||||
Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
|
||||
@@ -133,6 +140,8 @@ Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
|
||||
greatroar <61184462+greatroar@users.noreply.github.com>
|
||||
Greg <gco@jazzhaiku.com>
|
||||
guangwu <guoguangwu@magic-shield.com>
|
||||
gudvinr <gudvinr@gmail.com>
|
||||
Gusted <postmaster@gusted.xyz> <williamzijl7@hotmail.com>
|
||||
Han Boetes <han@boetes.org>
|
||||
HansK-p <42314815+HansK-p@users.noreply.github.com>
|
||||
Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
|
||||
@@ -149,13 +158,14 @@ Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
|
||||
Jack Croft <jccroft1@users.noreply.github.com>
|
||||
Jacob <jyundt@gmail.com>
|
||||
Jake Peterson (acogdev) <jake@acogdev.com>
|
||||
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
|
||||
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net> <jborg@coreweave.com>
|
||||
James O'Beirne <wild-github@au92.org>
|
||||
James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
|
||||
janost <janost@tuta.io>
|
||||
Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
|
||||
Jaroslav Malec (dzarda) <dzardacz@gmail.com>
|
||||
jaseg <githubaccount@jaseg.net>
|
||||
Jaspitta <ste.scarpitta@gmail.com>
|
||||
Jauder Ho <jauderho@users.noreply.github.com>
|
||||
Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
|
||||
Jaya Kumar <jaya.kumar@ict.nl>
|
||||
@@ -174,6 +184,7 @@ Jonathan Cross <jcross@gmail.com>
|
||||
Jonta <359397+Jonta@users.noreply.github.com>
|
||||
Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
|
||||
jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
|
||||
Julian Lehrhuber <jul13579@users.noreply.github.com>
|
||||
Jörg Thalheim <Mic92@users.noreply.github.com>
|
||||
Jędrzej Kula <kula.jedrek@gmail.com>
|
||||
K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
|
||||
@@ -189,6 +200,7 @@ Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.g
|
||||
Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
|
||||
klemens <ka7@github.com>
|
||||
Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
|
||||
kylosus <33132401+kylosus@users.noreply.github.com>
|
||||
Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
|
||||
Lars Lehtonen <lars.lehtonen@gmail.com>
|
||||
Laurent Arnoud <laurent@spkdev.net>
|
||||
@@ -198,7 +210,9 @@ Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
|
||||
Lode Hoste (Zillode) <zillode@zillode.be>
|
||||
Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
|
||||
LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
|
||||
luchenhan <168071714+luchenhan@users.noreply.github.com>
|
||||
Lukas Lihotzki <lukas@lihotzki.de>
|
||||
Luke Hamburg <1992842+luckman212@users.noreply.github.com>
|
||||
luzpaz <luzpaz@users.noreply.github.com>
|
||||
Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
|
||||
Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
|
||||
@@ -221,7 +235,7 @@ Max <github@germancoding.com>
|
||||
Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
|
||||
MaximAL <almaximal@ya.ru>
|
||||
Maxime Thirouin <m@moox.io>
|
||||
Maximilian <maxi.rostock@outlook.de>
|
||||
Maximilian <maxi.rostock@outlook.de> <public@complexvector.space>
|
||||
mclang <1721600+mclang@users.noreply.github.com>
|
||||
Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
|
||||
Michael Ploujnikov (plouj) <ploujj@gmail.com>
|
||||
@@ -236,6 +250,7 @@ Mingxuan Lin <gdlmx@users.noreply.github.com>
|
||||
mv1005 <49659413+mv1005@users.noreply.github.com>
|
||||
Nate Morrison (nrm21) <natemorrison@gmail.com>
|
||||
Naveen <172697+naveensrinivasan@users.noreply.github.com>
|
||||
nf <nf@wh3rd.net>
|
||||
Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
|
||||
Nick Busey <NickBusey@users.noreply.github.com>
|
||||
Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
|
||||
@@ -286,30 +301,39 @@ Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
|
||||
Scott Klupfel (kluppy) <kluppy@going2blue.com>
|
||||
sec65 <106604020+sec65@users.noreply.github.com>
|
||||
Sergey Mishin (ralder) <ralder@yandex.ru>
|
||||
Sertonix <83883937+Sertonix@users.noreply.github.com>
|
||||
Severin von Wnuck-Lipinski <ss7@live.de>
|
||||
Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
|
||||
Simon Frei (imsodin) <freisim93@gmail.com>
|
||||
Simon Mwepu <simonmwepu@gmail.com>
|
||||
Simon Pickup <simon@pickupinfinity.com>
|
||||
Sly_tom_cat <slytomcat@mail.ru>
|
||||
Sonu Kumar Saw <31889738+dev-saw99@users.noreply.github.com>
|
||||
Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
|
||||
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
|
||||
Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
|
||||
Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
|
||||
Sven Bachmann <dev@mcbachmann.de>
|
||||
Syncthing Automation <automation@syncthing.net>
|
||||
Syncthing Release Automation <release@syncthing.net>
|
||||
Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
|
||||
Thomas <9749173+uhthomas@users.noreply.github.com>
|
||||
Thomas Hipp <thomashipp@gmail.com>
|
||||
Tim Abell (timabell) <tim@timwise.co.uk>
|
||||
Tim Howes (timhowes) <timhowes@berkeley.edu>
|
||||
Tim Nordenfur <tim@gurka.se>
|
||||
Tobias Klauser <tobias.klauser@gmail.com>
|
||||
Tobias Nygren (tnn2) <tnn@nygren.pp.se>
|
||||
Tobias Tom (tobiastom) <t.tom@succont.de>
|
||||
Tom Jakubowski <tom@crystae.net>
|
||||
Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
|
||||
Tommy Thorn <tommy-github-email@thorn.ws>
|
||||
Tommy van der Vorst <tommy-github@pixelspark.nl> <tommy@pixelspark.nl>
|
||||
Tully Robinson (tojrobinson) <tully@tojr.org>
|
||||
Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
|
||||
Tyler Kropp <kropptyler@gmail.com>
|
||||
Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
|
||||
vapatel2 <149737089+vapatel2@users.noreply.github.com>
|
||||
Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
|
||||
Victor Buinsky (buinsky) <vix_booja@tut.by>
|
||||
Vik <63919734+ViktorOn@users.noreply.github.com>
|
||||
@@ -317,6 +341,7 @@ Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
|
||||
villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
|
||||
Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
|
||||
wangguoliang <liangcszzu@163.com>
|
||||
WangXi <xib1102@icloud.com>
|
||||
Will Rouesnel <wrouesnel@wrouesnel.com>
|
||||
William A. Kennington III (wkennington) <william@wkennington.com>
|
||||
wouter bolsterlee <wouter@bolsterl.ee>
|
||||
|
||||
@@ -29,6 +29,14 @@ RUN if [ ! -f syncthing-linux-$TARGETARCH ] ; then \
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing"
|
||||
|
||||
EXPOSE 8384 22000/tcp 22000/udp 21027/udp
|
||||
|
||||
VOLUME ["/var/syncthing"]
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
ARG GOVERSION=latest
|
||||
FROM golang:$GOVERSION
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Builder"
|
||||
|
||||
# FPM to build Debian packages
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
locales rubygems ruby-dev build-essential git \
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Crash Receiver"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
COPY stcrashreceiver-linux-${TARGETARCH} /bin/stcrashreceiver
|
||||
|
||||
@@ -16,6 +16,14 @@ RUN if [ ! -f stdiscosrv-linux-$TARGETARCH ] ; then \
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Discovery Server"
|
||||
|
||||
EXPOSE 19200 8443
|
||||
|
||||
VOLUME ["/var/stdiscosrv"]
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Relay Pool Server"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
RUN apk add --no-cache ca-certificates su-exec curl
|
||||
ENV PUID=1000 PGID=1000 MAXMIND_KEY=
|
||||
|
||||
RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
|
||||
USER 1000
|
||||
|
||||
COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
|
||||
COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
|
||||
|
||||
WORKDIR /var/strelaypoolsrv
|
||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]
|
||||
ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"]
|
||||
|
||||
@@ -16,6 +16,14 @@ RUN if [ ! -f strelaysrv-linux-$TARGETARCH ] ; then \
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Relay Server"
|
||||
|
||||
EXPOSE 22067 22070
|
||||
|
||||
VOLUME ["/var/strelaysrv"]
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Upgrades"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
COPY stupgrades-linux-${TARGETARCH} /bin/stupgrades
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Usage Reporting Server"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
COPY ursrv-linux-${TARGETARCH} /bin/ursrv
|
||||
|
||||
ENTRYPOINT [ "/bin/ursrv" ]
|
||||
@@ -15,6 +15,9 @@ To grant Syncthing additional capabilities without running as root, use the
|
||||
`PCAP` environment variable with the same syntax as that for `setcap(8)`.
|
||||
For example, `PCAP=cap_chown,cap_fowner+ep`.
|
||||
|
||||
To set a different umask value, use the `UMASK` environment variable. For
|
||||
example `UMASK=002`.
|
||||
|
||||
## Example Usage
|
||||
|
||||
**Docker cli**
|
||||
|
||||
@@ -63,12 +63,6 @@ implementations][11] for Windows, Mac, and Linux.
|
||||
|
||||
To run Syncthing in Docker, see [the Docker README][16].
|
||||
|
||||
## Vote on features/bugs
|
||||
|
||||
We'd like to encourage you to [vote][12] on issues that matter to you.
|
||||
This helps the team understand what are the biggest pain points for our
|
||||
users, and could potentially influence what is being worked on next.
|
||||
|
||||
## Getting in Touch
|
||||
|
||||
The first and best point of contact is the [Forum][8].
|
||||
@@ -89,7 +83,7 @@ build process.
|
||||
## Signed Releases
|
||||
|
||||
As of v0.10.15 and onwards, release binaries are GPG signed with the key
|
||||
D26E6ED000654A3E, available from https://syncthing.net/security.html and
|
||||
D26E6ED000654A3E, available from https://syncthing.net/security/ and
|
||||
most key servers.
|
||||
|
||||
There is also a built-in automatic upgrade mechanism (disabled in some
|
||||
@@ -111,7 +105,6 @@ All code is licensed under the [MPLv2 License][7].
|
||||
[8]: https://forum.syncthing.net/
|
||||
[10]: https://github.com/syncthing/syncthing/issues
|
||||
[11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
|
||||
[12]: https://www.bountysource.com/teams/syncthing/issues
|
||||
[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
|
||||
[14]: assets/logo-text-128.png
|
||||
[15]: https://syncthing.net/
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build ignore
|
||||
// +build ignore
|
||||
//go:build tools
|
||||
// +build tools
|
||||
|
||||
package main
|
||||
|
||||
@@ -34,6 +34,8 @@ import (
|
||||
"time"
|
||||
|
||||
buildpkg "github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -118,7 +120,6 @@ var targets = map[string]target{
|
||||
{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0644},
|
||||
{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0644},
|
||||
{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0644},
|
||||
{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
|
||||
{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
|
||||
{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0644},
|
||||
{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
|
||||
@@ -192,37 +193,37 @@ var targets = map[string]target{
|
||||
debname: "syncthing-relaypoolsrv",
|
||||
debdeps: []string{"libc6"},
|
||||
description: "Syncthing Relay Pool Server",
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv"},
|
||||
binaryName: "strelaypoolsrv", // .exe will be added automatically for Windows builds
|
||||
archiveFiles: []archiveFile{
|
||||
{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
|
||||
{src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
|
||||
{src: "cmd/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
|
||||
{src: "cmd/infra/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
|
||||
{src: "cmd/infra/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
|
||||
{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
|
||||
},
|
||||
installationFiles: []archiveFile{
|
||||
{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
|
||||
{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
|
||||
{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
|
||||
{src: "cmd/infra/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
|
||||
{src: "cmd/infra/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
|
||||
{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
|
||||
},
|
||||
},
|
||||
"stupgrades": {
|
||||
name: "stupgrades",
|
||||
description: "Syncthing Upgrade Check Server",
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/stupgrades"},
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/infra/stupgrades"},
|
||||
binaryName: "stupgrades",
|
||||
},
|
||||
"stcrashreceiver": {
|
||||
name: "stcrashreceiver",
|
||||
description: "Syncthing Crash Server",
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/stcrashreceiver"},
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/infra/stcrashreceiver"},
|
||||
binaryName: "stcrashreceiver",
|
||||
},
|
||||
"ursrv": {
|
||||
name: "ursrv",
|
||||
description: "Syncthing Usage Reporting Server",
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/ursrv"},
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/infra/ursrv"},
|
||||
binaryName: "ursrv",
|
||||
},
|
||||
}
|
||||
@@ -231,15 +232,11 @@ func initTargets() {
|
||||
all := targets["all"]
|
||||
pkgs, _ := filepath.Glob("cmd/*")
|
||||
for _, pkg := range pkgs {
|
||||
pkg = filepath.Base(pkg)
|
||||
if strings.HasPrefix(pkg, ".") {
|
||||
// ignore dotfiles
|
||||
if files, err := filepath.Glob(pkg + "/*.go"); err != nil || len(files) == 0 {
|
||||
// No go files in the directory
|
||||
continue
|
||||
}
|
||||
if noupgrade && pkg == "stupgrades" {
|
||||
continue
|
||||
}
|
||||
all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
|
||||
all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/%s", pkg))
|
||||
}
|
||||
targets["all"] = all
|
||||
|
||||
@@ -343,9 +340,11 @@ func runCommand(cmd string, target target) {
|
||||
|
||||
case "tar":
|
||||
buildTar(target, tags)
|
||||
writeCompatJSON()
|
||||
|
||||
case "zip":
|
||||
buildZip(target, tags)
|
||||
writeCompatJSON()
|
||||
|
||||
case "deb":
|
||||
buildDeb(target)
|
||||
@@ -835,12 +834,12 @@ func listFiles(dir string) []string {
|
||||
|
||||
func rebuildAssets() {
|
||||
os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
|
||||
runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
|
||||
runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto")
|
||||
}
|
||||
|
||||
func lazyRebuildAssets() {
|
||||
shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
|
||||
shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/gui")
|
||||
shouldRebuildAssets("cmd/infra/strelaypoolsrv/auto/gui.files.go", "cmd/infra/strelaypoolsrv/gui")
|
||||
|
||||
if withNextGenGUI {
|
||||
shouldRebuild = buildNextGenGUI() || shouldRebuild
|
||||
@@ -1558,3 +1557,29 @@ func nextPatchVersion(ver string) string {
|
||||
digits[len(digits)-1] = strconv.Itoa(n + 1)
|
||||
return strings.Join(digits, ".")
|
||||
}
|
||||
|
||||
func writeCompatJSON() {
|
||||
bs, err := os.ReadFile("compat.yaml")
|
||||
if err != nil {
|
||||
log.Fatal("Reading compat.yaml:", err)
|
||||
}
|
||||
|
||||
var entries []upgrade.ReleaseCompatibility
|
||||
if err := yaml.Unmarshal(bs, &entries); err != nil {
|
||||
log.Fatal("Parsing compat.yaml:", err)
|
||||
}
|
||||
|
||||
rt := runtime.Version()
|
||||
for _, e := range entries {
|
||||
if !strings.HasPrefix(rt, e.Runtime) {
|
||||
continue
|
||||
}
|
||||
bs, _ := json.MarshalIndent(e, "", " ")
|
||||
if err := os.WriteFile("compat.json", bs, 0o644); err != nil {
|
||||
log.Fatal("Writing compat.json:", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
log.Fatalf("runtime %v not found in compat.yaml", rt)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -15,7 +16,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/beacon"
|
||||
"github.com/syncthing/syncthing/lib/discover"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
)
|
||||
|
||||
type event struct {
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/scanner"
|
||||
)
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/discover"
|
||||
"github.com/syncthing/syncthing/lib/events"
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/ignore"
|
||||
)
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -43,7 +45,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
|
||||
}
|
||||
|
||||
p0 := filepath.Join(dir, string(n[0]), n[0:2])
|
||||
err = os.MkdirAll(p0, 0755)
|
||||
err = os.MkdirAll(p0, 0o755)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -82,7 +84,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
|
||||
os.Chmod(p1, os.FileMode(rand.Intn(0o777)|0o400))
|
||||
|
||||
t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
|
||||
return os.Chtimes(p1, t, t)
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/signature"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
)
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
@@ -157,7 +158,7 @@ func saveCert(priv interface{}, derBytes []byte) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
@@ -7,13 +7,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -40,7 +41,7 @@ type currentFile struct {
|
||||
}
|
||||
|
||||
func (d *diskStore) Serve(ctx context.Context) {
|
||||
if err := os.MkdirAll(d.dir, 0750); err != nil {
|
||||
if err := os.MkdirAll(d.dir, 0o700); err != nil {
|
||||
log.Println("Creating directory:", err)
|
||||
return
|
||||
}
|
||||
@@ -60,7 +61,7 @@ func (d *diskStore) Serve(ctx context.Context) {
|
||||
case entry := <-d.inbox:
|
||||
path := d.fullPath(entry.path)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
log.Println("Creating directory:", err)
|
||||
continue
|
||||
}
|
||||
@@ -75,7 +76,7 @@ func (d *diskStore) Serve(ctx context.Context) {
|
||||
log.Println("Failed to compress crash report:", err)
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(path, buf.Bytes(), 0644); err != nil {
|
||||
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
|
||||
log.Printf("Failed to write %s: %v", entry.path, err)
|
||||
_ = os.Remove(path)
|
||||
continue
|
||||
@@ -147,6 +148,11 @@ func (d *diskStore) clean() {
|
||||
if len(d.currentFiles) > 0 {
|
||||
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
|
||||
}
|
||||
|
||||
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
|
||||
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
|
||||
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
|
||||
|
||||
log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
|
||||
}
|
||||
|
||||
@@ -178,6 +184,11 @@ func (d *diskStore) inventory() error {
|
||||
if len(d.currentFiles) > 0 {
|
||||
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
|
||||
}
|
||||
|
||||
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
|
||||
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
|
||||
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
|
||||
|
||||
log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
|
||||
return err
|
||||
}
|
||||
@@ -14,6 +14,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -21,26 +22,29 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
"github.com/syncthing/syncthing/lib/ur"
|
||||
|
||||
raven "github.com/getsentry/raven-go"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/ur"
|
||||
)
|
||||
|
||||
const maxRequestSize = 1 << 20 // 1 MiB
|
||||
|
||||
type cli struct {
|
||||
Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
|
||||
DSN string `help:"Sentry DSN" env:"SENTRY_DSN"`
|
||||
Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
|
||||
MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
|
||||
MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
|
||||
CleanInterval time.Duration `help:"Interval between cleaning up old reports" default:"12h" env:"CLEAN_INTERVAL"`
|
||||
SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
|
||||
DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
|
||||
Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
|
||||
DSN string `help:"Sentry DSN" env:"SENTRY_DSN"`
|
||||
Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
|
||||
MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
|
||||
MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
|
||||
SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
|
||||
DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
|
||||
MetricsListen string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"`
|
||||
IngorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -63,9 +67,19 @@ func main() {
|
||||
}
|
||||
go ss.Serve(context.Background())
|
||||
|
||||
var ip *ignorePatterns
|
||||
if params.IngorePatterns != "" {
|
||||
var err error
|
||||
ip, err = loadIgnorePatterns(params.IngorePatterns)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load ignore patterns: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
cr := &crashReceiver{
|
||||
store: ds,
|
||||
sentry: ss,
|
||||
ignore: ip,
|
||||
}
|
||||
|
||||
mux.Handle("/", cr)
|
||||
@@ -73,8 +87,18 @@ func main() {
|
||||
w.Write([]byte("OK"))
|
||||
})
|
||||
|
||||
if params.MetricsListen != "" {
|
||||
mmux := http.NewServeMux()
|
||||
mmux.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil {
|
||||
log.Fatalln("HTTP serve metrics:", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if params.DSN != "" {
|
||||
mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
|
||||
mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"), ip))
|
||||
}
|
||||
|
||||
log.SetOutput(os.Stdout)
|
||||
@@ -83,8 +107,13 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
|
||||
func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http.ResponseWriter, req *http.Request) {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
result := "failure"
|
||||
defer func() {
|
||||
metricFailureReportsTotal.WithLabelValues(result).Inc()
|
||||
}()
|
||||
|
||||
lr := io.LimitReader(req.Body, maxRequestSize)
|
||||
bs, err := io.ReadAll(lr)
|
||||
req.Body.Close()
|
||||
@@ -93,6 +122,11 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht
|
||||
return
|
||||
}
|
||||
|
||||
if ignore.match(bs) {
|
||||
result = "ignored"
|
||||
return
|
||||
}
|
||||
|
||||
var reports []ur.FailureReport
|
||||
err = json.Unmarshal(bs, &reports)
|
||||
if err != nil {
|
||||
@@ -105,7 +139,7 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht
|
||||
return
|
||||
}
|
||||
|
||||
version, err := parseVersion(reports[0].Version)
|
||||
version, err := build.ParseVersion(reports[0].Version)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
@@ -135,6 +169,7 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht
|
||||
log.Println("Failed to send failure report:", err)
|
||||
} else {
|
||||
log.Println("Sent failure report:", r.Description)
|
||||
result = "success"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -152,3 +187,42 @@ func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string,
|
||||
}
|
||||
return reportServer + path, nil
|
||||
}
|
||||
|
||||
type ignorePatterns struct {
|
||||
patterns []*regexp.Regexp
|
||||
}
|
||||
|
||||
func loadIgnorePatterns(path string) (*ignorePatterns, error) {
|
||||
bs, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var patterns []*regexp.Regexp
|
||||
for _, line := range strings.Split(string(bs), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
re, err := regexp.Compile(line)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
patterns = append(patterns, re)
|
||||
}
|
||||
|
||||
log.Printf("Loaded %d ignore patterns", len(patterns))
|
||||
return &ignorePatterns{patterns: patterns}, nil
|
||||
}
|
||||
|
||||
func (i *ignorePatterns) match(report []byte) bool {
|
||||
if i == nil {
|
||||
return false
|
||||
}
|
||||
for _, re := range i.patterns {
|
||||
if re.Match(report) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright (C) 2023 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
metricCrashReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "crash_reports_total",
|
||||
}, []string{"result"})
|
||||
metricFailureReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "failure_reports_total",
|
||||
}, []string{"result"})
|
||||
metricDiskstoreFilesTotal = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_files_total",
|
||||
})
|
||||
metricDiskstoreBytesTotal = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_bytes_total",
|
||||
})
|
||||
metricDiskstoreOldestAgeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_oldest_age_seconds",
|
||||
})
|
||||
)
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
|
||||
raven "github.com/getsentry/raven-go"
|
||||
"github.com/maruel/panicparse/v2/stack"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
)
|
||||
|
||||
const reportServer = "https://crash.syncthing.net/report/"
|
||||
@@ -105,7 +106,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
|
||||
return nil, errors.New("no first line")
|
||||
}
|
||||
|
||||
version, err := parseVersion(string(parts[0]))
|
||||
version, err := build.ParseVersion(string(parts[0]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -143,12 +144,12 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
|
||||
}
|
||||
|
||||
// Lock the source code loader to the version we are processing here.
|
||||
if version.commit != "" {
|
||||
if version.Commit != "" {
|
||||
// We have a commit hash, so we know exactly which source to use
|
||||
loader.LockWithVersion(version.commit)
|
||||
} else if strings.HasPrefix(version.tag, "v") {
|
||||
loader.LockWithVersion(version.Commit)
|
||||
} else if strings.HasPrefix(version.Tag, "v") {
|
||||
// Lets hope the tag is close enough
|
||||
loader.LockWithVersion(version.tag)
|
||||
loader.LockWithVersion(version.Tag)
|
||||
} else {
|
||||
// Last resort
|
||||
loader.LockWithVersion("main")
|
||||
@@ -215,106 +216,26 @@ func crashReportFingerprint(message string) []string {
|
||||
return []string{"{{ default }}", message}
|
||||
}
|
||||
|
||||
// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
|
||||
// or, somewhere along the way the "+" in the version tag disappeared:
|
||||
// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
|
||||
var (
|
||||
longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
|
||||
gitExtraRE = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
|
||||
gitExtraSepRE = regexp.MustCompile(`[.-]`) // dot or dash
|
||||
)
|
||||
|
||||
type version struct {
|
||||
version string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
|
||||
tag string // "v1.1.4-rc.1"
|
||||
commit string // "6aaae618", blank when absent
|
||||
codename string // "Erbium Earthworm"
|
||||
runtime string // "go1.12.5"
|
||||
goos string // "darwin"
|
||||
goarch string // "amd64"
|
||||
builder string // "jb@kvin.kastelo.net"
|
||||
extra []string // "foo", "bar"
|
||||
}
|
||||
|
||||
func (v version) environment() string {
|
||||
if v.commit != "" {
|
||||
return "Development"
|
||||
}
|
||||
if strings.Contains(v.tag, "-rc.") {
|
||||
return "Candidate"
|
||||
}
|
||||
if strings.Contains(v.tag, "-") {
|
||||
return "Beta"
|
||||
}
|
||||
return "Stable"
|
||||
}
|
||||
|
||||
func parseVersion(line string) (version, error) {
|
||||
m := longVersionRE.FindStringSubmatch(line)
|
||||
if len(m) == 0 {
|
||||
return version{}, errors.New("unintelligeble version string")
|
||||
}
|
||||
|
||||
v := version{
|
||||
version: m[1],
|
||||
codename: m[2],
|
||||
runtime: m[3],
|
||||
goos: m[4],
|
||||
goarch: m[5],
|
||||
builder: m[6],
|
||||
}
|
||||
|
||||
// Split the version tag into tag and commit. This is old style
|
||||
// v1.2.3-something.4+11-g12345678 or newer with just dots
|
||||
// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
|
||||
parts := []string{v.version}
|
||||
if strings.Contains(v.version, "+") {
|
||||
parts = strings.Split(v.version, "+")
|
||||
} else {
|
||||
idxs := gitExtraRE.FindStringIndex(v.version)
|
||||
if len(idxs) > 0 {
|
||||
parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]}
|
||||
}
|
||||
}
|
||||
v.tag = parts[0]
|
||||
if len(parts) > 1 {
|
||||
fields := gitExtraSepRE.Split(parts[1], -1)
|
||||
if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
|
||||
v.commit = fields[1][1:]
|
||||
}
|
||||
}
|
||||
|
||||
if len(m) >= 8 && m[7] != "" {
|
||||
tags := strings.Split(m[7], ",")
|
||||
for i := range tags {
|
||||
tags[i] = strings.TrimSpace(tags[i])
|
||||
}
|
||||
v.extra = tags
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func packet(version version, reportType string) *raven.Packet {
|
||||
func packet(version build.VersionParts, reportType string) *raven.Packet {
|
||||
pkt := &raven.Packet{
|
||||
Platform: "go",
|
||||
Release: version.tag,
|
||||
Environment: version.environment(),
|
||||
Release: version.Tag,
|
||||
Environment: version.Environment(),
|
||||
Tags: raven.Tags{
|
||||
raven.Tag{Key: "version", Value: version.version},
|
||||
raven.Tag{Key: "tag", Value: version.tag},
|
||||
raven.Tag{Key: "codename", Value: version.codename},
|
||||
raven.Tag{Key: "runtime", Value: version.runtime},
|
||||
raven.Tag{Key: "goos", Value: version.goos},
|
||||
raven.Tag{Key: "goarch", Value: version.goarch},
|
||||
raven.Tag{Key: "builder", Value: version.builder},
|
||||
raven.Tag{Key: "version", Value: version.Version},
|
||||
raven.Tag{Key: "tag", Value: version.Tag},
|
||||
raven.Tag{Key: "codename", Value: version.Codename},
|
||||
raven.Tag{Key: "runtime", Value: version.Runtime},
|
||||
raven.Tag{Key: "goos", Value: version.GOOS},
|
||||
raven.Tag{Key: "goarch", Value: version.GOARCH},
|
||||
raven.Tag{Key: "builder", Value: version.Builder},
|
||||
raven.Tag{Key: "report_type", Value: reportType},
|
||||
},
|
||||
}
|
||||
if version.commit != "" {
|
||||
pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
|
||||
if version.Commit != "" {
|
||||
pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.Commit})
|
||||
}
|
||||
for _, tag := range version.extra {
|
||||
for _, tag := range version.Extra {
|
||||
pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
|
||||
}
|
||||
return pkt
|
||||
@@ -12,66 +12,6 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseVersion(t *testing.T) {
|
||||
cases := []struct {
|
||||
longVersion string
|
||||
parsed version
|
||||
}{
|
||||
{
|
||||
longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
|
||||
parsed: version{
|
||||
version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
|
||||
tag: "v1.1.4-rc.1",
|
||||
commit: "6aaae618",
|
||||
codename: "Erbium Earthworm",
|
||||
runtime: "go1.12.5",
|
||||
goos: "darwin",
|
||||
goarch: "amd64",
|
||||
builder: "jb@kvin.kastelo.net",
|
||||
},
|
||||
},
|
||||
{
|
||||
longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
|
||||
parsed: version{
|
||||
version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
|
||||
tag: "v1.1.4-rc.1",
|
||||
commit: "6aaae618",
|
||||
codename: "Erbium Earthworm",
|
||||
runtime: "go1.12.5",
|
||||
goos: "darwin",
|
||||
goarch: "amd64",
|
||||
builder: "jb@kvin.kastelo.net",
|
||||
extra: []string{"foo", "bar"},
|
||||
},
|
||||
},
|
||||
{
|
||||
longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
|
||||
parsed: version{
|
||||
version: "v1.23.7-dev.26.gdf7b56ae-stversionextra",
|
||||
tag: "v1.23.7-dev",
|
||||
commit: "df7b56ae",
|
||||
codename: "Fermium Flea",
|
||||
runtime: "go1.20.5",
|
||||
goos: "darwin",
|
||||
goarch: "arm64",
|
||||
builder: "jb@ok.kastelo.net",
|
||||
extra: []string{"Some Wrapper", "purego", "stnoupgrade"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
v, err := parseVersion(tc.longVersion)
|
||||
if err != nil {
|
||||
t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
|
||||
continue
|
||||
}
|
||||
if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
|
||||
t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReport(t *testing.T) {
|
||||
bs, err := os.ReadFile("_testdata/panic.log")
|
||||
if err != nil {
|
||||
@@ -12,11 +12,16 @@ import (
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type crashReceiver struct {
|
||||
store *diskStore
|
||||
sentry *sentryService
|
||||
ignore *ignorePatterns
|
||||
|
||||
ignoredMut sync.RWMutex
|
||||
ignored map[string]struct{}
|
||||
}
|
||||
|
||||
func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
@@ -64,6 +69,12 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http
|
||||
// serveHead responds to HEAD requests by checking if the named report
|
||||
// already exists in the system.
|
||||
func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
|
||||
r.ignoredMut.RLock()
|
||||
_, ignored := r.ignored[reportID]
|
||||
r.ignoredMut.RUnlock()
|
||||
if ignored {
|
||||
return // found
|
||||
}
|
||||
if !r.store.Exists(reportID) {
|
||||
http.Error(w, "Not found", http.StatusNotFound)
|
||||
}
|
||||
@@ -71,6 +82,20 @@ func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *htt
|
||||
|
||||
// servePut accepts and stores the given report.
|
||||
func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
|
||||
result := "receive_failure"
|
||||
defer func() {
|
||||
metricCrashReportsTotal.WithLabelValues(result).Inc()
|
||||
}()
|
||||
|
||||
r.ignoredMut.RLock()
|
||||
_, ignored := r.ignored[reportID]
|
||||
r.ignoredMut.RUnlock()
|
||||
if ignored {
|
||||
result = "ignored_cached"
|
||||
io.Copy(io.Discard, req.Body)
|
||||
return // found
|
||||
}
|
||||
|
||||
// Read at most maxRequestSize of report data.
|
||||
log.Println("Receiving report", reportID)
|
||||
lr := io.LimitReader(req.Body, maxRequestSize)
|
||||
@@ -81,13 +106,28 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
|
||||
return
|
||||
}
|
||||
|
||||
if r.ignore.match(bs) {
|
||||
r.ignoredMut.Lock()
|
||||
if r.ignored == nil {
|
||||
r.ignored = make(map[string]struct{})
|
||||
}
|
||||
r.ignored[reportID] = struct{}{}
|
||||
r.ignoredMut.Unlock()
|
||||
result = "ignored"
|
||||
return
|
||||
}
|
||||
|
||||
result = "success"
|
||||
|
||||
// Store the report
|
||||
if !r.store.Put(reportID, bs) {
|
||||
log.Println("Failed to store report (queue full):", reportID)
|
||||
result = "queue_failure"
|
||||
}
|
||||
|
||||
// Send the report to Sentry
|
||||
if !r.sentry.Send(reportID, userIDFor(req), bs) {
|
||||
log.Println("Failed to send report to sentry (queue full):", reportID)
|
||||
result = "sentry_failure"
|
||||
}
|
||||
}
|
||||
@@ -9,14 +9,13 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
)
|
||||
|
||||
// userIDFor returns a string we can use as the user ID for the purpose of
|
||||
@@ -4,7 +4,7 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:generate go run ../../../script/genassets.go -o gui.files.go ../gui
|
||||
//go:generate go run ../../../../script/genassets.go -o gui.files.go ../gui
|
||||
|
||||
// Package auto contains auto generated files for web assets.
|
||||
package auto
|
||||
@@ -259,7 +259,7 @@
|
||||
return a.value > b.value ? 1 : -1;
|
||||
}
|
||||
|
||||
$http.get("/endpoint").then(function(response) {
|
||||
$http.get("/endpoint/full").then(function(response) {
|
||||
$scope.relays = response.data.relays;
|
||||
|
||||
angular.forEach($scope.relays, function(relay) {
|
||||
@@ -338,7 +338,7 @@
|
||||
relay.showMarker = function() {
|
||||
relay.marker.openPopup();
|
||||
}
|
||||
|
||||
|
||||
relay.hideMarker = function() {
|
||||
relay.marker.closePopup();
|
||||
}
|
||||
@@ -347,7 +347,7 @@
|
||||
|
||||
function addCircleToMap(relay) {
|
||||
console.log(relay.location.latitude)
|
||||
L.circle([relay.location.latitude, relay.location.longitude],
|
||||
L.circle([relay.location.latitude, relay.location.longitude],
|
||||
{
|
||||
radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
|
||||
color: "FF0000",
|
||||
@@ -21,15 +21,13 @@ import (
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/syncthing/syncthing/lib/httpcache"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
|
||||
"github.com/oschwald/geoip2-golang"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
|
||||
"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto"
|
||||
"github.com/syncthing/syncthing/lib/assets"
|
||||
"github.com/syncthing/syncthing/lib/rand"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/geoip"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/relay/client"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
@@ -51,6 +49,10 @@ type relay struct {
|
||||
StatsRetrieved time.Time `json:"statsRetrieved"`
|
||||
}
|
||||
|
||||
type relayShort struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type stats struct {
|
||||
StartTime time.Time `json:"startTime"`
|
||||
UptimeSeconds int `json:"uptimeSeconds"`
|
||||
@@ -95,16 +97,18 @@ var (
|
||||
testCert tls.Certificate
|
||||
knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
|
||||
listen = ":80"
|
||||
metricsListen = ":8081"
|
||||
dir string
|
||||
evictionTime = time.Hour
|
||||
debug bool
|
||||
permRelaysFile string
|
||||
ipHeader string
|
||||
geoipPath string
|
||||
proto string
|
||||
statsRefresh = time.Minute
|
||||
requestQueueLen = 64
|
||||
requestProcessors = 8
|
||||
geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY")
|
||||
geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
|
||||
|
||||
requests chan request
|
||||
|
||||
@@ -124,40 +128,45 @@ func main() {
|
||||
log.SetFlags(log.Lshortfile)
|
||||
|
||||
flag.StringVar(&listen, "listen", listen, "Listen address")
|
||||
flag.StringVar(&metricsListen, "metrics-listen", metricsListen, "Metrics listen address")
|
||||
flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
|
||||
flag.BoolVar(&debug, "debug", debug, "Enable debug output")
|
||||
flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
|
||||
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
|
||||
flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
|
||||
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
|
||||
flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
|
||||
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
|
||||
flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
|
||||
flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
|
||||
flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
|
||||
flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
requests = make(chan request, requestQueueLen)
|
||||
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir())
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to create GeoIP provider:", err)
|
||||
}
|
||||
go geoip.Serve(context.TODO())
|
||||
|
||||
var listener net.Listener
|
||||
var err error
|
||||
|
||||
if permRelaysFile != "" {
|
||||
permanentRelays = loadRelays(permRelaysFile)
|
||||
permanentRelays = loadRelays(permRelaysFile, geoip)
|
||||
}
|
||||
|
||||
testCert = createTestCertificate()
|
||||
|
||||
for i := 0; i < requestProcessors; i++ {
|
||||
go requestProcessor()
|
||||
go requestProcessor(geoip)
|
||||
}
|
||||
|
||||
// Load relays from cache in the background.
|
||||
// Load them in a serial fashion to make sure any genuine requests
|
||||
// are not dropped.
|
||||
go func() {
|
||||
for _, relay := range loadRelays(knownRelaysFile) {
|
||||
for _, relay := range loadRelays(knownRelaysFile, geoip) {
|
||||
resultChan := make(chan result)
|
||||
requests <- request{relay, resultChan, nil}
|
||||
result := <-resultChan
|
||||
@@ -213,15 +222,40 @@ func main() {
|
||||
log.Fatalln("listen:", err)
|
||||
}
|
||||
|
||||
handler := http.NewServeMux()
|
||||
handler.HandleFunc("/", handleAssets)
|
||||
handler.Handle("/endpoint", httpcache.SinglePath(http.HandlerFunc(handleRequest), 15*time.Second))
|
||||
handler.HandleFunc("/metrics", handleMetrics)
|
||||
if metricsListen != "" {
|
||||
mmux := http.NewServeMux()
|
||||
mmux.HandleFunc("/metrics", handleMetrics)
|
||||
go func() {
|
||||
if err := http.ListenAndServe(metricsListen, mmux); err != nil {
|
||||
log.Fatalln("HTTP serve metrics:", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
getMux := http.NewServeMux()
|
||||
getMux.HandleFunc("/", handleAssets)
|
||||
getMux.HandleFunc("/endpoint", withAPIMetrics(handleEndpointShort))
|
||||
getMux.HandleFunc("/endpoint/full", withAPIMetrics(handleEndpointFull))
|
||||
|
||||
postMux := http.NewServeMux()
|
||||
postMux.HandleFunc("/endpoint", withAPIMetrics(handleRegister))
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodOptions:
|
||||
getMux.ServeHTTP(w, r)
|
||||
case http.MethodPost:
|
||||
postMux.ServeHTTP(w, r)
|
||||
default:
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
})
|
||||
|
||||
srv := http.Server{
|
||||
Handler: handler,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
}
|
||||
srv.SetKeepAlivesEnabled(false)
|
||||
|
||||
err = srv.Serve(listener)
|
||||
if err != nil {
|
||||
@@ -255,39 +289,24 @@ func handleAssets(w http.ResponseWriter, r *http.Request) {
|
||||
assets.Serve(w, r, as)
|
||||
}
|
||||
|
||||
func handleRequest(w http.ResponseWriter, r *http.Request) {
|
||||
timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
|
||||
|
||||
w = NewLoggingResponseWriter(w)
|
||||
defer func() {
|
||||
timer.ObserveDuration()
|
||||
lw := w.(*loggingResponseWriter)
|
||||
apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
|
||||
}()
|
||||
|
||||
if ipHeader != "" {
|
||||
hdr := r.Header.Get(ipHeader)
|
||||
fields := strings.Split(hdr, ",")
|
||||
if len(fields) > 0 {
|
||||
r.RemoteAddr = strings.TrimSpace(fields[len(fields)-1])
|
||||
}
|
||||
}
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
handleGetRequest(w, r)
|
||||
case "POST":
|
||||
handlePostRequest(w, r)
|
||||
default:
|
||||
if debug {
|
||||
log.Println("Unhandled HTTP method", r.Method)
|
||||
}
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
func withAPIMetrics(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
|
||||
w = NewLoggingResponseWriter(w)
|
||||
defer func() {
|
||||
timer.ObserveDuration()
|
||||
lw := w.(*loggingResponseWriter)
|
||||
apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
|
||||
}()
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
|
||||
// handleEndpointFull returns the relay list with full metadata and
|
||||
// statistics. Large, and expensive.
|
||||
func handleEndpointFull(rw http.ResponseWriter, r *http.Request) {
|
||||
rw.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
rw.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
mut.RLock()
|
||||
relays := make([]*relay, len(permanentRelays)+len(knownRelays))
|
||||
@@ -295,17 +314,38 @@ func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
|
||||
copy(relays[n:], knownRelays)
|
||||
mut.RUnlock()
|
||||
|
||||
// Shuffle
|
||||
rand.Shuffle(relays)
|
||||
|
||||
_ = json.NewEncoder(rw).Encode(map[string][]*relay{
|
||||
"relays": relays,
|
||||
})
|
||||
}
|
||||
|
||||
func handlePostRequest(w http.ResponseWriter, r *http.Request) {
|
||||
// handleEndpointShort returns the relay list with only the URL.
|
||||
func handleEndpointShort(rw http.ResponseWriter, r *http.Request) {
|
||||
rw.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
rw.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
mut.RLock()
|
||||
relays := make([]relayShort, 0, len(permanentRelays)+len(knownRelays))
|
||||
for _, r := range append(permanentRelays, knownRelays...) {
|
||||
relays = append(relays, relayShort{URL: slimURL(r.URL)})
|
||||
}
|
||||
mut.RUnlock()
|
||||
|
||||
_ = json.NewEncoder(rw).Encode(map[string][]relayShort{
|
||||
"relays": relays,
|
||||
})
|
||||
}
|
||||
|
||||
func handleRegister(w http.ResponseWriter, r *http.Request) {
|
||||
// Get the IP address of the client
|
||||
rhost := r.RemoteAddr
|
||||
if ipHeader != "" {
|
||||
hdr := r.Header.Get(ipHeader)
|
||||
fields := strings.Split(hdr, ",")
|
||||
if len(fields) > 0 {
|
||||
rhost = strings.TrimSpace(fields[len(fields)-1])
|
||||
}
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(rhost); err == nil {
|
||||
rhost = host
|
||||
}
|
||||
@@ -425,19 +465,19 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func requestProcessor() {
|
||||
func requestProcessor(geoip *geoip.Provider) {
|
||||
for request := range requests {
|
||||
if request.queueTimer != nil {
|
||||
request.queueTimer.ObserveDuration()
|
||||
}
|
||||
|
||||
timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
|
||||
handleRelayTest(request)
|
||||
handleRelayTest(request, geoip)
|
||||
timer.ObserveDuration()
|
||||
}
|
||||
}
|
||||
|
||||
func handleRelayTest(request request) {
|
||||
func handleRelayTest(request request, geoip *geoip.Provider) {
|
||||
if debug {
|
||||
log.Println("Request for", request.relay)
|
||||
}
|
||||
@@ -450,7 +490,7 @@ func handleRelayTest(request request) {
|
||||
}
|
||||
|
||||
stats := fetchStats(request.relay)
|
||||
location := getLocation(request.relay.uri.Host)
|
||||
location := getLocation(request.relay.uri.Host, geoip)
|
||||
|
||||
mut.Lock()
|
||||
if stats != nil {
|
||||
@@ -523,7 +563,7 @@ func evict(relay *relay) func() {
|
||||
}
|
||||
}
|
||||
|
||||
func loadRelays(file string) []*relay {
|
||||
func loadRelays(file string, geoip *geoip.Provider) []*relay {
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
log.Println("Failed to load relays: " + err.Error())
|
||||
@@ -547,7 +587,7 @@ func loadRelays(file string) []*relay {
|
||||
|
||||
relays = append(relays, &relay{
|
||||
URL: line,
|
||||
Location: getLocation(uri.Host),
|
||||
Location: getLocation(uri.Host, geoip),
|
||||
uri: uri,
|
||||
})
|
||||
if debug {
|
||||
@@ -580,21 +620,16 @@ func createTestCertificate() tls.Certificate {
|
||||
return cert
|
||||
}
|
||||
|
||||
func getLocation(host string) location {
|
||||
func getLocation(host string, geoip *geoip.Provider) location {
|
||||
timer := prometheus.NewTimer(locationLookupSeconds)
|
||||
defer timer.ObserveDuration()
|
||||
db, err := geoip2.Open(geoipPath)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
addr, err := net.ResolveTCPAddr("tcp", host)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
|
||||
city, err := db.City(addr.IP)
|
||||
city, err := geoip.City(addr.IP)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
@@ -660,3 +695,16 @@ func (b *errorTracker) IsBlocked(host string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func slimURL(u string) string {
|
||||
p, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return u
|
||||
}
|
||||
newQuery := url.Values{}
|
||||
if id := p.Query().Get("id"); id != "" {
|
||||
newQuery.Set("id", id)
|
||||
}
|
||||
p.RawQuery = newQuery.Encode()
|
||||
return p.String()
|
||||
}
|
||||
@@ -42,7 +42,7 @@ func TestHandleGetRequest(t *testing.T) {
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
w.Body = new(bytes.Buffer)
|
||||
handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
|
||||
handleEndpointFull(w, httptest.NewRequest("GET", "/", nil))
|
||||
|
||||
result := make(map[string][]*relay)
|
||||
err := json.NewDecoder(w.Body).Decode(&result)
|
||||
@@ -92,3 +92,18 @@ func TestCanonicalizeQueryValues(t *testing.T) {
|
||||
t.Errorf("expected %q, got %q", exp, str)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlimURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, out string
|
||||
}{
|
||||
{"http://example.com/", "http://example.com/"},
|
||||
{"relay://192.0.2.42:22067/?globalLimitBps=0&id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M&networkTimeout=2m0s&pingInterval=1m0s&providedBy=Test&sessionLimitBps=0&statusAddr=%3A22070", "relay://192.0.2.42:22067/?id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
if got := slimURL(c.in); got != c.out {
|
||||
t.Errorf("expected %q, got %q", c.out, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,27 +6,12 @@ import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
)
|
||||
|
||||
func init() {
|
||||
processCollectorOpts := collectors.ProcessCollectorOpts{
|
||||
Namespace: "syncthing_relaypoolsrv",
|
||||
PidFn: func() (int, error) {
|
||||
return os.Getpid(), nil
|
||||
},
|
||||
}
|
||||
|
||||
prometheus.MustRegister(
|
||||
collectors.NewProcessCollector(processCollectorOpts),
|
||||
)
|
||||
}
|
||||
|
||||
var (
|
||||
statusClient = http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
@@ -19,15 +19,18 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/httpcache"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
)
|
||||
|
||||
type cli struct {
|
||||
Listen string `default:":8080" help:"Listen address"`
|
||||
URL string `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
|
||||
Forward []string `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
|
||||
CacheTime time.Duration `default:"15m" help:"Cache time"`
|
||||
Listen string `default:":8080" help:"Listen address"`
|
||||
MetricsListen string `default:":8081" help:"Listen address for metrics"`
|
||||
URL string `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
|
||||
Forward []string `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
|
||||
CacheTime time.Duration `default:"15m" help:"Cache time"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -40,17 +43,37 @@ func main() {
|
||||
}
|
||||
|
||||
func server(params *cli) error {
|
||||
http.Handle("/meta.json", httpcache.SinglePath(&githubReleases{url: params.URL}, params.CacheTime))
|
||||
if params.MetricsListen != "" {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
log.Println("Listening for metrics on", params.MetricsListen)
|
||||
if err := http.ListenAndServe(params.MetricsListen, mux); err != nil {
|
||||
log.Fatalf("Failed to start metrics server: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/meta.json", httpcache.SinglePath(&githubReleases{url: params.URL}, params.CacheTime))
|
||||
|
||||
for _, fwd := range params.Forward {
|
||||
path, url, ok := strings.Cut(fwd, "->")
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid forward: %q", fwd)
|
||||
}
|
||||
http.Handle(path, httpcache.SinglePath(&proxy{url: url}, params.CacheTime))
|
||||
log.Println("Forwarding", path, "to", url)
|
||||
mux.Handle(path, httpcache.SinglePath(&proxy{url: url}, params.CacheTime))
|
||||
}
|
||||
|
||||
return http.ListenAndServe(params.Listen, nil)
|
||||
srv := &http.Server{
|
||||
Addr: params.Listen,
|
||||
Handler: mux,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
}
|
||||
srv.SetKeepAlivesEnabled(false)
|
||||
return srv.ListenAndServe()
|
||||
}
|
||||
|
||||
type githubReleases struct {
|
||||
@@ -11,8 +11,9 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/syncthing/syncthing/cmd/ursrv/aggregate"
|
||||
"github.com/syncthing/syncthing/cmd/ursrv/serve"
|
||||
"github.com/syncthing/syncthing/cmd/infra/ursrv/aggregate"
|
||||
"github.com/syncthing/syncthing/cmd/infra/ursrv/serve"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
)
|
||||
|
||||
type CLI struct {
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright (C) 2023 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package serve
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var metricReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "ursrv",
|
||||
Name: "reports_total",
|
||||
}, []string{"version"})
|
||||
|
||||
func init() {
|
||||
metricReportsTotal.WithLabelValues("fail")
|
||||
metricReportsTotal.WithLabelValues("duplicate")
|
||||
metricReportsTotal.WithLabelValues("v1")
|
||||
metricReportsTotal.WithLabelValues("v2")
|
||||
metricReportsTotal.WithLabelValues("v3")
|
||||
}
|
||||
@@ -8,14 +8,17 @@ package serve
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -25,19 +28,21 @@ import (
|
||||
"unicode"
|
||||
|
||||
_ "github.com/lib/pq" // PostgreSQL driver
|
||||
"github.com/oschwald/geoip2-golang"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/geoip"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
"github.com/syncthing/syncthing/lib/ur/contract"
|
||||
)
|
||||
|
||||
type CLI struct {
|
||||
Debug bool `env:"UR_DEBUG"`
|
||||
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
|
||||
Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
|
||||
GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"`
|
||||
Debug bool `env:"UR_DEBUG"`
|
||||
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
|
||||
Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
|
||||
GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"`
|
||||
GeoIPAccountID int `env:"UR_GEOIP_ACCOUNT_ID"`
|
||||
}
|
||||
|
||||
//go:embed static
|
||||
@@ -53,26 +58,30 @@ var (
|
||||
// Maps well known builders to the official distribution method that
|
||||
// they represent
|
||||
|
||||
{regexp.MustCompile(`teamcity@build\.syncthing\.net`), "GitHub"},
|
||||
{regexp.MustCompile(`jenkins@build\.syncthing\.net`), "GitHub"},
|
||||
{regexp.MustCompile(`builder@github\.syncthing\.net`), "GitHub"},
|
||||
{regexp.MustCompile(`\steamcity@build\.syncthing\.net`), "GitHub"},
|
||||
{regexp.MustCompile(`\sjenkins@build\.syncthing\.net`), "GitHub"},
|
||||
{regexp.MustCompile(`\sbuilder@github\.syncthing\.net`), "GitHub"},
|
||||
|
||||
{regexp.MustCompile(`deb@build\.syncthing\.net`), "APT"},
|
||||
{regexp.MustCompile(`debian@github\.syncthing\.net`), "APT"},
|
||||
{regexp.MustCompile(`\sdeb@build\.syncthing\.net`), "APT"},
|
||||
{regexp.MustCompile(`\sdebian@github\.syncthing\.net`), "APT"},
|
||||
|
||||
{regexp.MustCompile(`docker@syncthing\.net`), "Docker Hub"},
|
||||
{regexp.MustCompile(`docker@build.syncthing\.net`), "Docker Hub"},
|
||||
{regexp.MustCompile(`docker@github.syncthing\.net`), "Docker Hub"},
|
||||
{regexp.MustCompile(`\sdocker@syncthing\.net`), "Docker Hub"},
|
||||
{regexp.MustCompile(`\sdocker@build.syncthing\.net`), "Docker Hub"},
|
||||
{regexp.MustCompile(`\sdocker@github.syncthing\.net`), "Docker Hub"},
|
||||
|
||||
{regexp.MustCompile(`android-builder@github\.syncthing\.net`), "Google Play"},
|
||||
{regexp.MustCompile(`android-.*teamcity@build\.syncthing\.net`), "Google Play"},
|
||||
{regexp.MustCompile(`android-.*vagrant@basebox-stretch64`), "F-Droid"},
|
||||
{regexp.MustCompile(`vagrant@bullseye`), "F-Droid"},
|
||||
{regexp.MustCompile(`builduser@(archlinux|svetlemodry)`), "Arch (3rd party)"},
|
||||
{regexp.MustCompile(`\sandroid-builder@github\.syncthing\.net`), "Google Play"},
|
||||
{regexp.MustCompile(`\sandroid-.*teamcity@build\.syncthing\.net`), "Google Play"},
|
||||
{regexp.MustCompile(`\sandroid-.*vagrant@basebox-stretch64`), "F-Droid"},
|
||||
{regexp.MustCompile(`\svagrant@bookworm`), "F-Droid"},
|
||||
{regexp.MustCompile(`\svagrant@bullseye`), "F-Droid"},
|
||||
{regexp.MustCompile(`\sbuilduser@(archlinux|svetlemodry)`), "Arch (3rd party)"},
|
||||
{regexp.MustCompile(`\ssyncthing@archlinux`), "Arch (3rd party)"},
|
||||
{regexp.MustCompile(`@debian`), "Debian (3rd party)"},
|
||||
{regexp.MustCompile(`@fedora`), "Fedora (3rd party)"},
|
||||
{regexp.MustCompile(`\bbrew@`), "Homebrew (3rd party)"},
|
||||
{regexp.MustCompile(`root@buildkitsandbox`), "LinuxServer.io (3rd party)"},
|
||||
{regexp.MustCompile(`\sbrew@`), "Homebrew (3rd party)"},
|
||||
{regexp.MustCompile(`\sroot@buildkitsandbox`), "LinuxServer.io (3rd party)"},
|
||||
{regexp.MustCompile(`\sports@freebsd`), "FreeBSD (3rd party)"},
|
||||
{regexp.MustCompile(`\snix@nix`), "Nix (3rd party)"},
|
||||
{regexp.MustCompile(`.`), "Others"},
|
||||
}
|
||||
)
|
||||
@@ -185,10 +194,16 @@ func (cli *CLI) Run() error {
|
||||
log.Fatalln("listen:", err)
|
||||
}
|
||||
|
||||
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir())
|
||||
if err != nil {
|
||||
log.Fatalln("geoip:", err)
|
||||
}
|
||||
go geoip.Serve(context.TODO())
|
||||
|
||||
srv := &server{
|
||||
db: db,
|
||||
debug: cli.Debug,
|
||||
geoIPPath: cli.GeoIPPath,
|
||||
db: db,
|
||||
debug: cli.Debug,
|
||||
geoip: geoip,
|
||||
}
|
||||
http.HandleFunc("/", srv.rootHandler)
|
||||
http.HandleFunc("/newdata", srv.newDataHandler)
|
||||
@@ -196,6 +211,7 @@ func (cli *CLI) Run() error {
|
||||
http.HandleFunc("/performance.json", srv.performanceHandler)
|
||||
http.HandleFunc("/blockstats.json", srv.blockStatsHandler)
|
||||
http.HandleFunc("/locations.json", srv.locationsHandler)
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
http.Handle("/static/", http.FileServer(http.FS(statics)))
|
||||
|
||||
go srv.cacheRefresher()
|
||||
@@ -208,9 +224,9 @@ func (cli *CLI) Run() error {
|
||||
}
|
||||
|
||||
type server struct {
|
||||
debug bool
|
||||
db *sql.DB
|
||||
geoIPPath string
|
||||
debug bool
|
||||
db *sql.DB
|
||||
geoip *geoip.Provider
|
||||
|
||||
cacheMut sync.Mutex
|
||||
cachedIndex []byte
|
||||
@@ -233,7 +249,7 @@ func (s *server) cacheRefresher() {
|
||||
}
|
||||
|
||||
func (s *server) refreshCacheLocked() error {
|
||||
rep := getReport(s.db, s.geoIPPath)
|
||||
rep := getReport(s.db, s.geoip)
|
||||
buf := new(bytes.Buffer)
|
||||
err := tpl.Execute(buf, rep)
|
||||
if err != nil {
|
||||
@@ -289,6 +305,12 @@ func (s *server) locationsHandler(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
|
||||
func (s *server) newDataHandler(w http.ResponseWriter, r *http.Request) {
|
||||
version := "fail"
|
||||
defer func() {
|
||||
// Version is "fail", "duplicate", "v2", "v3", ...
|
||||
metricReportsTotal.WithLabelValues(version).Inc()
|
||||
}()
|
||||
|
||||
defer r.Body.Close()
|
||||
|
||||
addr := r.Header.Get("X-Forwarded-For")
|
||||
@@ -334,6 +356,7 @@ func (s *server) newDataHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if err.Error() == `pq: duplicate key value violates unique constraint "uniqueidjsonindex"` {
|
||||
// We already have a report today for the same unique ID; drop
|
||||
// this one without complaining.
|
||||
version = "duplicate"
|
||||
return
|
||||
}
|
||||
log.Println("insert:", err)
|
||||
@@ -343,6 +366,8 @@ func (s *server) newDataHandler(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "Database Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
version = fmt.Sprintf("v%d", rep.URVersion)
|
||||
}
|
||||
|
||||
func (s *server) summaryHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -478,15 +503,7 @@ type weightedLocation struct {
|
||||
Weight int `json:"weight"`
|
||||
}
|
||||
|
||||
func getReport(db *sql.DB, geoIPPath string) map[string]interface{} {
|
||||
geoip, err := geoip2.Open(geoIPPath)
|
||||
if err != nil {
|
||||
log.Println("opening geoip db", err)
|
||||
geoip = nil
|
||||
} else {
|
||||
defer geoip.Close()
|
||||
}
|
||||
|
||||
func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} {
|
||||
nodes := 0
|
||||
countriesTotal := 0
|
||||
var versions []string
|
||||
@@ -722,6 +739,10 @@ func getReport(db *sql.DB, geoIPPath string) map[string]interface{} {
|
||||
add(featureGroups["Folder"]["v3"], "Pull Order", prettyCase(key), value)
|
||||
}
|
||||
|
||||
for key, value := range rep.FolderUsesV3.CopyRangeMethod {
|
||||
add(featureGroups["Folder"]["v3"], "Copy Range Method", prettyCase(key), value)
|
||||
}
|
||||
|
||||
inc(features["Device"]["v3"], "Untrusted", rep.DeviceUsesV3.Untrusted)
|
||||
|
||||
totals["GUI"] += rep.GUIStats.Enabled
|
||||
|
Before Width: | Height: | Size: 4.8 KiB After Width: | Height: | Size: 4.8 KiB |
Vendored
Executable → Regular
Vendored
Executable → Regular
Executable → Regular
Executable → Regular
|
Before Width: | Height: | Size: 61 KiB After Width: | Height: | Size: 61 KiB |
Executable → Regular
Executable → Regular
@@ -270,7 +270,7 @@ found in the LICENSE file.
|
||||
</h4>
|
||||
</div>
|
||||
<div id="collapseTwo" class="panel-collapse collapse">
|
||||
<div class="panel-body">
|
||||
<div class="panel-body less-padding">
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<table class="table table-striped">
|
||||
@@ -611,6 +611,7 @@ found in the LICENSE file.
|
||||
</div>
|
||||
<hr>
|
||||
<p>
|
||||
<a href="https://github.com/syncthing/syncthing/">Source code</a>.
|
||||
This product includes GeoLite2 data created by MaxMind, available from
|
||||
<a href="http://www.maxmind.com">http://www.maxmind.com</a>.
|
||||
</p>
|
||||
@@ -0,0 +1,257 @@
|
||||
// Copyright (C) 2024 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/thejerf/suture/v4"
|
||||
)
|
||||
|
||||
type amqpReplicator struct {
|
||||
suture.Service
|
||||
broker string
|
||||
sender *amqpSender
|
||||
receiver *amqpReceiver
|
||||
outbox chan ReplicationRecord
|
||||
}
|
||||
|
||||
func newAMQPReplicator(broker, clientID string, db database) *amqpReplicator {
|
||||
svc := suture.New("amqpReplicator", suture.Spec{PassThroughPanics: true})
|
||||
|
||||
sender := &amqpSender{
|
||||
broker: broker,
|
||||
clientID: clientID,
|
||||
outbox: make(chan ReplicationRecord, replicationOutboxSize),
|
||||
}
|
||||
svc.Add(sender)
|
||||
|
||||
receiver := &amqpReceiver{
|
||||
broker: broker,
|
||||
clientID: clientID,
|
||||
db: db,
|
||||
}
|
||||
svc.Add(receiver)
|
||||
|
||||
return &amqpReplicator{
|
||||
Service: svc,
|
||||
broker: broker,
|
||||
sender: sender,
|
||||
receiver: receiver,
|
||||
outbox: make(chan ReplicationRecord, replicationOutboxSize),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *amqpReplicator) send(key *protocol.DeviceID, ps []DatabaseAddress, seen int64) {
|
||||
s.sender.send(key, ps, seen)
|
||||
}
|
||||
|
||||
type amqpSender struct {
|
||||
broker string
|
||||
clientID string
|
||||
outbox chan ReplicationRecord
|
||||
}
|
||||
|
||||
func (s *amqpSender) Serve(ctx context.Context) error {
|
||||
conn, ch, err := amqpChannel(s.broker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ch.Close()
|
||||
defer conn.Close()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
select {
|
||||
case rec := <-s.outbox:
|
||||
size := rec.Size()
|
||||
if len(buf) < size {
|
||||
buf = make([]byte, size)
|
||||
}
|
||||
|
||||
n, err := rec.MarshalTo(buf)
|
||||
if err != nil {
|
||||
replicationSendsTotal.WithLabelValues("error").Inc()
|
||||
return fmt.Errorf("replication marshal: %w", err)
|
||||
}
|
||||
|
||||
err = ch.PublishWithContext(ctx,
|
||||
"discovery", // exchange
|
||||
"", // routing key
|
||||
false, // mandatory
|
||||
false, // immediate
|
||||
amqp.Publishing{
|
||||
ContentType: "application/protobuf",
|
||||
Body: buf[:n],
|
||||
AppId: s.clientID,
|
||||
})
|
||||
if err != nil {
|
||||
replicationSendsTotal.WithLabelValues("error").Inc()
|
||||
return fmt.Errorf("replication publish: %w", err)
|
||||
}
|
||||
|
||||
replicationSendsTotal.WithLabelValues("success").Inc()
|
||||
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *amqpSender) String() string {
|
||||
return fmt.Sprintf("amqpSender(%q)", s.broker)
|
||||
}
|
||||
|
||||
func (s *amqpSender) send(key *protocol.DeviceID, ps []DatabaseAddress, seen int64) {
|
||||
item := ReplicationRecord{
|
||||
Key: key[:],
|
||||
Addresses: ps,
|
||||
Seen: seen,
|
||||
}
|
||||
|
||||
// The send should never block. The inbox is suitably buffered for at
|
||||
// least a few seconds of stalls, which shouldn't happen in practice.
|
||||
select {
|
||||
case s.outbox <- item:
|
||||
default:
|
||||
replicationSendsTotal.WithLabelValues("drop").Inc()
|
||||
}
|
||||
}
|
||||
|
||||
type amqpReceiver struct {
|
||||
broker string
|
||||
clientID string
|
||||
db database
|
||||
}
|
||||
|
||||
func (s *amqpReceiver) Serve(ctx context.Context) error {
|
||||
conn, ch, err := amqpChannel(s.broker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ch.Close()
|
||||
defer conn.Close()
|
||||
|
||||
msgs, err := amqpConsume(ch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case msg, ok := <-msgs:
|
||||
if !ok {
|
||||
return fmt.Errorf("subscription closed: %w", io.EOF)
|
||||
}
|
||||
|
||||
// ignore messages from ourself
|
||||
if msg.AppId == s.clientID {
|
||||
continue
|
||||
}
|
||||
|
||||
var rec ReplicationRecord
|
||||
if err := rec.Unmarshal(msg.Body); err != nil {
|
||||
replicationRecvsTotal.WithLabelValues("error").Inc()
|
||||
return fmt.Errorf("replication unmarshal: %w", err)
|
||||
}
|
||||
id, err := protocol.DeviceIDFromBytes(rec.Key)
|
||||
if err != nil {
|
||||
id, err = protocol.DeviceIDFromString(string(rec.Key))
|
||||
}
|
||||
if err != nil {
|
||||
log.Println("Replication device ID:", err)
|
||||
replicationRecvsTotal.WithLabelValues("error").Inc()
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.db.merge(&id, rec.Addresses, rec.Seen); err != nil {
|
||||
return fmt.Errorf("replication database merge: %w", err)
|
||||
}
|
||||
|
||||
replicationRecvsTotal.WithLabelValues("success").Inc()
|
||||
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *amqpReceiver) String() string {
|
||||
return fmt.Sprintf("amqpReceiver(%q)", s.broker)
|
||||
}
|
||||
|
||||
func amqpChannel(dst string) (*amqp.Connection, *amqp.Channel, error) {
|
||||
conn, err := amqp.Dial(dst)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("AMQP dial: %w", err)
|
||||
}
|
||||
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("AMQP channel: %w", err)
|
||||
}
|
||||
|
||||
err = ch.ExchangeDeclare(
|
||||
"discovery", // name
|
||||
"fanout", // type
|
||||
false, // durable
|
||||
false, // auto-deleted
|
||||
false, // internal
|
||||
false, // no-wait
|
||||
nil, // arguments
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("AMQP declare exchange: %w", err)
|
||||
}
|
||||
|
||||
return conn, ch, nil
|
||||
}
|
||||
|
||||
func amqpConsume(ch *amqp.Channel) (<-chan amqp.Delivery, error) {
|
||||
q, err := ch.QueueDeclare(
|
||||
"", // name
|
||||
false, // durable
|
||||
false, // delete when unused
|
||||
true, // exclusive
|
||||
false, // no-wait
|
||||
nil, // arguments
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("AMQP declare queue: %w", err)
|
||||
}
|
||||
|
||||
err = ch.QueueBind(
|
||||
q.Name, // queue name
|
||||
"", // routing key
|
||||
"discovery", // exchange
|
||||
false,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("AMQP bind queue: %w", err)
|
||||
}
|
||||
|
||||
msgs, err := ch.Consume(
|
||||
q.Name, // queue
|
||||
"", // consumer
|
||||
true, // auto-ack
|
||||
false, // exclusive
|
||||
false, // no-local
|
||||
false, // no-wait
|
||||
nil, // args
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("AMQP consume: %w", err)
|
||||
}
|
||||
|
||||
return msgs, nil
|
||||
}
|
||||
+148
-72
@@ -22,7 +22,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -39,15 +39,20 @@ type announcement struct {
|
||||
}
|
||||
|
||||
type apiSrv struct {
|
||||
addr string
|
||||
cert tls.Certificate
|
||||
db database
|
||||
listener net.Listener
|
||||
repl replicator // optional
|
||||
useHTTP bool
|
||||
addr string
|
||||
cert tls.Certificate
|
||||
db database
|
||||
listener net.Listener
|
||||
repl replicator // optional
|
||||
useHTTP bool
|
||||
compression bool
|
||||
gzipWriters sync.Pool
|
||||
seenTracker *retryAfterTracker
|
||||
notSeenTracker *retryAfterTracker
|
||||
}
|
||||
|
||||
mapsMut sync.Mutex
|
||||
misses map[string]int32
|
||||
type replicator interface {
|
||||
send(key *protocol.DeviceID, addrs []DatabaseAddress, seen int64)
|
||||
}
|
||||
|
||||
type requestID int64
|
||||
@@ -60,18 +65,30 @@ type contextKey int
|
||||
|
||||
const idKey contextKey = iota
|
||||
|
||||
func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
|
||||
func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP, compression bool) *apiSrv {
|
||||
return &apiSrv{
|
||||
addr: addr,
|
||||
cert: cert,
|
||||
db: db,
|
||||
repl: repl,
|
||||
useHTTP: useHTTP,
|
||||
misses: make(map[string]int32),
|
||||
addr: addr,
|
||||
cert: cert,
|
||||
db: db,
|
||||
repl: repl,
|
||||
useHTTP: useHTTP,
|
||||
compression: compression,
|
||||
seenTracker: &retryAfterTracker{
|
||||
name: "seenTracker",
|
||||
bucketStarts: time.Now(),
|
||||
desiredRate: 250,
|
||||
currentDelay: notFoundRetryUnknownMinSeconds,
|
||||
},
|
||||
notSeenTracker: &retryAfterTracker{
|
||||
name: "notSeenTracker",
|
||||
bucketStarts: time.Now(),
|
||||
desiredRate: 250,
|
||||
currentDelay: notFoundRetryUnknownMaxSeconds / 2,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *apiSrv) Serve(_ context.Context) error {
|
||||
func (s *apiSrv) Serve(ctx context.Context) error {
|
||||
if s.useHTTP {
|
||||
listener, err := net.Listen("tcp", s.addr)
|
||||
if err != nil {
|
||||
@@ -105,6 +122,11 @@ func (s *apiSrv) Serve(_ context.Context) error {
|
||||
ErrorLog: log.New(io.Discard, "", 0),
|
||||
}
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
srv.Shutdown(context.Background())
|
||||
}()
|
||||
|
||||
err := srv.Serve(s.listener)
|
||||
if err != nil {
|
||||
log.Println("Serve:", err)
|
||||
@@ -136,7 +158,12 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if s.useHTTP {
|
||||
remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
|
||||
// X-Forwarded-For can have multiple client IPs; split using the comma separator
|
||||
forwardIP, _, _ := strings.Cut(req.Header.Get("X-Forwarded-For"), ",")
|
||||
|
||||
// net.ParseIP will return nil if leading/trailing whitespace exists; use strings.TrimSpace()
|
||||
remoteAddr.IP = net.ParseIP(strings.TrimSpace(forwardIP))
|
||||
|
||||
if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
|
||||
remoteAddr.Port = int(parsedPort)
|
||||
}
|
||||
@@ -176,8 +203,7 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
key := deviceID.String()
|
||||
rec, err := s.db.get(key)
|
||||
rec, err := s.db.get(&deviceID)
|
||||
if err != nil {
|
||||
// some sort of internal error
|
||||
lookupRequestsTotal.WithLabelValues("internal_error").Inc()
|
||||
@@ -187,27 +213,15 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if len(rec.Addresses) == 0 {
|
||||
lookupRequestsTotal.WithLabelValues("not_found").Inc()
|
||||
|
||||
s.mapsMut.Lock()
|
||||
misses := s.misses[key]
|
||||
if misses < rec.Misses {
|
||||
misses = rec.Misses + 1
|
||||
var afterS int
|
||||
if rec.Seen == 0 {
|
||||
afterS = s.notSeenTracker.retryAfterS()
|
||||
lookupRequestsTotal.WithLabelValues("not_found_ever").Inc()
|
||||
} else {
|
||||
misses++
|
||||
afterS = s.seenTracker.retryAfterS()
|
||||
lookupRequestsTotal.WithLabelValues("not_found_recent").Inc()
|
||||
}
|
||||
s.misses[key] = misses
|
||||
s.mapsMut.Unlock()
|
||||
|
||||
if misses%notFoundMissesWriteInterval == 0 {
|
||||
rec.Misses = misses
|
||||
rec.Missed = time.Now().UnixNano()
|
||||
rec.Addresses = nil
|
||||
// rec.Seen retained from get
|
||||
s.db.put(key, rec)
|
||||
}
|
||||
|
||||
w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
|
||||
w.Header().Set("Retry-After", strconv.Itoa(afterS))
|
||||
http.Error(w, "Not Found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
@@ -218,10 +232,16 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
|
||||
var bw io.Writer = w
|
||||
|
||||
// Use compression if the client asks for it
|
||||
if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
|
||||
if s.compression && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
|
||||
gw, ok := s.gzipWriters.Get().(*gzip.Writer)
|
||||
if ok {
|
||||
gw.Reset(w)
|
||||
} else {
|
||||
gw = gzip.NewWriter(w)
|
||||
}
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
gw := gzip.NewWriter(bw)
|
||||
defer gw.Close()
|
||||
defer s.gzipWriters.Put(gw)
|
||||
bw = gw
|
||||
}
|
||||
|
||||
@@ -284,25 +304,25 @@ func (s *apiSrv) Stop() {
|
||||
}
|
||||
|
||||
func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
|
||||
key := deviceID.String()
|
||||
now := time.Now()
|
||||
expire := now.Add(addressExpiryTime).UnixNano()
|
||||
|
||||
// The address slice must always be sorted for database merges to work
|
||||
// properly.
|
||||
slices.Sort(addresses)
|
||||
addresses = slices.Compact(addresses)
|
||||
|
||||
dbAddrs := make([]DatabaseAddress, len(addresses))
|
||||
for i := range addresses {
|
||||
dbAddrs[i].Address = addresses[i]
|
||||
dbAddrs[i].Expires = expire
|
||||
}
|
||||
|
||||
// The address slice must always be sorted for database merges to work
|
||||
// properly.
|
||||
sort.Sort(databaseAddressOrder(dbAddrs))
|
||||
|
||||
seen := now.UnixNano()
|
||||
if s.repl != nil {
|
||||
s.repl.send(key, dbAddrs, seen)
|
||||
s.repl.send(&deviceID, dbAddrs, seen)
|
||||
}
|
||||
return s.db.merge(key, dbAddrs, seen)
|
||||
return s.db.merge(&deviceID, dbAddrs, seen)
|
||||
}
|
||||
|
||||
func handlePing(w http.ResponseWriter, _ *http.Request) {
|
||||
@@ -352,7 +372,7 @@ func certificateBytes(req *http.Request) ([]byte, error) {
|
||||
}
|
||||
|
||||
bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
|
||||
} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
|
||||
} else if cert := req.Header.Get("X-Forwarded-Tls-Client-Cert"); cert != "" {
|
||||
// Traefik 2 passtlsclientcert
|
||||
//
|
||||
// The certificate is in PEM format, maybe with URL encoding
|
||||
@@ -360,19 +380,36 @@ func certificateBytes(req *http.Request) ([]byte, error) {
|
||||
// statements. We need to decode, reinstate the newlines every 64
|
||||
// character and add statements for the PEM decoder
|
||||
|
||||
if strings.Contains(hdr, "%") {
|
||||
if unesc, err := url.QueryUnescape(hdr); err == nil {
|
||||
hdr = unesc
|
||||
if strings.Contains(cert, "%") {
|
||||
if unesc, err := url.QueryUnescape(cert); err == nil {
|
||||
cert = unesc
|
||||
}
|
||||
}
|
||||
|
||||
for i := 64; i < len(hdr); i += 65 {
|
||||
hdr = hdr[:i] + "\n" + hdr[i:]
|
||||
const (
|
||||
header = "-----BEGIN CERTIFICATE-----"
|
||||
footer = "-----END CERTIFICATE-----"
|
||||
)
|
||||
|
||||
var b bytes.Buffer
|
||||
b.Grow(len(header) + 1 + len(cert) + len(cert)/64 + 1 + len(footer) + 1)
|
||||
|
||||
b.WriteString(header)
|
||||
b.WriteByte('\n')
|
||||
|
||||
for i := 0; i < len(cert); i += 64 {
|
||||
end := i + 64
|
||||
if end > len(cert) {
|
||||
end = len(cert)
|
||||
}
|
||||
b.WriteString(cert[i:end])
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
|
||||
hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
|
||||
hdr += "\n-----END CERTIFICATE-----\n"
|
||||
bs = []byte(hdr)
|
||||
b.WriteString(footer)
|
||||
b.WriteByte('\n')
|
||||
|
||||
bs = b.Bytes()
|
||||
}
|
||||
|
||||
if bs == nil {
|
||||
@@ -410,13 +447,13 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
|
||||
continue
|
||||
}
|
||||
|
||||
if remote != nil {
|
||||
if host == "" || ip.IsUnspecified() {
|
||||
if host == "" || ip.IsUnspecified() {
|
||||
if remote != nil {
|
||||
// Replace the unspecified IP with the request source.
|
||||
|
||||
// ... unless the request source is the loopback address or
|
||||
// multicast/unspecified (can't happen, really).
|
||||
if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
|
||||
if remote.IP == nil || remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -432,11 +469,21 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
|
||||
}
|
||||
|
||||
host = remote.IP.String()
|
||||
}
|
||||
|
||||
// If zero port was specified, use remote port.
|
||||
if port == "0" && remote.Port > 0 {
|
||||
} else {
|
||||
// remote is nil, unable to determine host IP
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// If zero port was specified, use remote port.
|
||||
if port == "0" {
|
||||
if remote != nil && remote.Port > 0 {
|
||||
// use remote port
|
||||
port = strconv.Itoa(remote.Port)
|
||||
} else {
|
||||
// unable to determine remote port
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -476,15 +523,44 @@ func errorRetryAfterString() string {
|
||||
return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
|
||||
}
|
||||
|
||||
func notFoundRetryAfterString(misses int) string {
|
||||
retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
|
||||
if retryAfterS > notFoundRetryMaxSeconds {
|
||||
retryAfterS = notFoundRetryMaxSeconds
|
||||
}
|
||||
retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
|
||||
return strconv.Itoa(retryAfterS)
|
||||
}
|
||||
|
||||
func reannounceAfterString() string {
|
||||
return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
|
||||
}
|
||||
|
||||
type retryAfterTracker struct {
|
||||
name string
|
||||
desiredRate float64 // requests per second
|
||||
|
||||
mut sync.Mutex
|
||||
lastCount int // requests in the last bucket
|
||||
curCount int // requests in the current bucket
|
||||
bucketStarts time.Time // start of the current bucket
|
||||
currentDelay int // current delay in seconds
|
||||
}
|
||||
|
||||
func (t *retryAfterTracker) retryAfterS() int {
|
||||
now := time.Now()
|
||||
t.mut.Lock()
|
||||
if durS := now.Sub(t.bucketStarts).Seconds(); durS > float64(t.currentDelay) {
|
||||
t.bucketStarts = now
|
||||
t.lastCount = t.curCount
|
||||
lastRate := float64(t.lastCount) / durS
|
||||
|
||||
switch {
|
||||
case t.currentDelay > notFoundRetryUnknownMinSeconds &&
|
||||
lastRate < 0.75*t.desiredRate:
|
||||
t.currentDelay = max(8*t.currentDelay/10, notFoundRetryUnknownMinSeconds)
|
||||
case t.currentDelay < notFoundRetryUnknownMaxSeconds &&
|
||||
lastRate > 1.25*t.desiredRate:
|
||||
t.currentDelay = min(3*t.currentDelay/2, notFoundRetryUnknownMaxSeconds)
|
||||
}
|
||||
|
||||
t.curCount = 0
|
||||
}
|
||||
if t.curCount == 0 {
|
||||
retryAfterLevel.WithLabelValues(t.name).Set(float64(t.currentDelay))
|
||||
}
|
||||
t.curCount++
|
||||
t.mut.Unlock()
|
||||
return t.currentDelay + rand.Intn(t.currentDelay/4)
|
||||
}
|
||||
|
||||
@@ -7,9 +7,20 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
)
|
||||
|
||||
func TestFixupAddresses(t *testing.T) {
|
||||
@@ -69,6 +80,14 @@ func TestFixupAddresses(t *testing.T) {
|
||||
remote: addr("123.123.123.123", 9000),
|
||||
in: []string{"tcp://44.44.44.44:0"},
|
||||
out: []string{"tcp://44.44.44.44:9000"},
|
||||
}, { // remote ip nil
|
||||
remote: addr("", 9000),
|
||||
in: []string{"tcp://:22000", "tcp://44.44.44.44:9000"},
|
||||
out: []string{"tcp://44.44.44.44:9000"},
|
||||
}, { // remote port 0
|
||||
remote: addr("123.123.123.123", 0),
|
||||
in: []string{"tcp://:22000", "tcp://44.44.44.44"},
|
||||
out: []string{"tcp://123.123.123.123:22000"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -86,3 +105,79 @@ func addr(host string, port int) *net.TCPAddr {
|
||||
Port: port,
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAPIRequests(b *testing.B) {
|
||||
db := newInMemoryStore(b.TempDir(), 0, nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go db.Serve(ctx)
|
||||
api := newAPISrv("127.0.0.1:0", tls.Certificate{}, db, nil, true, true)
|
||||
srv := httptest.NewServer(http.HandlerFunc(api.handler))
|
||||
|
||||
kf := b.TempDir() + "/cert"
|
||||
crt, err := tlsutil.NewCertificate(kf+".crt", kf+".key", "localhost", 7)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
certBs, err := os.ReadFile(kf + ".crt")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
certBs = regexp.MustCompile(`---[^\n]+---\n`).ReplaceAll(certBs, nil)
|
||||
certString := string(strings.ReplaceAll(string(certBs), "\n", " "))
|
||||
|
||||
devID := protocol.NewDeviceID(crt.Certificate[0])
|
||||
devIDString := devID.String()
|
||||
|
||||
b.Run("Announce", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
url := srv.URL + "/v2/?device=" + devIDString
|
||||
for i := 0; i < b.N; i++ {
|
||||
req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(`{"addresses":["tcp://10.10.10.10:42000"]}`))
|
||||
req.Header.Set("X-Forwarded-Tls-Client-Cert", certString)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
b.Fatalf("unexpected status %s", resp.Status)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("Lookup", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
url := srv.URL + "/v2/?device=" + devIDString
|
||||
for i := 0; i < b.N; i++ {
|
||||
req, _ := http.NewRequest(http.MethodGet, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b.Fatalf("unexpected status %s", resp.Status)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("LookupNoCompression", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
url := srv.URL + "/v2/?device=" + devIDString
|
||||
for i := 0; i < b.N; i++ {
|
||||
req, _ := http.NewRequest(http.MethodGet, url, nil)
|
||||
req.Header.Set("Accept-Encoding", "identity") // disable compression
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b.Fatalf("unexpected status %s", resp.Status)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+313
-269
@@ -10,15 +10,22 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"sort"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/sliceutil"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
type clock interface {
|
||||
@@ -32,352 +39,389 @@ func (defaultClock) Now() time.Time {
|
||||
}
|
||||
|
||||
type database interface {
|
||||
put(key string, rec DatabaseRecord) error
|
||||
merge(key string, addrs []DatabaseAddress, seen int64) error
|
||||
get(key string) (DatabaseRecord, error)
|
||||
put(key *protocol.DeviceID, rec DatabaseRecord) error
|
||||
merge(key *protocol.DeviceID, addrs []DatabaseAddress, seen int64) error
|
||||
get(key *protocol.DeviceID) (DatabaseRecord, error)
|
||||
}
|
||||
|
||||
type levelDBStore struct {
|
||||
db *leveldb.DB
|
||||
inbox chan func()
|
||||
clock clock
|
||||
marshalBuf []byte
|
||||
type inMemoryStore struct {
|
||||
m *xsync.MapOf[protocol.DeviceID, DatabaseRecord]
|
||||
dir string
|
||||
flushInterval time.Duration
|
||||
s3 *s3Copier
|
||||
clock clock
|
||||
}
|
||||
|
||||
func newLevelDBStore(dir string) (*levelDBStore, error) {
|
||||
db, err := leveldb.OpenFile(dir, levelDBOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func newInMemoryStore(dir string, flushInterval time.Duration, s3 *s3Copier) *inMemoryStore {
|
||||
s := &inMemoryStore{
|
||||
m: xsync.NewMapOf[protocol.DeviceID, DatabaseRecord](),
|
||||
dir: dir,
|
||||
flushInterval: flushInterval,
|
||||
s3: s3,
|
||||
clock: defaultClock{},
|
||||
}
|
||||
return &levelDBStore{
|
||||
db: db,
|
||||
inbox: make(chan func(), 16),
|
||||
clock: defaultClock{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newMemoryLevelDBStore() (*levelDBStore, error) {
|
||||
db, err := leveldb.Open(storage.NewMemStorage(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &levelDBStore{
|
||||
db: db,
|
||||
inbox: make(chan func(), 16),
|
||||
clock: defaultClock{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
|
||||
t0 := time.Now()
|
||||
defer func() {
|
||||
databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
|
||||
}()
|
||||
|
||||
rc := make(chan error)
|
||||
|
||||
s.inbox <- func() {
|
||||
size := rec.Size()
|
||||
if len(s.marshalBuf) < size {
|
||||
s.marshalBuf = make([]byte, size)
|
||||
nr, err := s.read()
|
||||
if os.IsNotExist(err) && s3 != nil {
|
||||
// Try to read from AWS
|
||||
fd, cerr := os.Create(path.Join(s.dir, "records.db"))
|
||||
if cerr != nil {
|
||||
log.Println("Error creating database file:", err)
|
||||
return s
|
||||
}
|
||||
n, _ := rec.MarshalTo(s.marshalBuf)
|
||||
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
|
||||
if err := s3.downloadLatest(fd); err != nil {
|
||||
log.Printf("Error reading database from S3: %v", err)
|
||||
}
|
||||
_ = fd.Close()
|
||||
nr, err = s.read()
|
||||
}
|
||||
|
||||
err := <-rc
|
||||
if err != nil {
|
||||
databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
|
||||
} else {
|
||||
databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
|
||||
log.Println("Error reading database:", err)
|
||||
}
|
||||
|
||||
return err
|
||||
log.Printf("Read %d records from database", nr)
|
||||
s.expireAndCalculateStatistics()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
|
||||
func (s *inMemoryStore) put(key *protocol.DeviceID, rec DatabaseRecord) error {
|
||||
t0 := time.Now()
|
||||
s.m.Store(*key, rec)
|
||||
databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
|
||||
databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *inMemoryStore) merge(key *protocol.DeviceID, addrs []DatabaseAddress, seen int64) error {
|
||||
t0 := time.Now()
|
||||
defer func() {
|
||||
databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
|
||||
}()
|
||||
|
||||
rc := make(chan error)
|
||||
newRec := DatabaseRecord{
|
||||
Addresses: addrs,
|
||||
Seen: seen,
|
||||
}
|
||||
|
||||
s.inbox <- func() {
|
||||
// grab the existing record
|
||||
oldRec, err := s.get(key)
|
||||
if err != nil {
|
||||
// "not found" is not an error from get, so this is serious
|
||||
// stuff only
|
||||
rc <- err
|
||||
return
|
||||
}
|
||||
newRec = merge(newRec, oldRec)
|
||||
oldRec, _ := s.m.Load(*key)
|
||||
newRec = merge(oldRec, newRec)
|
||||
s.m.Store(*key, newRec)
|
||||
|
||||
// We replicate s.put() functionality here ourselves instead of
|
||||
// calling it because we want to serialize our get above together
|
||||
// with the put in the same function.
|
||||
size := newRec.Size()
|
||||
if len(s.marshalBuf) < size {
|
||||
s.marshalBuf = make([]byte, size)
|
||||
}
|
||||
n, _ := newRec.MarshalTo(s.marshalBuf)
|
||||
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
|
||||
}
|
||||
databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
|
||||
databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
|
||||
|
||||
err := <-rc
|
||||
if err != nil {
|
||||
databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
|
||||
} else {
|
||||
databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
|
||||
}
|
||||
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
|
||||
func (s *inMemoryStore) get(key *protocol.DeviceID) (DatabaseRecord, error) {
|
||||
t0 := time.Now()
|
||||
defer func() {
|
||||
databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
|
||||
}()
|
||||
|
||||
keyBs := []byte(key)
|
||||
val, err := s.db.Get(keyBs, nil)
|
||||
if err == leveldb.ErrNotFound {
|
||||
rec, ok := s.m.Load(*key)
|
||||
if !ok {
|
||||
databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
|
||||
return DatabaseRecord{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
|
||||
return DatabaseRecord{}, err
|
||||
}
|
||||
|
||||
var rec DatabaseRecord
|
||||
|
||||
if err := rec.Unmarshal(val); err != nil {
|
||||
databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
|
||||
return DatabaseRecord{}, nil
|
||||
}
|
||||
|
||||
rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
|
||||
rec.Addresses = expire(rec.Addresses, s.clock.Now())
|
||||
databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
func (s *levelDBStore) Serve(ctx context.Context) error {
|
||||
t := time.NewTimer(0)
|
||||
defer t.Stop()
|
||||
defer s.db.Close()
|
||||
func (s *inMemoryStore) Serve(ctx context.Context) error {
|
||||
if s.flushInterval <= 0 {
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start the statistics serve routine. It will exit with us when
|
||||
// statisticsTrigger is closed.
|
||||
statisticsTrigger := make(chan struct{})
|
||||
statisticsDone := make(chan struct{})
|
||||
go s.statisticsServe(statisticsTrigger, statisticsDone)
|
||||
t := time.NewTimer(s.flushInterval)
|
||||
defer t.Stop()
|
||||
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
case fn := <-s.inbox:
|
||||
// Run function in serialized order.
|
||||
fn()
|
||||
|
||||
case <-t.C:
|
||||
// Trigger the statistics routine to do its thing in the
|
||||
// background.
|
||||
statisticsTrigger <- struct{}{}
|
||||
|
||||
case <-statisticsDone:
|
||||
// The statistics routine is done with one iteratation, schedule
|
||||
// the next.
|
||||
t.Reset(databaseStatisticsInterval)
|
||||
log.Println("Calculating statistics")
|
||||
s.expireAndCalculateStatistics()
|
||||
log.Println("Flushing database")
|
||||
if err := s.write(); err != nil {
|
||||
log.Println("Error writing database:", err)
|
||||
}
|
||||
log.Println("Finished flushing database")
|
||||
t.Reset(s.flushInterval)
|
||||
|
||||
case <-ctx.Done():
|
||||
// We're done.
|
||||
close(statisticsTrigger)
|
||||
break loop
|
||||
}
|
||||
}
|
||||
|
||||
// Also wait for statisticsServe to return
|
||||
<-statisticsDone
|
||||
return s.write()
|
||||
}
|
||||
|
||||
func (s *inMemoryStore) expireAndCalculateStatistics() {
|
||||
now := s.clock.Now()
|
||||
cutoff24h := now.Add(-24 * time.Hour).UnixNano()
|
||||
cutoff1w := now.Add(-7 * 24 * time.Hour).UnixNano()
|
||||
current, currentIPv4, currentIPv6, currentIPv6GUA, last24h, last1w := 0, 0, 0, 0, 0, 0
|
||||
|
||||
n := 0
|
||||
s.m.Range(func(key protocol.DeviceID, rec DatabaseRecord) bool {
|
||||
if n%1000 == 0 {
|
||||
runtime.Gosched()
|
||||
}
|
||||
n++
|
||||
|
||||
addresses := expire(rec.Addresses, now)
|
||||
if len(addresses) == 0 {
|
||||
rec.Addresses = nil
|
||||
s.m.Store(key, rec)
|
||||
} else if len(addresses) != len(rec.Addresses) {
|
||||
rec.Addresses = addresses
|
||||
s.m.Store(key, rec)
|
||||
}
|
||||
|
||||
switch {
|
||||
case len(rec.Addresses) > 0:
|
||||
current++
|
||||
seenIPv4, seenIPv6, seenIPv6GUA := false, false, false
|
||||
for _, addr := range rec.Addresses {
|
||||
// We do fast and loose matching on strings here instead of
|
||||
// parsing the address and the IP and doing "proper" checks,
|
||||
// to keep things fast and generate less garbage.
|
||||
if strings.Contains(addr.Address, "[") {
|
||||
seenIPv6 = true
|
||||
if strings.Contains(addr.Address, "[2") {
|
||||
seenIPv6GUA = true
|
||||
}
|
||||
} else {
|
||||
seenIPv4 = true
|
||||
}
|
||||
if seenIPv4 && seenIPv6 && seenIPv6GUA {
|
||||
break
|
||||
}
|
||||
}
|
||||
if seenIPv4 {
|
||||
currentIPv4++
|
||||
}
|
||||
if seenIPv6 {
|
||||
currentIPv6++
|
||||
}
|
||||
if seenIPv6GUA {
|
||||
currentIPv6GUA++
|
||||
}
|
||||
case rec.Seen > cutoff24h:
|
||||
last24h++
|
||||
case rec.Seen > cutoff1w:
|
||||
last1w++
|
||||
default:
|
||||
// drop the record if it's older than a week
|
||||
s.m.Delete(key)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
databaseKeys.WithLabelValues("current").Set(float64(current))
|
||||
databaseKeys.WithLabelValues("currentIPv4").Set(float64(currentIPv4))
|
||||
databaseKeys.WithLabelValues("currentIPv6").Set(float64(currentIPv6))
|
||||
databaseKeys.WithLabelValues("currentIPv6GUA").Set(float64(currentIPv6GUA))
|
||||
databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
|
||||
databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
|
||||
databaseStatisticsSeconds.Set(time.Since(now).Seconds())
|
||||
}
|
||||
|
||||
func (s *inMemoryStore) write() (err error) {
|
||||
t0 := time.Now()
|
||||
defer func() {
|
||||
if err == nil {
|
||||
databaseWriteSeconds.Set(time.Since(t0).Seconds())
|
||||
databaseLastWritten.Set(float64(t0.Unix()))
|
||||
}
|
||||
}()
|
||||
|
||||
dbf := path.Join(s.dir, "records.db")
|
||||
fd, err := os.Create(dbf + ".tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bw := bufio.NewWriter(fd)
|
||||
|
||||
var buf []byte
|
||||
var rangeErr error
|
||||
now := s.clock.Now()
|
||||
cutoff1w := now.Add(-7 * 24 * time.Hour).UnixNano()
|
||||
n := 0
|
||||
s.m.Range(func(key protocol.DeviceID, value DatabaseRecord) bool {
|
||||
if n%1000 == 0 {
|
||||
runtime.Gosched()
|
||||
}
|
||||
n++
|
||||
|
||||
if value.Seen < cutoff1w {
|
||||
// drop the record if it's older than a week
|
||||
return true
|
||||
}
|
||||
rec := ReplicationRecord{
|
||||
Key: key[:],
|
||||
Addresses: value.Addresses,
|
||||
Seen: value.Seen,
|
||||
}
|
||||
s := rec.Size()
|
||||
if s+4 > len(buf) {
|
||||
buf = make([]byte, s+4)
|
||||
}
|
||||
n, err := rec.MarshalTo(buf[4:])
|
||||
if err != nil {
|
||||
rangeErr = err
|
||||
return false
|
||||
}
|
||||
binary.BigEndian.PutUint32(buf, uint32(n))
|
||||
if _, err := bw.Write(buf[:n+4]); err != nil {
|
||||
rangeErr = err
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
if rangeErr != nil {
|
||||
_ = fd.Close()
|
||||
return rangeErr
|
||||
}
|
||||
|
||||
if err := bw.Flush(); err != nil {
|
||||
_ = fd.Close
|
||||
return err
|
||||
}
|
||||
if err := fd.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(dbf+".tmp", dbf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload to S3
|
||||
if s.s3 != nil {
|
||||
fd, err = os.Open(dbf)
|
||||
if err != nil {
|
||||
log.Printf("Error uploading database to S3: %v", err)
|
||||
return nil
|
||||
}
|
||||
defer fd.Close()
|
||||
if err := s.s3.upload(fd); err != nil {
|
||||
log.Printf("Error uploading database to S3: %v", err)
|
||||
}
|
||||
log.Println("Finished uploading database")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
|
||||
defer close(done)
|
||||
func (s *inMemoryStore) read() (int, error) {
|
||||
fd, err := os.Open(path.Join(s.dir, "records.db"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
for range trigger {
|
||||
t0 := time.Now()
|
||||
nowNanos := t0.UnixNano()
|
||||
cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
|
||||
cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
|
||||
cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
|
||||
current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
|
||||
|
||||
iter := s.db.NewIterator(&util.Range{}, nil)
|
||||
for iter.Next() {
|
||||
// Attempt to unmarshal the record and count the
|
||||
// failure if there's something wrong with it.
|
||||
var rec DatabaseRecord
|
||||
if err := rec.Unmarshal(iter.Value()); err != nil {
|
||||
errors++
|
||||
continue
|
||||
}
|
||||
|
||||
// If there are addresses that have not expired it's a current
|
||||
// record, otherwise account it based on when it was last seen
|
||||
// (last 24 hours or last week) or finally as inactice.
|
||||
switch {
|
||||
case len(expire(rec.Addresses, nowNanos)) > 0:
|
||||
current++
|
||||
case rec.Seen > cutoff24h:
|
||||
last24h++
|
||||
case rec.Seen > cutoff1w:
|
||||
last1w++
|
||||
case rec.Seen > cutoff2Mon:
|
||||
inactive++
|
||||
case rec.Missed < cutoff2Mon:
|
||||
// It hasn't been seen lately and we haven't recorded
|
||||
// someone asking for this device in a long time either;
|
||||
// delete the record.
|
||||
if err := s.db.Delete(iter.Key(), nil); err != nil {
|
||||
databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
|
||||
} else {
|
||||
databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
|
||||
}
|
||||
default:
|
||||
inactive++
|
||||
br := bufio.NewReader(fd)
|
||||
var buf []byte
|
||||
nr := 0
|
||||
for {
|
||||
var n uint32
|
||||
if err := binary.Read(br, binary.BigEndian, &n); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
return nr, err
|
||||
}
|
||||
if int(n) > len(buf) {
|
||||
buf = make([]byte, n)
|
||||
}
|
||||
if _, err := io.ReadFull(br, buf[:n]); err != nil {
|
||||
return nr, err
|
||||
}
|
||||
rec := ReplicationRecord{}
|
||||
if err := rec.Unmarshal(buf[:n]); err != nil {
|
||||
return nr, err
|
||||
}
|
||||
key, err := protocol.DeviceIDFromBytes(rec.Key)
|
||||
if err != nil {
|
||||
key, err = protocol.DeviceIDFromString(string(rec.Key))
|
||||
}
|
||||
if err != nil {
|
||||
log.Println("Bad device ID:", err)
|
||||
continue
|
||||
}
|
||||
|
||||
iter.Release()
|
||||
|
||||
databaseKeys.WithLabelValues("current").Set(float64(current))
|
||||
databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
|
||||
databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
|
||||
databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
|
||||
databaseKeys.WithLabelValues("error").Set(float64(errors))
|
||||
databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
|
||||
|
||||
// Signal that we are done and can be scheduled again.
|
||||
done <- struct{}{}
|
||||
slices.SortFunc(rec.Addresses, DatabaseAddress.Cmp)
|
||||
rec.Addresses = slices.CompactFunc(rec.Addresses, DatabaseAddress.Equal)
|
||||
s.m.Store(key, DatabaseRecord{
|
||||
Addresses: expire(rec.Addresses, s.clock.Now()),
|
||||
Seen: rec.Seen,
|
||||
})
|
||||
nr++
|
||||
}
|
||||
return nr, nil
|
||||
}
|
||||
|
||||
// merge returns the merged result of the two database records a and b. The
|
||||
// result is the union of the two address sets, with the newer expiry time
|
||||
// chosen for any duplicates.
|
||||
// chosen for any duplicates. The address list in a is overwritten and
|
||||
// reused for the result.
|
||||
func merge(a, b DatabaseRecord) DatabaseRecord {
|
||||
// Both lists must be sorted for this to work.
|
||||
if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
|
||||
log.Println("Warning: bug: addresses not correctly sorted in merge")
|
||||
a.Addresses = sortedAddressCopy(a.Addresses)
|
||||
}
|
||||
if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
|
||||
// no warning because this is the side we read from disk and it may
|
||||
// legitimately predate correct sorting.
|
||||
b.Addresses = sortedAddressCopy(b.Addresses)
|
||||
}
|
||||
|
||||
res := DatabaseRecord{
|
||||
Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
|
||||
Seen: a.Seen,
|
||||
}
|
||||
if b.Seen > a.Seen {
|
||||
res.Seen = b.Seen
|
||||
}
|
||||
a.Seen = max(a.Seen, b.Seen)
|
||||
|
||||
aIdx := 0
|
||||
bIdx := 0
|
||||
aAddrs := a.Addresses
|
||||
bAddrs := b.Addresses
|
||||
loop:
|
||||
for {
|
||||
switch {
|
||||
case aIdx == len(aAddrs) && bIdx == len(bAddrs):
|
||||
// both lists are exhausted, we are done
|
||||
break loop
|
||||
|
||||
case aIdx == len(aAddrs):
|
||||
// a is exhausted, pick from b and continue
|
||||
res.Addresses = append(res.Addresses, bAddrs[bIdx])
|
||||
bIdx++
|
||||
continue
|
||||
|
||||
case bIdx == len(bAddrs):
|
||||
// b is exhausted, pick from a and continue
|
||||
res.Addresses = append(res.Addresses, aAddrs[aIdx])
|
||||
aIdx++
|
||||
continue
|
||||
}
|
||||
|
||||
// We have values left on both sides.
|
||||
aVal := aAddrs[aIdx]
|
||||
bVal := bAddrs[bIdx]
|
||||
|
||||
switch {
|
||||
case aVal.Address == bVal.Address:
|
||||
// update for same address, pick newer
|
||||
if aVal.Expires > bVal.Expires {
|
||||
res.Addresses = append(res.Addresses, aVal)
|
||||
} else {
|
||||
res.Addresses = append(res.Addresses, bVal)
|
||||
}
|
||||
for aIdx < len(a.Addresses) && bIdx < len(b.Addresses) {
|
||||
switch cmp.Compare(a.Addresses[aIdx].Address, b.Addresses[bIdx].Address) {
|
||||
case 0:
|
||||
// a == b, choose the newer expiry time
|
||||
a.Addresses[aIdx].Expires = max(a.Addresses[aIdx].Expires, b.Addresses[bIdx].Expires)
|
||||
aIdx++
|
||||
bIdx++
|
||||
|
||||
case aVal.Address < bVal.Address:
|
||||
// a is smallest, pick it and continue
|
||||
res.Addresses = append(res.Addresses, aVal)
|
||||
case -1:
|
||||
// a < b, keep a and move on
|
||||
aIdx++
|
||||
|
||||
default:
|
||||
// b is smallest, pick it and continue
|
||||
res.Addresses = append(res.Addresses, bVal)
|
||||
case 1:
|
||||
// a > b, insert b before a
|
||||
a.Addresses = append(a.Addresses[:aIdx], append([]DatabaseAddress{b.Addresses[bIdx]}, a.Addresses[aIdx:]...)...)
|
||||
bIdx++
|
||||
}
|
||||
}
|
||||
return res
|
||||
if bIdx < len(b.Addresses) {
|
||||
a.Addresses = append(a.Addresses, b.Addresses[bIdx:]...)
|
||||
}
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
// expire returns the list of addresses after removing expired entries.
|
||||
// Expiration happen in place, so the slice given as the parameter is
|
||||
// destroyed. Internal order is not preserved.
|
||||
func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
|
||||
i := 0
|
||||
for i < len(addrs) {
|
||||
if addrs[i].Expires < now {
|
||||
addrs = sliceutil.RemoveAndZero(addrs, i)
|
||||
// destroyed. Internal order is preserved.
|
||||
func expire(addrs []DatabaseAddress, now time.Time) []DatabaseAddress {
|
||||
cutoff := now.UnixNano()
|
||||
naddrs := addrs[:0]
|
||||
for i := range addrs {
|
||||
if i > 0 && addrs[i].Address == addrs[i-1].Address {
|
||||
// Skip duplicates
|
||||
continue
|
||||
}
|
||||
i++
|
||||
if addrs[i].Expires >= cutoff {
|
||||
naddrs = append(naddrs, addrs[i])
|
||||
}
|
||||
}
|
||||
return addrs
|
||||
if len(naddrs) == 0 {
|
||||
return nil
|
||||
}
|
||||
return naddrs
|
||||
}
|
||||
|
||||
func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
|
||||
sorted := make([]DatabaseAddress, len(addrs))
|
||||
copy(sorted, addrs)
|
||||
sort.Sort(databaseAddressOrder(sorted))
|
||||
return sorted
|
||||
func (d DatabaseAddress) Cmp(other DatabaseAddress) (n int) {
|
||||
if c := cmp.Compare(d.Address, other.Address); c != 0 {
|
||||
return c
|
||||
}
|
||||
return cmp.Compare(d.Expires, other.Expires)
|
||||
}
|
||||
|
||||
type databaseAddressOrder []DatabaseAddress
|
||||
|
||||
func (s databaseAddressOrder) Less(a, b int) bool {
|
||||
return s[a].Address < s[b].Address
|
||||
}
|
||||
|
||||
func (s databaseAddressOrder) Swap(a, b int) {
|
||||
s[a], s[b] = s[b], s[a]
|
||||
}
|
||||
|
||||
func (s databaseAddressOrder) Len() int {
|
||||
return len(s)
|
||||
func (d DatabaseAddress) Equal(other DatabaseAddress) bool {
|
||||
return d.Address == other.Address
|
||||
}
|
||||
|
||||
@@ -25,9 +25,7 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
|
||||
|
||||
type DatabaseRecord struct {
|
||||
Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
|
||||
Misses int32 `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
|
||||
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
|
||||
Missed int64 `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
|
||||
}
|
||||
|
||||
func (m *DatabaseRecord) Reset() { *m = DatabaseRecord{} }
|
||||
@@ -64,7 +62,7 @@ func (m *DatabaseRecord) XXX_DiscardUnknown() {
|
||||
var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
|
||||
|
||||
type ReplicationRecord struct {
|
||||
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
|
||||
Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
|
||||
Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
|
||||
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
|
||||
}
|
||||
@@ -149,24 +147,23 @@ func init() {
|
||||
func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
|
||||
|
||||
var fileDescriptor_b90fe3356ea5df07 = []byte{
|
||||
// 270 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
|
||||
0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
|
||||
0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
|
||||
0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
|
||||
0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
|
||||
0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
|
||||
0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
|
||||
0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
|
||||
0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
|
||||
0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
|
||||
0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
|
||||
0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
|
||||
0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
|
||||
0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
|
||||
0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
|
||||
0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
|
||||
0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
|
||||
// 243 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x49, 0x2c, 0x49,
|
||||
0x4c, 0x4a, 0x2c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc,
|
||||
0x93, 0x52, 0x2e, 0x4a, 0x2d, 0xc8, 0x2f, 0xd6, 0x07, 0x0b, 0x25, 0x95, 0xa6, 0xe9, 0xa7, 0xe7,
|
||||
0xa7, 0xe7, 0x83, 0x39, 0x60, 0x16, 0x44, 0xa9, 0x52, 0x3c, 0x17, 0x9f, 0x0b, 0x54, 0x73, 0x50,
|
||||
0x6a, 0x72, 0x7e, 0x51, 0x8a, 0x90, 0x25, 0x17, 0x67, 0x62, 0x4a, 0x4a, 0x51, 0x6a, 0x71, 0x71,
|
||||
0x6a, 0xb1, 0x04, 0xa3, 0x02, 0xb3, 0x06, 0xb7, 0x91, 0xa8, 0x1e, 0xc8, 0x40, 0x3d, 0x98, 0x42,
|
||||
0x47, 0x88, 0xb4, 0x13, 0xcb, 0x89, 0x7b, 0xf2, 0x0c, 0x41, 0x08, 0xd5, 0x42, 0x42, 0x5c, 0x2c,
|
||||
0xc5, 0xa9, 0xa9, 0x79, 0x12, 0xcc, 0x0a, 0x8c, 0x1a, 0xcc, 0x41, 0x60, 0xb6, 0x52, 0x09, 0x97,
|
||||
0x60, 0x50, 0x6a, 0x41, 0x4e, 0x66, 0x72, 0x62, 0x49, 0x66, 0x7e, 0x1e, 0xd4, 0x0e, 0x01, 0x2e,
|
||||
0xe6, 0xec, 0xd4, 0x4a, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x10, 0x13, 0xd5, 0x56, 0x26,
|
||||
0x8a, 0x6d, 0x75, 0xe5, 0xe2, 0x47, 0xd3, 0x27, 0x24, 0xc1, 0xc5, 0x0e, 0xd5, 0x03, 0xb6, 0x97,
|
||||
0x33, 0x08, 0xc6, 0x05, 0xc9, 0xa4, 0x56, 0x14, 0x64, 0x16, 0x81, 0x6d, 0x06, 0x99, 0x01, 0xe3,
|
||||
0x3a, 0xc9, 0x9c, 0x78, 0x28, 0xc7, 0x70, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f,
|
||||
0x1e, 0xc9, 0x31, 0x4e, 0x78, 0x2c, 0xc7, 0x70, 0xe1, 0xb1, 0x1c, 0xc3, 0x8d, 0xc7, 0x72, 0x0c,
|
||||
0x49, 0x6c, 0xe0, 0x20, 0x34, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xc6, 0x0b, 0x9b, 0x77, 0x7f,
|
||||
0x01, 0x00, 0x00,
|
||||
}
|
||||
|
||||
func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
|
||||
@@ -189,21 +186,11 @@ func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
|
||||
_ = i
|
||||
var l int
|
||||
_ = l
|
||||
if m.Missed != 0 {
|
||||
i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
|
||||
i--
|
||||
dAtA[i] = 0x20
|
||||
}
|
||||
if m.Seen != 0 {
|
||||
i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
|
||||
i--
|
||||
dAtA[i] = 0x18
|
||||
}
|
||||
if m.Misses != 0 {
|
||||
i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
|
||||
i--
|
||||
dAtA[i] = 0x10
|
||||
}
|
||||
if len(m.Addresses) > 0 {
|
||||
for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
|
||||
{
|
||||
@@ -328,15 +315,9 @@ func (m *DatabaseRecord) Size() (n int) {
|
||||
n += 1 + l + sovDatabase(uint64(l))
|
||||
}
|
||||
}
|
||||
if m.Misses != 0 {
|
||||
n += 1 + sovDatabase(uint64(m.Misses))
|
||||
}
|
||||
if m.Seen != 0 {
|
||||
n += 1 + sovDatabase(uint64(m.Seen))
|
||||
}
|
||||
if m.Missed != 0 {
|
||||
n += 1 + sovDatabase(uint64(m.Missed))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -447,25 +428,6 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
|
||||
return err
|
||||
}
|
||||
iNdEx = postIndex
|
||||
case 2:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
|
||||
}
|
||||
m.Misses = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDatabase
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.Misses |= int32(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
case 3:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
|
||||
@@ -485,25 +447,6 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
|
||||
break
|
||||
}
|
||||
}
|
||||
case 4:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
|
||||
}
|
||||
m.Missed = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDatabase
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.Missed |= int64(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := skipDatabase(dAtA[iNdEx:])
|
||||
@@ -558,7 +501,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
|
||||
if wireType != 2 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
|
||||
}
|
||||
var stringLen uint64
|
||||
var byteLen int
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDatabase
|
||||
@@ -568,23 +511,25 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
stringLen |= uint64(b&0x7F) << shift
|
||||
byteLen |= int(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
intStringLen := int(stringLen)
|
||||
if intStringLen < 0 {
|
||||
if byteLen < 0 {
|
||||
return ErrInvalidLengthDatabase
|
||||
}
|
||||
postIndex := iNdEx + intStringLen
|
||||
postIndex := iNdEx + byteLen
|
||||
if postIndex < 0 {
|
||||
return ErrInvalidLengthDatabase
|
||||
}
|
||||
if postIndex > l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
m.Key = string(dAtA[iNdEx:postIndex])
|
||||
m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...)
|
||||
if m.Key == nil {
|
||||
m.Key = []byte{}
|
||||
}
|
||||
iNdEx = postIndex
|
||||
case 2:
|
||||
if wireType != 2 {
|
||||
|
||||
@@ -17,15 +17,11 @@ option (gogoproto.goproto_sizecache_all) = false;
|
||||
|
||||
message DatabaseRecord {
|
||||
repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
|
||||
int32 misses = 2; // Number of lookups* without hits
|
||||
int64 seen = 3; // Unix nanos, last device announce
|
||||
int64 missed = 4; // Unix nanos, last* failed lookup
|
||||
}
|
||||
|
||||
// *) Not every lookup results in a write, so may not be completely accurate
|
||||
|
||||
message ReplicationRecord {
|
||||
string key = 1;
|
||||
bytes key = 1; // raw 32 byte device ID
|
||||
repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
|
||||
int64 seen = 3; // Unix nanos, last device announce
|
||||
}
|
||||
|
||||
@@ -11,29 +11,25 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
func TestDatabaseGetSet(t *testing.T) {
|
||||
db, err := newMemoryLevelDBStore()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db := newInMemoryStore(t.TempDir(), 0, nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go db.Serve(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Check missing record
|
||||
|
||||
rec, err := db.get("abcd")
|
||||
rec, err := db.get(&protocol.EmptyDeviceID)
|
||||
if err != nil {
|
||||
t.Error("not found should not be an error")
|
||||
}
|
||||
if len(rec.Addresses) != 0 {
|
||||
t.Error("addresses should be empty")
|
||||
}
|
||||
if rec.Misses != 0 {
|
||||
t.Error("missing should be zero")
|
||||
}
|
||||
|
||||
// Set up a clock
|
||||
|
||||
@@ -46,13 +42,13 @@ func TestDatabaseGetSet(t *testing.T) {
|
||||
rec.Addresses = []DatabaseAddress{
|
||||
{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
|
||||
}
|
||||
if err := db.put("abcd", rec); err != nil {
|
||||
if err := db.put(&protocol.EmptyDeviceID, rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify it
|
||||
|
||||
rec, err = db.get("abcd")
|
||||
rec, err = db.get(&protocol.EmptyDeviceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -72,13 +68,13 @@ func TestDatabaseGetSet(t *testing.T) {
|
||||
addrs := []DatabaseAddress{
|
||||
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
|
||||
}
|
||||
if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
|
||||
if err := db.merge(&protocol.EmptyDeviceID, addrs, tc.Now().UnixNano()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify it
|
||||
|
||||
rec, err = db.get("abcd")
|
||||
rec, err = db.get(&protocol.EmptyDeviceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -101,7 +97,7 @@ func TestDatabaseGetSet(t *testing.T) {
|
||||
|
||||
// Verify it
|
||||
|
||||
rec, err = db.get("abcd")
|
||||
rec, err = db.get(&protocol.EmptyDeviceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -114,40 +110,18 @@ func TestDatabaseGetSet(t *testing.T) {
|
||||
t.Error("incorrect address")
|
||||
}
|
||||
|
||||
// Put a record with misses
|
||||
|
||||
rec = DatabaseRecord{Misses: 42, Missed: tc.Now().UnixNano()}
|
||||
if err := db.put("efgh", rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify it
|
||||
|
||||
rec, err = db.get("efgh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rec.Addresses) != 0 {
|
||||
t.Log(rec.Addresses)
|
||||
t.Fatal("should have no addresses")
|
||||
}
|
||||
if rec.Misses != 42 {
|
||||
t.Log(rec.Misses)
|
||||
t.Error("incorrect misses")
|
||||
}
|
||||
|
||||
// Set an address
|
||||
|
||||
addrs = []DatabaseAddress{
|
||||
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
|
||||
}
|
||||
if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
|
||||
if err := db.merge(&protocol.GlobalDeviceID, addrs, tc.Now().UnixNano()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify it
|
||||
|
||||
rec, err = db.get("efgh")
|
||||
rec, err = db.get(&protocol.GlobalDeviceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -155,10 +129,6 @@ func TestDatabaseGetSet(t *testing.T) {
|
||||
t.Log(rec.Addresses)
|
||||
t.Fatal("should have one address")
|
||||
}
|
||||
if rec.Misses != 0 {
|
||||
t.Log(rec.Misses)
|
||||
t.Error("should have no misses")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilter(t *testing.T) {
|
||||
@@ -190,13 +160,95 @@ func TestFilter(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
res := expire(tc.a, 10)
|
||||
res := expire(tc.a, time.Unix(0, 10))
|
||||
if fmt.Sprint(res) != fmt.Sprint(tc.b) {
|
||||
t.Errorf("Incorrect result %v, expected %v", res, tc.b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMerge(t *testing.T) {
|
||||
cases := []struct {
|
||||
a, b, res []DatabaseAddress
|
||||
}{
|
||||
{nil, nil, nil},
|
||||
{
|
||||
nil,
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}},
|
||||
},
|
||||
{
|
||||
nil,
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
|
||||
},
|
||||
{
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 15}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 15}},
|
||||
},
|
||||
{
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}},
|
||||
[]DatabaseAddress{{Address: "b", Expires: 15}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
|
||||
},
|
||||
{
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
|
||||
},
|
||||
{
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
|
||||
[]DatabaseAddress{{Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
|
||||
},
|
||||
{
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
|
||||
[]DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
|
||||
},
|
||||
{
|
||||
[]DatabaseAddress{{Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
|
||||
},
|
||||
{
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "d", Expires: 10}},
|
||||
[]DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
|
||||
[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}, {Address: "d", Expires: 10}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
rec := merge(DatabaseRecord{Addresses: tc.a}, DatabaseRecord{Addresses: tc.b})
|
||||
if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
|
||||
t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
|
||||
}
|
||||
rec = merge(DatabaseRecord{Addresses: tc.b}, DatabaseRecord{Addresses: tc.a})
|
||||
if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
|
||||
t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMergeEqual(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
ar := []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}}
|
||||
br := []DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 10}}
|
||||
res := merge(DatabaseRecord{Addresses: ar}, DatabaseRecord{Addresses: br})
|
||||
if len(res.Addresses) != 2 {
|
||||
b.Fatal("wrong length")
|
||||
}
|
||||
if res.Addresses[0].Address != "a" || res.Addresses[1].Address != "b" {
|
||||
b.Fatal("wrong address")
|
||||
}
|
||||
if res.Addresses[0].Expires != 15 || res.Addresses[1].Expires != 15 {
|
||||
b.Fatal("wrong expiry")
|
||||
}
|
||||
}
|
||||
b.ReportAllocs() // should be zero per operation
|
||||
}
|
||||
|
||||
type testClock struct {
|
||||
now time.Time
|
||||
}
|
||||
|
||||
+89
-136
@@ -9,19 +9,22 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"flag"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
_ "net/http/pprof"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/rand"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/thejerf/suture/v4"
|
||||
)
|
||||
|
||||
@@ -36,17 +39,12 @@ const (
|
||||
errorRetryAfterSeconds = 1500
|
||||
errorRetryFuzzSeconds = 300
|
||||
|
||||
// Retry for not found is minSeconds + failures * incSeconds +
|
||||
// random(fuzz), where failures is the number of consecutive lookups
|
||||
// with no answer, up to maxSeconds. The fuzz is applied after capping
|
||||
// to maxSeconds.
|
||||
notFoundRetryMinSeconds = 60
|
||||
notFoundRetryMaxSeconds = 3540
|
||||
notFoundRetryIncSeconds = 10
|
||||
notFoundRetryFuzzSeconds = 60
|
||||
|
||||
// How often (in requests) we serialize the missed counter to database.
|
||||
notFoundMissesWriteInterval = 10
|
||||
// Retry for not found is notFoundRetrySeenSeconds for records we have
|
||||
// seen an announcement for (but it's not active right now) and
|
||||
// notFoundRetryUnknownSeconds for records we have never seen (or not
|
||||
// seen within the last week).
|
||||
notFoundRetryUnknownMinSeconds = 60
|
||||
notFoundRetryUnknownMaxSeconds = 3600
|
||||
|
||||
httpReadTimeout = 5 * time.Second
|
||||
httpWriteTimeout = 5 * time.Second
|
||||
@@ -56,162 +54,117 @@ const (
|
||||
replicationOutboxSize = 10000
|
||||
)
|
||||
|
||||
// These options make the database a little more optimized for writes, at
|
||||
// the expense of some memory usage and risk of losing writes in a (system)
|
||||
// crash.
|
||||
var levelDBOptions = &opt.Options{
|
||||
NoSync: true,
|
||||
WriteBuffer: 32 << 20, // default 4<<20
|
||||
}
|
||||
|
||||
var debug = false
|
||||
|
||||
type CLI struct {
|
||||
Cert string `group:"Listen" help:"Certificate file" default:"./cert.pem" env:"DISCOVERY_CERT_FILE"`
|
||||
Key string `group:"Listen" help:"Key file" default:"./key.pem" env:"DISCOVERY_KEY_FILE"`
|
||||
HTTP bool `group:"Listen" help:"Listen on HTTP (behind an HTTPS proxy)" env:"DISCOVERY_HTTP"`
|
||||
Compression bool `group:"Listen" help:"Enable GZIP compression of responses" env:"DISCOVERY_COMPRESSION"`
|
||||
Listen string `group:"Listen" help:"Listen address" default:":8443" env:"DISCOVERY_LISTEN"`
|
||||
MetricsListen string `group:"Listen" help:"Metrics listen address" env:"DISCOVERY_METRICS_LISTEN"`
|
||||
|
||||
DBDir string `group:"Database" help:"Database directory" default:"." env:"DISCOVERY_DB_DIR"`
|
||||
DBFlushInterval time.Duration `group:"Database" help:"Interval between database flushes" default:"5m" env:"DISCOVERY_DB_FLUSH_INTERVAL"`
|
||||
|
||||
DBS3Endpoint string `name:"db-s3-endpoint" group:"Database (S3 backup)" hidden:"true" help:"S3 endpoint for database" env:"DISCOVERY_DB_S3_ENDPOINT"`
|
||||
DBS3Region string `name:"db-s3-region" group:"Database (S3 backup)" hidden:"true" help:"S3 region for database" env:"DISCOVERY_DB_S3_REGION"`
|
||||
DBS3Bucket string `name:"db-s3-bucket" group:"Database (S3 backup)" hidden:"true" help:"S3 bucket for database" env:"DISCOVERY_DB_S3_BUCKET"`
|
||||
DBS3AccessKeyID string `name:"db-s3-access-key-id" group:"Database (S3 backup)" hidden:"true" help:"S3 access key ID for database" env:"DISCOVERY_DB_S3_ACCESS_KEY_ID"`
|
||||
DBS3SecretKey string `name:"db-s3-secret-key" group:"Database (S3 backup)" hidden:"true" help:"S3 secret key for database" env:"DISCOVERY_DB_S3_SECRET_KEY"`
|
||||
|
||||
AMQPAddress string `group:"AMQP replication" hidden:"true" help:"Address to AMQP broker" env:"DISCOVERY_AMQP_ADDRESS"`
|
||||
|
||||
Debug bool `short:"d" help:"Print debug output" env:"DISCOVERY_DEBUG"`
|
||||
Version bool `short:"v" help:"Print version and exit"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var listen string
|
||||
var dir string
|
||||
var metricsListen string
|
||||
var replicationListen string
|
||||
var replicationPeers string
|
||||
var certFile string
|
||||
var keyFile string
|
||||
var replCertFile string
|
||||
var replKeyFile string
|
||||
var useHTTP bool
|
||||
var largeDB bool
|
||||
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(0)
|
||||
|
||||
flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
|
||||
flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
|
||||
flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
|
||||
flag.BoolVar(&debug, "debug", false, "Print debug output")
|
||||
flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
|
||||
flag.StringVar(&listen, "listen", ":8443", "Listen address")
|
||||
flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
|
||||
flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
|
||||
flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
|
||||
flag.StringVar(&replCertFile, "replication-cert", "", "Certificate file for replication")
|
||||
flag.StringVar(&replKeyFile, "replication-key", "", "Key file for replication")
|
||||
flag.BoolVar(&largeDB, "large-db", false, "Use larger database settings")
|
||||
showVersion := flag.Bool("version", false, "Show version")
|
||||
flag.Parse()
|
||||
var cli CLI
|
||||
kong.Parse(&cli)
|
||||
debug = cli.Debug
|
||||
|
||||
log.Println(build.LongVersionFor("stdiscosrv"))
|
||||
if *showVersion {
|
||||
if cli.Version {
|
||||
return
|
||||
}
|
||||
|
||||
if largeDB {
|
||||
levelDBOptions.BlockCacheCapacity = 64 << 20
|
||||
levelDBOptions.BlockSize = 64 << 10
|
||||
levelDBOptions.CompactionTableSize = 16 << 20
|
||||
levelDBOptions.CompactionTableSizeMultiplier = 2.0
|
||||
levelDBOptions.WriteBuffer = 64 << 20
|
||||
levelDBOptions.CompactionL0Trigger = 8
|
||||
}
|
||||
buildInfo.WithLabelValues(build.Version, runtime.Version(), build.User, build.Date.UTC().Format("2006-01-02T15:04:05Z")).Set(1)
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if os.IsNotExist(err) {
|
||||
log.Println("Failed to load keypair. Generating one, this might take a while...")
|
||||
cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to generate X509 key pair:", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
log.Fatalln("Failed to load keypair:", err)
|
||||
}
|
||||
devID := protocol.NewDeviceID(cert.Certificate[0])
|
||||
log.Println("Server device ID is", devID)
|
||||
|
||||
replCert := cert
|
||||
if replCertFile != "" && replKeyFile != "" {
|
||||
replCert, err = tls.LoadX509KeyPair(replCertFile, replKeyFile)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to load replication keypair:", err)
|
||||
}
|
||||
}
|
||||
replDevID := protocol.NewDeviceID(replCert.Certificate[0])
|
||||
log.Println("Replication device ID is", replDevID)
|
||||
|
||||
// Parse the replication specs, if any.
|
||||
var allowedReplicationPeers []protocol.DeviceID
|
||||
var replicationDestinations []string
|
||||
parts := strings.Split(replicationPeers, ",")
|
||||
for _, part := range parts {
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
fields := strings.Split(part, "@")
|
||||
switch len(fields) {
|
||||
case 2:
|
||||
// This is an id@address specification. Grab the address for the
|
||||
// destination list. Try to resolve it once to catch obvious
|
||||
// syntax errors here rather than having the sender service fail
|
||||
// repeatedly later.
|
||||
_, err := net.ResolveTCPAddr("tcp", fields[1])
|
||||
var cert tls.Certificate
|
||||
if !cli.HTTP {
|
||||
var err error
|
||||
cert, err = tls.LoadX509KeyPair(cli.Cert, cli.Key)
|
||||
if os.IsNotExist(err) {
|
||||
log.Println("Failed to load keypair. Generating one, this might take a while...")
|
||||
cert, err = tlsutil.NewCertificate(cli.Cert, cli.Key, "stdiscosrv", 20*365)
|
||||
if err != nil {
|
||||
log.Fatalln("Resolving address:", err)
|
||||
log.Fatalln("Failed to generate X509 key pair:", err)
|
||||
}
|
||||
replicationDestinations = append(replicationDestinations, fields[1])
|
||||
fallthrough // N.B.
|
||||
|
||||
case 1:
|
||||
// The first part is always a device ID.
|
||||
id, err := protocol.DeviceIDFromString(fields[0])
|
||||
if err != nil {
|
||||
log.Fatalln("Parsing device ID:", err)
|
||||
}
|
||||
if id == protocol.EmptyDeviceID {
|
||||
log.Fatalf("Missing device ID for peer in %q", part)
|
||||
}
|
||||
allowedReplicationPeers = append(allowedReplicationPeers, id)
|
||||
|
||||
default:
|
||||
log.Fatalln("Unrecognized replication spec:", part)
|
||||
} else if err != nil {
|
||||
log.Fatalln("Failed to load keypair:", err)
|
||||
}
|
||||
devID := protocol.NewDeviceID(cert.Certificate[0])
|
||||
log.Println("Server device ID is", devID)
|
||||
}
|
||||
|
||||
// Root of the service tree.
|
||||
main := suture.New("main", suture.Spec{
|
||||
PassThroughPanics: true,
|
||||
Timeout: 2 * time.Minute,
|
||||
})
|
||||
|
||||
// Start the database.
|
||||
db, err := newLevelDBStore(dir)
|
||||
if err != nil {
|
||||
log.Fatalln("Open database:", err)
|
||||
// If configured, use S3 for database backups.
|
||||
var s3c *s3Copier
|
||||
if cli.DBS3Endpoint != "" {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get hostname: %v", err)
|
||||
}
|
||||
key := hostname + ".db"
|
||||
s3c = newS3Copier(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, key, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
|
||||
}
|
||||
|
||||
// Start the database.
|
||||
db := newInMemoryStore(cli.DBDir, cli.DBFlushInterval, s3c)
|
||||
main.Add(db)
|
||||
|
||||
// Start any replication senders.
|
||||
var repl replicationMultiplexer
|
||||
for _, dst := range replicationDestinations {
|
||||
rs := newReplicationSender(dst, replCert, allowedReplicationPeers)
|
||||
main.Add(rs)
|
||||
repl = append(repl, rs)
|
||||
}
|
||||
|
||||
// If we have replication configured, start the replication listener.
|
||||
if len(allowedReplicationPeers) > 0 {
|
||||
rl := newReplicationListener(replicationListen, replCert, allowedReplicationPeers, db)
|
||||
main.Add(rl)
|
||||
// If we have an AMQP broker for replication, start that
|
||||
var repl replicator
|
||||
if cli.AMQPAddress != "" {
|
||||
clientID := rand.String(10)
|
||||
kr := newAMQPReplicator(cli.AMQPAddress, clientID, db)
|
||||
main.Add(kr)
|
||||
repl = kr
|
||||
}
|
||||
|
||||
// Start the main API server.
|
||||
qs := newAPISrv(listen, cert, db, repl, useHTTP)
|
||||
qs := newAPISrv(cli.Listen, cert, db, repl, cli.HTTP, cli.Compression)
|
||||
main.Add(qs)
|
||||
|
||||
// If we have a metrics port configured, start a metrics handler.
|
||||
if metricsListen != "" {
|
||||
if cli.MetricsListen != "" {
|
||||
go func() {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
log.Fatal(http.ListenAndServe(metricsListen, mux))
|
||||
log.Fatal(http.ListenAndServe(cli.MetricsListen, mux))
|
||||
}()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Cancel on signal
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
signal.Notify(signalChan, os.Interrupt)
|
||||
go func() {
|
||||
sig := <-signalChan
|
||||
log.Printf("Received signal %s; shutting down", sig)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// Engage!
|
||||
main.Serve(context.Background())
|
||||
main.Serve(ctx)
|
||||
}
|
||||
|
||||
@@ -1,324 +0,0 @@
|
||||
// Copyright (C) 2018 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
io "io"
|
||||
"log"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
replicationReadTimeout = time.Minute
|
||||
replicationWriteTimeout = 30 * time.Second
|
||||
replicationHeartbeatInterval = time.Second * 30
|
||||
)
|
||||
|
||||
type replicator interface {
|
||||
send(key string, addrs []DatabaseAddress, seen int64)
|
||||
}
|
||||
|
||||
// a replicationSender tries to connect to the remote address and provide
|
||||
// them with a feed of replication updates.
|
||||
type replicationSender struct {
|
||||
dst string
|
||||
cert tls.Certificate // our certificate
|
||||
allowedIDs []protocol.DeviceID
|
||||
outbox chan ReplicationRecord
|
||||
}
|
||||
|
||||
func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
|
||||
return &replicationSender{
|
||||
dst: dst,
|
||||
cert: cert,
|
||||
allowedIDs: allowedIDs,
|
||||
outbox: make(chan ReplicationRecord, replicationOutboxSize),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *replicationSender) Serve(ctx context.Context) error {
|
||||
// Sleep a little at startup. Peers often restart at the same time, and
|
||||
// this avoid the service failing and entering backoff state
|
||||
// unnecessarily, while also reducing the reconnect rate to something
|
||||
// reasonable by default.
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
tlsCfg := &tls.Config{
|
||||
Certificates: []tls.Certificate{s.cert},
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
|
||||
// Dial the TLS connection.
|
||||
conn, err := tls.Dial("tcp", s.dst, tlsCfg)
|
||||
if err != nil {
|
||||
log.Println("Replication connect:", err)
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
conn.SetWriteDeadline(time.Now().Add(time.Second))
|
||||
conn.Close()
|
||||
}()
|
||||
|
||||
// The replication stream is not especially latency sensitive, but it is
|
||||
// quite a lot of data in small writes. Make it more efficient.
|
||||
if tcpc, ok := conn.NetConn().(*net.TCPConn); ok {
|
||||
_ = tcpc.SetNoDelay(false)
|
||||
}
|
||||
|
||||
// Get the other side device ID.
|
||||
remoteID, err := deviceID(conn)
|
||||
if err != nil {
|
||||
log.Println("Replication connect:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Verify it's in the set of allowed device IDs.
|
||||
if !deviceIDIn(remoteID, s.allowedIDs) {
|
||||
log.Println("Replication connect: unexpected device ID:", remoteID)
|
||||
return err
|
||||
}
|
||||
|
||||
heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
|
||||
defer heartBeatTicker.Stop()
|
||||
|
||||
// Send records.
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
select {
|
||||
case <-heartBeatTicker.C:
|
||||
if len(s.outbox) > 0 {
|
||||
// No need to send heartbeats if there are events/prevrious
|
||||
// heartbeats to send, they will keep the connection alive.
|
||||
continue
|
||||
}
|
||||
// Empty replication message is the heartbeat:
|
||||
s.outbox <- ReplicationRecord{}
|
||||
|
||||
case rec := <-s.outbox:
|
||||
// Buffer must hold record plus four bytes for size
|
||||
size := rec.Size()
|
||||
if len(buf) < size+4 {
|
||||
buf = make([]byte, size+4)
|
||||
}
|
||||
|
||||
// Record comes after the four bytes size
|
||||
n, err := rec.MarshalTo(buf[4:])
|
||||
if err != nil {
|
||||
// odd to get an error here, but we haven't sent anything
|
||||
// yet so it's not fatal
|
||||
replicationSendsTotal.WithLabelValues("error").Inc()
|
||||
log.Println("Replication marshal:", err)
|
||||
continue
|
||||
}
|
||||
binary.BigEndian.PutUint32(buf, uint32(n))
|
||||
|
||||
// Send
|
||||
conn.SetWriteDeadline(time.Now().Add(replicationWriteTimeout))
|
||||
if _, err := conn.Write(buf[:4+n]); err != nil {
|
||||
replicationSendsTotal.WithLabelValues("error").Inc()
|
||||
log.Println("Replication write:", err)
|
||||
// Yes, we are losing the replication event here.
|
||||
return err
|
||||
}
|
||||
replicationSendsTotal.WithLabelValues("success").Inc()
|
||||
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *replicationSender) String() string {
|
||||
return fmt.Sprintf("replicationSender(%q)", s.dst)
|
||||
}
|
||||
|
||||
func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
|
||||
item := ReplicationRecord{
|
||||
Key: key,
|
||||
Addresses: ps,
|
||||
}
|
||||
|
||||
// The send should never block. The inbox is suitably buffered for at
|
||||
// least a few seconds of stalls, which shouldn't happen in practice.
|
||||
select {
|
||||
case s.outbox <- item:
|
||||
default:
|
||||
replicationSendsTotal.WithLabelValues("drop").Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// a replicationMultiplexer sends to multiple replicators
|
||||
type replicationMultiplexer []replicator
|
||||
|
||||
func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
|
||||
for _, s := range m {
|
||||
// each send is nonblocking
|
||||
s.send(key, ps, seen)
|
||||
}
|
||||
}
|
||||
|
||||
// replicationListener accepts incoming connections and reads replication
|
||||
// items from them. Incoming items are applied to the KV store.
|
||||
type replicationListener struct {
|
||||
addr string
|
||||
cert tls.Certificate
|
||||
allowedIDs []protocol.DeviceID
|
||||
db database
|
||||
}
|
||||
|
||||
func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
|
||||
return &replicationListener{
|
||||
addr: addr,
|
||||
cert: cert,
|
||||
allowedIDs: allowedIDs,
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
||||
func (l *replicationListener) Serve(ctx context.Context) error {
|
||||
tlsCfg := &tls.Config{
|
||||
Certificates: []tls.Certificate{l.cert},
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
|
||||
lst, err := tls.Listen("tcp", l.addr, tlsCfg)
|
||||
if err != nil {
|
||||
log.Println("Replication listen:", err)
|
||||
return err
|
||||
}
|
||||
defer lst.Close()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
// Accept a connection
|
||||
conn, err := lst.Accept()
|
||||
if err != nil {
|
||||
log.Println("Replication accept:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Figure out the other side device ID
|
||||
remoteID, err := deviceID(conn.(*tls.Conn))
|
||||
if err != nil {
|
||||
log.Println("Replication accept:", err)
|
||||
conn.SetWriteDeadline(time.Now().Add(time.Second))
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
// Verify it is in the set of allowed device IDs
|
||||
if !deviceIDIn(remoteID, l.allowedIDs) {
|
||||
log.Println("Replication accept: unexpected device ID:", remoteID)
|
||||
conn.SetWriteDeadline(time.Now().Add(time.Second))
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
go l.handle(ctx, conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *replicationListener) String() string {
|
||||
return fmt.Sprintf("replicationListener(%q)", l.addr)
|
||||
}
|
||||
|
||||
func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
|
||||
defer func() {
|
||||
conn.SetWriteDeadline(time.Now().Add(time.Second))
|
||||
conn.Close()
|
||||
}()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
|
||||
|
||||
// First four bytes are the size
|
||||
if _, err := io.ReadFull(conn, buf[:4]); err != nil {
|
||||
log.Println("Replication read size:", err)
|
||||
replicationRecvsTotal.WithLabelValues("error").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
// Read the rest of the record
|
||||
size := int(binary.BigEndian.Uint32(buf[:4]))
|
||||
if len(buf) < size {
|
||||
buf = make([]byte, size)
|
||||
}
|
||||
|
||||
if size == 0 {
|
||||
// Heartbeat, ignore
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := io.ReadFull(conn, buf[:size]); err != nil {
|
||||
log.Println("Replication read record:", err)
|
||||
replicationRecvsTotal.WithLabelValues("error").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
// Unmarshal
|
||||
var rec ReplicationRecord
|
||||
if err := rec.Unmarshal(buf[:size]); err != nil {
|
||||
log.Println("Replication unmarshal:", err)
|
||||
replicationRecvsTotal.WithLabelValues("error").Inc()
|
||||
continue
|
||||
}
|
||||
|
||||
// Store
|
||||
l.db.merge(rec.Key, rec.Addresses, rec.Seen)
|
||||
replicationRecvsTotal.WithLabelValues("success").Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
|
||||
// Handshake may not be complete on the server side yet, which we need
|
||||
// to get the client certificate.
|
||||
if !conn.ConnectionState().HandshakeComplete {
|
||||
if err := conn.Handshake(); err != nil {
|
||||
return protocol.DeviceID{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// We expect exactly one certificate.
|
||||
certs := conn.ConnectionState().PeerCertificates
|
||||
if len(certs) != 1 {
|
||||
return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
|
||||
}
|
||||
|
||||
return protocol.NewDeviceID(certs[0].Raw), nil
|
||||
}
|
||||
|
||||
func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
|
||||
for _, candidate := range ids {
|
||||
if id == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright (C) 2024 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
type s3Copier struct {
|
||||
endpoint string
|
||||
region string
|
||||
bucket string
|
||||
key string
|
||||
accessKeyID string
|
||||
secretKey string
|
||||
}
|
||||
|
||||
func newS3Copier(endpoint, region, bucket, key, accessKeyID, secretKey string) *s3Copier {
|
||||
return &s3Copier{
|
||||
endpoint: endpoint,
|
||||
region: region,
|
||||
bucket: bucket,
|
||||
key: key,
|
||||
accessKeyID: accessKeyID,
|
||||
secretKey: secretKey,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *s3Copier) upload(r io.Reader) error {
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String(s.region),
|
||||
Endpoint: aws.String(s.endpoint),
|
||||
Credentials: credentials.NewStaticCredentials(s.accessKeyID, s.secretKey, ""),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uploader := s3manager.NewUploader(sess)
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(s.key),
|
||||
Body: r,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *s3Copier) downloadLatest(w io.WriterAt) error {
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String(s.region),
|
||||
Endpoint: aws.String(s.endpoint),
|
||||
Credentials: credentials.NewStaticCredentials(s.accessKeyID, s.secretKey, ""),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
svc := s3.New(sess)
|
||||
resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.bucket)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var lastKey string
|
||||
var lastModified time.Time
|
||||
var lastSize int64
|
||||
for _, item := range resp.Contents {
|
||||
if item.LastModified.After(lastModified) && *item.Size > lastSize {
|
||||
lastKey = *item.Key
|
||||
lastModified = *item.LastModified
|
||||
lastSize = *item.Size
|
||||
} else if lastModified.Sub(*item.LastModified) < 5*time.Minute && *item.Size > lastSize {
|
||||
lastKey = *item.Key
|
||||
lastSize = *item.Size
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("Downloading database from", lastKey)
|
||||
downloader := s3manager.NewDownloader(sess)
|
||||
_, err = downloader.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(lastKey),
|
||||
})
|
||||
return err
|
||||
}
|
||||
+36
-17
@@ -7,13 +7,18 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
)
|
||||
|
||||
var (
|
||||
buildInfo = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "discovery",
|
||||
Name: "build_info",
|
||||
Help: "A metric with a constant '1' value labeled by version, goversion, builduser and builddate from which stdiscosrv was built.",
|
||||
}, []string{"version", "goversion", "builduser", "builddate"})
|
||||
|
||||
apiRequestsTotal = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
@@ -90,6 +95,29 @@ var (
|
||||
Help: "Latency of database operations.",
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
}, []string{"operation"})
|
||||
|
||||
databaseWriteSeconds = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "discovery",
|
||||
Name: "database_write_seconds",
|
||||
Help: "Time spent writing the database.",
|
||||
})
|
||||
databaseLastWritten = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "discovery",
|
||||
Name: "database_last_written",
|
||||
Help: "Timestamp of the last successful database write.",
|
||||
})
|
||||
|
||||
retryAfterLevel = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "discovery",
|
||||
Name: "retry_after_seconds",
|
||||
Help: "Retry-After header value in seconds.",
|
||||
}, []string{"name"})
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -104,21 +132,12 @@ const (
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
|
||||
prometheus.MustRegister(buildInfo,
|
||||
apiRequestsTotal, apiRequestsSeconds,
|
||||
lookupRequestsTotal, announceRequestsTotal,
|
||||
replicationSendsTotal, replicationRecvsTotal,
|
||||
databaseKeys, databaseStatisticsSeconds,
|
||||
databaseOperations, databaseOperationSeconds)
|
||||
|
||||
processCollectorOpts := collectors.ProcessCollectorOpts{
|
||||
Namespace: "syncthing_discovery",
|
||||
PidFn: func() (int, error) {
|
||||
return os.Getpid(), nil
|
||||
},
|
||||
}
|
||||
|
||||
prometheus.MustRegister(
|
||||
collectors.NewProcessCollector(processCollectorOpts),
|
||||
)
|
||||
|
||||
databaseOperations, databaseOperationSeconds,
|
||||
databaseWriteSeconds, databaseLastWritten,
|
||||
retryAfterLevel)
|
||||
}
|
||||
|
||||
+15
-8
@@ -19,19 +19,18 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/events"
|
||||
"github.com/syncthing/syncthing/lib/nat"
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
_ "github.com/syncthing/syncthing/lib/pmp"
|
||||
syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/relay/protocol"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/nat"
|
||||
_ "github.com/syncthing/syncthing/lib/pmp"
|
||||
_ "github.com/syncthing/syncthing/lib/upnp"
|
||||
|
||||
syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -194,7 +193,15 @@ func main() {
|
||||
cfg.Options.NATTimeoutS = natTimeout
|
||||
})
|
||||
natSvc := nat.NewService(id, wrapper)
|
||||
mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
|
||||
var ipVersion nat.IPVersion
|
||||
if strings.HasSuffix(proto, "4") {
|
||||
ipVersion = nat.IPv4Only
|
||||
} else if strings.HasSuffix(proto, "6") {
|
||||
ipVersion = nat.IPv6Only
|
||||
} else {
|
||||
ipVersion = nat.IPvAny
|
||||
}
|
||||
mapping := mapping{natSvc.NewMapping(nat.TCP, ipVersion, addr.IP, addr.Port)}
|
||||
|
||||
if natEnabled {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/relay/client"
|
||||
"github.com/syncthing/syncthing/lib/relay/protocol"
|
||||
|
||||
+21
-11
@@ -13,6 +13,7 @@ import (
|
||||
"reflect"
|
||||
|
||||
"github.com/AudriusButkevicius/recli"
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
@@ -23,9 +24,20 @@ type configHandler struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func getConfigCommand(f *apiClientFactory) (cli.Command, error) {
|
||||
type configCommand struct {
|
||||
Args []string `arg:"" default:"-h"`
|
||||
}
|
||||
|
||||
func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
|
||||
app := cli.NewApp()
|
||||
app.Name = "syncthing"
|
||||
app.Author = "The Syncthing Authors"
|
||||
app.Metadata = map[string]interface{}{
|
||||
"clientFactory": ctx.clientFactory,
|
||||
}
|
||||
|
||||
h := new(configHandler)
|
||||
h.client, h.err = f.getClient()
|
||||
h.client, h.err = ctx.clientFactory.getClient()
|
||||
if h.err == nil {
|
||||
h.cfg, h.err = getConfig(h.client)
|
||||
}
|
||||
@@ -38,17 +50,15 @@ func getConfigCommand(f *apiClientFactory) (cli.Command, error) {
|
||||
|
||||
commands, err := recli.New(recliCfg).Construct(&h.cfg)
|
||||
if err != nil {
|
||||
return cli.Command{}, fmt.Errorf("config reflect: %w", err)
|
||||
return fmt.Errorf("config reflect: %w", err)
|
||||
}
|
||||
|
||||
return cli.Command{
|
||||
Name: "config",
|
||||
HideHelp: true,
|
||||
Usage: "Configuration modification command group",
|
||||
Subcommands: commands,
|
||||
Before: h.configBefore,
|
||||
After: h.configAfter,
|
||||
}, nil
|
||||
app.Commands = commands
|
||||
app.HideHelp = true
|
||||
app.Before = h.configBefore
|
||||
app.After = h.configAfter
|
||||
|
||||
return app.Run(append([]string{app.Name}, c.Args...))
|
||||
}
|
||||
|
||||
func (h *configHandler) configBefore(c *cli.Context) error {
|
||||
|
||||
+26
-36
@@ -9,47 +9,37 @@ package cli
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
var debugCommand = cli.Command{
|
||||
Name: "debug",
|
||||
HideHelp: true,
|
||||
Usage: "Debug command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "file",
|
||||
Usage: "Show information about a file (or directory/symlink)",
|
||||
ArgsUsage: "FOLDER-ID PATH",
|
||||
Action: expects(2, debugFile()),
|
||||
},
|
||||
indexCommand,
|
||||
{
|
||||
Name: "profile",
|
||||
Usage: "Save a profile to help figuring out what Syncthing does.",
|
||||
ArgsUsage: "cpu | heap",
|
||||
Action: expects(1, profile()),
|
||||
},
|
||||
},
|
||||
type fileCommand struct {
|
||||
FolderID string `arg:""`
|
||||
Path string `arg:""`
|
||||
}
|
||||
|
||||
func debugFile() cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
query := make(url.Values)
|
||||
query.Set("folder", c.Args()[0])
|
||||
query.Set("file", normalizePath(c.Args()[1]))
|
||||
return indexDumpOutput("debug/file?" + query.Encode())(c)
|
||||
func (f *fileCommand) Run(ctx Context) error {
|
||||
indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
|
||||
|
||||
query := make(url.Values)
|
||||
query.Set("folder", f.FolderID)
|
||||
query.Set("file", normalizePath(f.Path))
|
||||
return indexDumpOutput("debug/file?" + query.Encode())
|
||||
}
|
||||
|
||||
type profileCommand struct {
|
||||
Type string `arg:"" help:"cpu | heap"`
|
||||
}
|
||||
|
||||
func (p *profileCommand) Run(ctx Context) error {
|
||||
switch t := p.Type; t {
|
||||
case "cpu", "heap":
|
||||
return saveToFile(fmt.Sprintf("debug/%vprof", p.Type), ctx.clientFactory)
|
||||
default:
|
||||
return fmt.Errorf("expected cpu or heap as argument, got %v", t)
|
||||
}
|
||||
}
|
||||
|
||||
func profile() cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
switch t := c.Args()[0]; t {
|
||||
case "cpu", "heap":
|
||||
return saveToFile(fmt.Sprintf("debug/%vprof", c.Args()[0]))(c)
|
||||
default:
|
||||
return fmt.Errorf("expected cpu or heap as argument, got %v", t)
|
||||
}
|
||||
}
|
||||
type debugCommand struct {
|
||||
File fileCommand `cmd:"" help:"Show information about a file (or directory/symlink)"`
|
||||
Profile profileCommand `cmd:"" help:"Save a profile to help figuring out what Syncthing does"`
|
||||
Index indexCommand `cmd:"" help:"Show information about the index (database)"`
|
||||
}
|
||||
|
||||
+25
-26
@@ -11,36 +11,25 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
"github.com/alecthomas/kong"
|
||||
)
|
||||
|
||||
var errorsCommand = cli.Command{
|
||||
Name: "errors",
|
||||
HideHelp: true,
|
||||
Usage: "Error command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "show",
|
||||
Usage: "Show pending errors",
|
||||
Action: expects(0, indexDumpOutput("system/error")),
|
||||
},
|
||||
{
|
||||
Name: "push",
|
||||
Usage: "Push an error to active clients",
|
||||
ArgsUsage: "ERROR-MESSAGE",
|
||||
Action: expects(1, errorsPush),
|
||||
},
|
||||
{
|
||||
Name: "clear",
|
||||
Usage: "Clear pending errors",
|
||||
Action: expects(0, emptyPost("system/error/clear")),
|
||||
},
|
||||
},
|
||||
type errorsCommand struct {
|
||||
Show struct{} `cmd:"" help:"Show pending errors"`
|
||||
Push errorsPushCommand `cmd:"" help:"Push an error to active clients"`
|
||||
Clear struct{} `cmd:"" help:"Clear pending errors"`
|
||||
}
|
||||
|
||||
func errorsPush(c *cli.Context) error {
|
||||
client := c.App.Metadata["client"].(APIClient)
|
||||
errStr := strings.Join(c.Args(), " ")
|
||||
type errorsPushCommand struct {
|
||||
ErrorMessage string `arg:""`
|
||||
}
|
||||
|
||||
func (e *errorsPushCommand) Run(ctx Context) error {
|
||||
client, err := ctx.clientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errStr := e.ErrorMessage
|
||||
response, err := client.Post("system/error", strings.TrimSpace(errStr))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -59,3 +48,13 @@ func errorsPush(c *cli.Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*errorsCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
switch kongCtx.Selected().Name {
|
||||
case "show":
|
||||
return indexDumpOutput("system/error", ctx.clientFactory)
|
||||
case "clear":
|
||||
return emptyPost("system/error/clear", ctx.clientFactory)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+20
-26
@@ -7,32 +7,26 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/urfave/cli"
|
||||
"github.com/alecthomas/kong"
|
||||
)
|
||||
|
||||
var indexCommand = cli.Command{
|
||||
Name: "index",
|
||||
Usage: "Show information about the index (database)",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "dump",
|
||||
Usage: "Print the entire db",
|
||||
Action: expects(0, indexDump),
|
||||
},
|
||||
{
|
||||
Name: "dump-size",
|
||||
Usage: "Print the db size of different categories of information",
|
||||
Action: expects(0, indexDumpSize),
|
||||
},
|
||||
{
|
||||
Name: "check",
|
||||
Usage: "Check the database for inconsistencies",
|
||||
Action: expects(0, indexCheck),
|
||||
},
|
||||
{
|
||||
Name: "account",
|
||||
Usage: "Print key and value size statistics per key type",
|
||||
Action: expects(0, indexAccount),
|
||||
},
|
||||
},
|
||||
type indexCommand struct {
|
||||
Dump struct{} `cmd:"" help:"Print the entire db"`
|
||||
DumpSize struct{} `cmd:"" help:"Print the db size of different categories of information"`
|
||||
Check struct{} `cmd:"" help:"Check the database for inconsistencies"`
|
||||
Account struct{} `cmd:"" help:"Print key and value size statistics per key type"`
|
||||
}
|
||||
|
||||
func (*indexCommand) Run(kongCtx *kong.Context) error {
|
||||
switch kongCtx.Selected().Name {
|
||||
case "dump":
|
||||
return indexDump()
|
||||
case "dump-size":
|
||||
return indexDumpSize()
|
||||
case "check":
|
||||
return indexCheck()
|
||||
case "account":
|
||||
return indexAccount()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -10,12 +10,10 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
// indexAccount prints key and data size statistics per class
|
||||
func indexAccount(*cli.Context) error {
|
||||
func indexAccount() error {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -11,13 +11,11 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
func indexDump(*cli.Context) error {
|
||||
func indexDump() error {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -11,12 +11,10 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
)
|
||||
|
||||
func indexDumpSize(*cli.Context) error {
|
||||
func indexDumpSize() error {
|
||||
type sizedElement struct {
|
||||
key string
|
||||
size int
|
||||
|
||||
@@ -13,8 +13,6 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
@@ -35,7 +33,7 @@ type sequenceKey struct {
|
||||
sequence uint64
|
||||
}
|
||||
|
||||
func indexCheck(*cli.Context) (err error) {
|
||||
func indexCheck() (err error) {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+54
-131
@@ -8,165 +8,88 @@ package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/flynn-archive/go-shlex"
|
||||
"github.com/urfave/cli"
|
||||
"github.com/kballard/go-shellquote"
|
||||
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
)
|
||||
|
||||
type preCli struct {
|
||||
type CLI struct {
|
||||
cmdutil.CommonOptions
|
||||
DataDir string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
|
||||
GUIAddress string `name:"gui-address"`
|
||||
GUIAPIKey string `name:"gui-apikey"`
|
||||
HomeDir string `name:"home"`
|
||||
ConfDir string `name:"config"`
|
||||
DataDir string `name:"data"`
|
||||
|
||||
Show showCommand `cmd:"" help:"Show command group"`
|
||||
Debug debugCommand `cmd:"" help:"Debug command group"`
|
||||
Operations operationCommand `cmd:"" help:"Operation command group"`
|
||||
Errors errorsCommand `cmd:"" help:"Error command group"`
|
||||
Config configCommand `cmd:"" help:"Configuration modification command group" passthrough:""`
|
||||
Stdin stdinCommand `cmd:"" name:"-" help:"Read commands from stdin"`
|
||||
}
|
||||
|
||||
func Run() error {
|
||||
// This is somewhat a hack around a chicken and egg problem. We need to set
|
||||
// the home directory and potentially other flags to know where the
|
||||
// syncthing instance is running in order to get it's config ... which we
|
||||
// then use to construct the actual CLI ... at which point it's too late to
|
||||
// add flags there...
|
||||
c := preCli{}
|
||||
parseFlags(&c)
|
||||
return runInternal(c, os.Args)
|
||||
type Context struct {
|
||||
clientFactory *apiClientFactory
|
||||
}
|
||||
|
||||
func RunWithArgs(cliArgs []string) error {
|
||||
c := preCli{}
|
||||
parseFlagsWithArgs(cliArgs, &c)
|
||||
return runInternal(c, cliArgs)
|
||||
}
|
||||
|
||||
func runInternal(c preCli, cliArgs []string) error {
|
||||
// Not set as default above because the strings can be really long.
|
||||
err := cmdutil.SetConfigDataLocationsFromFlags(c.HomeDir, c.ConfDir, c.DataDir)
|
||||
func (cli CLI) AfterApply(kongCtx *kong.Context) error {
|
||||
err := cmdutil.SetConfigDataLocationsFromFlags(cli.HomeDir, cli.ConfDir, cli.DataDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Command line options: %w", err)
|
||||
return fmt.Errorf("command line options: %w", err)
|
||||
}
|
||||
|
||||
clientFactory := &apiClientFactory{
|
||||
cfg: config.GUIConfiguration{
|
||||
RawAddress: c.GUIAddress,
|
||||
APIKey: c.GUIAPIKey,
|
||||
RawAddress: cli.GUIAddress,
|
||||
APIKey: cli.GUIAPIKey,
|
||||
},
|
||||
}
|
||||
|
||||
configCommand, err := getConfigCommand(clientFactory)
|
||||
if err != nil {
|
||||
return err
|
||||
context := Context{
|
||||
clientFactory: clientFactory,
|
||||
}
|
||||
|
||||
// Implement the same flags at the upper CLI, but do nothing with them.
|
||||
// This is so that the usage text is the same
|
||||
fakeFlags := []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "gui-address",
|
||||
Usage: "Override GUI address to `URL` (e.g. \"192.0.2.42:8443\")",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "gui-apikey",
|
||||
Usage: "Override GUI API key to `API-KEY`",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "home",
|
||||
Usage: "Set configuration and data directory to `PATH`",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "config",
|
||||
Usage: "Set configuration directory (config and keys) to `PATH`",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "data",
|
||||
Usage: "Set data directory (database and logs) to `PATH`",
|
||||
},
|
||||
}
|
||||
|
||||
// Construct the actual CLI
|
||||
app := cli.NewApp()
|
||||
app.Author = "The Syncthing Authors"
|
||||
app.Metadata = map[string]interface{}{
|
||||
"clientFactory": clientFactory,
|
||||
}
|
||||
app.Commands = []cli.Command{{
|
||||
Name: "cli",
|
||||
Usage: "Syncthing command line interface",
|
||||
Flags: fakeFlags,
|
||||
Subcommands: []cli.Command{
|
||||
configCommand,
|
||||
showCommand,
|
||||
operationCommand,
|
||||
errorsCommand,
|
||||
debugCommand,
|
||||
{
|
||||
Name: "-",
|
||||
HideHelp: true,
|
||||
Usage: "Read commands from stdin",
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if ctx.NArg() > 0 {
|
||||
return errors.New("command does not expect any arguments")
|
||||
}
|
||||
|
||||
// Drop the `-` not to recurse into self.
|
||||
args := make([]string, len(cliArgs)-1)
|
||||
copy(args, cliArgs)
|
||||
|
||||
fmt.Println("Reading commands from stdin...", args)
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
input, err := shlex.Split(scanner.Text())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing input: %w", err)
|
||||
}
|
||||
if len(input) == 0 {
|
||||
continue
|
||||
}
|
||||
err = app.Run(append(args, input...))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
},
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
return app.Run(cliArgs)
|
||||
kongCtx.Bind(context)
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseFlags(c *preCli) error {
|
||||
// kong only needs to parse the global arguments after "cli" and before the
|
||||
// subcommand (if any).
|
||||
if len(os.Args) <= 2 {
|
||||
return nil
|
||||
}
|
||||
return parseFlagsWithArgs(os.Args[2:], c)
|
||||
}
|
||||
type stdinCommand struct{}
|
||||
|
||||
func parseFlagsWithArgs(args []string, c *preCli) error {
|
||||
for i := 0; i < len(args); i++ {
|
||||
if !strings.HasPrefix(args[i], "--") {
|
||||
args = args[:i]
|
||||
break
|
||||
func (*stdinCommand) Run() error {
|
||||
// Drop the `-` not to recurse into self.
|
||||
args := make([]string, len(os.Args)-1)
|
||||
copy(args, os.Args)
|
||||
|
||||
fmt.Println("Reading commands from stdin...", args)
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
input, err := shellquote.Split(scanner.Text())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing input: %w", err)
|
||||
}
|
||||
if !strings.Contains(args[i], "=") {
|
||||
i++
|
||||
if len(input) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var cli CLI
|
||||
p, err := kong.New(&cli)
|
||||
if err != nil {
|
||||
// can't happen, really
|
||||
return fmt.Errorf("creating parser: %w", err)
|
||||
}
|
||||
ctx, err := p.Parse(input)
|
||||
if err != nil {
|
||||
fmt.Println("Error:", err)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Run(); err != nil {
|
||||
fmt.Println("Error:", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
// We don't want kong to print anything nor os.Exit (e.g. on -h)
|
||||
parser, err := kong.New(c, kong.Writers(io.Discard, io.Discard), kong.Exit(func(int) {}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = parser.Parse(args)
|
||||
return err
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
@@ -12,48 +12,43 @@ import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
var operationCommand = cli.Command{
|
||||
Name: "operations",
|
||||
HideHelp: true,
|
||||
Usage: "Operation command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "restart",
|
||||
Usage: "Restart syncthing",
|
||||
Action: expects(0, emptyPost("system/restart")),
|
||||
},
|
||||
{
|
||||
Name: "shutdown",
|
||||
Usage: "Shutdown syncthing",
|
||||
Action: expects(0, emptyPost("system/shutdown")),
|
||||
},
|
||||
{
|
||||
Name: "upgrade",
|
||||
Usage: "Upgrade syncthing (if a newer version is available)",
|
||||
Action: expects(0, emptyPost("system/upgrade")),
|
||||
},
|
||||
{
|
||||
Name: "folder-override",
|
||||
Usage: "Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data.",
|
||||
ArgsUsage: "FOLDER-ID",
|
||||
Action: expects(1, foldersOverride),
|
||||
},
|
||||
{
|
||||
Name: "default-ignores",
|
||||
Usage: "Set the default ignores (config) from a file",
|
||||
ArgsUsage: "PATH",
|
||||
Action: expects(1, setDefaultIgnores),
|
||||
},
|
||||
},
|
||||
type folderOverrideCommand struct {
|
||||
FolderID string `arg:""`
|
||||
}
|
||||
|
||||
func foldersOverride(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
type defaultIgnoresCommand struct {
|
||||
Path string `arg:""`
|
||||
}
|
||||
|
||||
type operationCommand struct {
|
||||
Restart struct{} `cmd:"" help:"Restart syncthing"`
|
||||
Shutdown struct{} `cmd:"" help:"Shutdown syncthing"`
|
||||
Upgrade struct{} `cmd:"" help:"Upgrade syncthing (if a newer version is available)"`
|
||||
FolderOverride folderOverrideCommand `cmd:"" help:"Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data"`
|
||||
DefaultIgnores defaultIgnoresCommand `cmd:"" help:"Set the default ignores (config) from a file"`
|
||||
}
|
||||
|
||||
func (*operationCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
f := ctx.clientFactory
|
||||
|
||||
switch kongCtx.Selected().Name {
|
||||
case "restart":
|
||||
return emptyPost("system/restart", f)
|
||||
case "shutdown":
|
||||
return emptyPost("system/shutdown", f)
|
||||
case "upgrade":
|
||||
return emptyPost("system/upgrade", f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *folderOverrideCommand) Run(ctx Context) error {
|
||||
client, err := ctx.clientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -61,7 +56,7 @@ func foldersOverride(c *cli.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rid := c.Args()[0]
|
||||
rid := f.FolderID
|
||||
for _, folder := range cfg.Folders {
|
||||
if folder.ID == rid {
|
||||
response, err := client.Post("db/override", "")
|
||||
@@ -86,12 +81,12 @@ func foldersOverride(c *cli.Context) error {
|
||||
return fmt.Errorf("Folder %q not found", rid)
|
||||
}
|
||||
|
||||
func setDefaultIgnores(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
func (d *defaultIgnoresCommand) Run(ctx Context) error {
|
||||
client, err := ctx.clientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, file := filepath.Split(c.Args()[0])
|
||||
dir, file := filepath.Split(d.Path)
|
||||
filesystem := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
|
||||
|
||||
fd, err := filesystem.Open(file)
|
||||
|
||||
@@ -9,37 +9,30 @@ package cli
|
||||
import (
|
||||
"net/url"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
"github.com/alecthomas/kong"
|
||||
)
|
||||
|
||||
var pendingCommand = cli.Command{
|
||||
Name: "pending",
|
||||
HideHelp: true,
|
||||
Usage: "Pending subcommand group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "devices",
|
||||
Usage: "Show pending devices",
|
||||
Action: expects(0, indexDumpOutput("cluster/pending/devices")),
|
||||
},
|
||||
{
|
||||
Name: "folders",
|
||||
Usage: "Show pending folders",
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{Name: "device", Usage: "Show pending folders offered by given device"},
|
||||
},
|
||||
Action: expects(0, folders()),
|
||||
},
|
||||
},
|
||||
type pendingCommand struct {
|
||||
Devices struct{} `cmd:"" help:"Show pending devices"`
|
||||
Folders struct {
|
||||
Device string `help:"Show pending folders offered by given device"`
|
||||
} `cmd:"" help:"Show pending folders"`
|
||||
}
|
||||
|
||||
func folders() cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
if c.String("device") != "" {
|
||||
func (p *pendingCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
|
||||
|
||||
switch kongCtx.Selected().Name {
|
||||
case "devices":
|
||||
return indexDumpOutput("cluster/pending/devices")
|
||||
case "folders":
|
||||
if p.Folders.Device != "" {
|
||||
query := make(url.Values)
|
||||
query.Set("device", c.String("device"))
|
||||
return indexDumpOutput("cluster/pending/folders?" + query.Encode())(c)
|
||||
query.Set("device", p.Folders.Device)
|
||||
return indexDumpOutput("cluster/pending/folders?" + query.Encode())
|
||||
}
|
||||
return indexDumpOutput("cluster/pending/folders")(c)
|
||||
return indexDumpOutput("cluster/pending/folders")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+30
-38
@@ -7,44 +7,36 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/urfave/cli"
|
||||
"github.com/alecthomas/kong"
|
||||
)
|
||||
|
||||
var showCommand = cli.Command{
|
||||
Name: "show",
|
||||
HideHelp: true,
|
||||
Usage: "Show command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "version",
|
||||
Usage: "Show syncthing client version",
|
||||
Action: expects(0, indexDumpOutput("system/version")),
|
||||
},
|
||||
{
|
||||
Name: "config-status",
|
||||
Usage: "Show configuration status, whether or not a restart is required for changes to take effect",
|
||||
Action: expects(0, indexDumpOutput("config/restart-required")),
|
||||
},
|
||||
{
|
||||
Name: "system",
|
||||
Usage: "Show system status",
|
||||
Action: expects(0, indexDumpOutput("system/status")),
|
||||
},
|
||||
{
|
||||
Name: "connections",
|
||||
Usage: "Report about connections to other devices",
|
||||
Action: expects(0, indexDumpOutput("system/connections")),
|
||||
},
|
||||
{
|
||||
Name: "discovery",
|
||||
Usage: "Show the discovered addresses of remote devices (from cache of the running syncthing instance)",
|
||||
Action: expects(0, indexDumpOutput("system/discovery")),
|
||||
},
|
||||
pendingCommand,
|
||||
{
|
||||
Name: "usage",
|
||||
Usage: "Show usage report",
|
||||
Action: expects(0, indexDumpOutput("svc/report")),
|
||||
},
|
||||
},
|
||||
type showCommand struct {
|
||||
Version struct{} `cmd:"" help:"Show syncthing client version"`
|
||||
ConfigStatus struct{} `cmd:"" help:"Show configuration status, whether or not a restart is required for changes to take effect"`
|
||||
System struct{} `cmd:"" help:"Show system status"`
|
||||
Connections struct{} `cmd:"" help:"Report about connections to other devices"`
|
||||
Discovery struct{} `cmd:"" help:"Show the discovered addresses of remote devices (from cache of the running syncthing instance)"`
|
||||
Usage struct{} `cmd:"" help:"Show usage report"`
|
||||
Pending pendingCommand `cmd:"" help:"Pending subcommand group"`
|
||||
}
|
||||
|
||||
func (*showCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
|
||||
|
||||
switch kongCtx.Selected().Name {
|
||||
case "version":
|
||||
return indexDumpOutput("system/version")
|
||||
case "config-status":
|
||||
return indexDumpOutput("config/restart-required")
|
||||
case "system":
|
||||
return indexDumpOutput("system/status")
|
||||
case "connections":
|
||||
return indexDumpOutput("system/connections")
|
||||
case "discovery":
|
||||
return indexDumpOutput("system/discovery")
|
||||
case "usage":
|
||||
return indexDumpOutput("svc/report")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+60
-74
@@ -19,7 +19,6 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/db/backend"
|
||||
"github.com/syncthing/syncthing/lib/locations"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
func responseToBArray(response *http.Response) ([]byte, error) {
|
||||
@@ -30,68 +29,72 @@ func responseToBArray(response *http.Response) ([]byte, error) {
|
||||
return bytes, response.Body.Close()
|
||||
}
|
||||
|
||||
func emptyPost(url string) cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = client.Post(url, "")
|
||||
func emptyPost(url string, apiClientFactory *apiClientFactory) error {
|
||||
client, err := apiClientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = client.Post(url, "")
|
||||
return err
|
||||
}
|
||||
|
||||
func indexDumpOutputWrapper(apiClientFactory *apiClientFactory) func(url string) error {
|
||||
return func(url string) error {
|
||||
return indexDumpOutput(url, apiClientFactory)
|
||||
}
|
||||
}
|
||||
|
||||
func indexDumpOutput(url string) cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if errors.Is(err, errNotFound) {
|
||||
return errors.New("not found (folder/file not in database)")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return prettyPrintResponse(response)
|
||||
}
|
||||
}
|
||||
|
||||
func saveToFile(url string) cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, params, err := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filename := params["filename"]
|
||||
if filename == "" {
|
||||
return errors.New("Missing filename in response")
|
||||
}
|
||||
bs, err := responseToBArray(response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = f.Write(bs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("Wrote results to", filename)
|
||||
func indexDumpOutput(url string, apiClientFactory *apiClientFactory) error {
|
||||
client, err := apiClientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if errors.Is(err, errNotFound) {
|
||||
return errors.New("not found (folder/file not in database)")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return prettyPrintResponse(response)
|
||||
}
|
||||
|
||||
func saveToFile(url string, apiClientFactory *apiClientFactory) error {
|
||||
client, err := apiClientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, params, err := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filename := params["filename"]
|
||||
if filename == "" {
|
||||
return errors.New("Missing filename in response")
|
||||
}
|
||||
bs, err := responseToBArray(response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = f.Write(bs)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("Wrote results to", filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getConfig(c APIClient) (config.Configuration, error) {
|
||||
@@ -111,19 +114,6 @@ func getConfig(c APIClient) (config.Configuration, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
|
||||
return func(ctx *cli.Context) error {
|
||||
if ctx.NArg() != n {
|
||||
plural := ""
|
||||
if n != 1 {
|
||||
plural = "s"
|
||||
}
|
||||
return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
|
||||
}
|
||||
return actionFunc(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func prettyPrintJSON(data interface{}) error {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
@@ -159,7 +149,3 @@ func nulString(bs []byte) string {
|
||||
func normalizePath(path string) string {
|
||||
return filepath.ToSlash(filepath.Clean(path))
|
||||
}
|
||||
|
||||
func getClientFactory(c *cli.Context) *apiClientFactory {
|
||||
return c.App.Metadata["clientFactory"].(*apiClientFactory)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -16,8 +17,6 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+18
-35
@@ -22,7 +22,6 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -31,7 +30,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/thejerf/suture/v4"
|
||||
"github.com/willabides/kongplete"
|
||||
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cli"
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
|
||||
@@ -88,17 +89,9 @@ above.
|
||||
STTRACE A comma separated string of facilities to trace. The valid
|
||||
facility strings are listed below.
|
||||
|
||||
STDEADLOCKTIMEOUT Used for debugging internal deadlocks; sets debug
|
||||
sensitivity. Use only under direction of a developer.
|
||||
|
||||
STLOCKTHRESHOLD Used for debugging internal deadlocks; sets debug
|
||||
sensitivity. Use only under direction of a developer.
|
||||
|
||||
STHASHING Select the SHA256 hashing package to use. Possible values
|
||||
are "standard" for the Go standard library implementation,
|
||||
"minio" for the github.com/minio/sha256-simd implementation,
|
||||
and blank (the default) for auto detection.
|
||||
|
||||
STVERSIONEXTRA Add extra information to the version string in logs and the
|
||||
version line in the GUI. Can be set to the name of a wrapper
|
||||
or tool controlling syncthing to communicate this to the end
|
||||
@@ -136,10 +129,11 @@ var (
|
||||
// commands and options here are top level commands to syncthing.
|
||||
// Cli is just a placeholder for the help text (see main).
|
||||
var entrypoint struct {
|
||||
Serve serveOptions `cmd:"" help:"Run Syncthing"`
|
||||
Generate generate.CLI `cmd:"" help:"Generate key and config, then exit"`
|
||||
Decrypt decrypt.CLI `cmd:"" help:"Decrypt or verify an encrypted folder"`
|
||||
Cli struct{} `cmd:"" help:"Command line interface for Syncthing"`
|
||||
Serve serveOptions `cmd:"" help:"Run Syncthing"`
|
||||
Generate generate.CLI `cmd:"" help:"Generate key and config, then exit"`
|
||||
Decrypt decrypt.CLI `cmd:"" help:"Decrypt or verify an encrypted folder"`
|
||||
Cli cli.CLI `cmd:"" help:"Command line interface for Syncthing"`
|
||||
InstallCompletions kongplete.InstallCompletions `cmd:"" help:"Print commands to install shell completions"`
|
||||
}
|
||||
|
||||
// serveOptions are the options for the `syncthing serve` command.
|
||||
@@ -173,7 +167,6 @@ type serveOptions struct {
|
||||
// Debug options below
|
||||
DebugDBIndirectGCInterval time.Duration `env:"STGCINDIRECTEVERY" help:"Database indirection GC interval"`
|
||||
DebugDBRecheckInterval time.Duration `env:"STRECHECKDBEVERY" help:"Database metadata recalculation interval"`
|
||||
DebugDeadlockTimeout int `placeholder:"SECONDS" env:"STDEADLOCKTIMEOUT" help:"Used for debugging internal deadlocks"`
|
||||
DebugGUIAssetsDir string `placeholder:"PATH" help:"Directory to load GUI assets from" env:"STGUIASSETS"`
|
||||
DebugPerfStats bool `env:"STPERFSTATS" help:"Write running performance statistics to perf-$pid.csv (Unix only)"`
|
||||
DebugProfileBlock bool `env:"STBLOCKPROFILE" help:"Write block profiles to block-$pid-$timestamp.pprof every 20 seconds"`
|
||||
@@ -213,17 +206,6 @@ func defaultVars() kong.Vars {
|
||||
}
|
||||
|
||||
func main() {
|
||||
// The "cli" subcommand uses a different command line parser, and e.g. help
|
||||
// gets mangled when integrating it as a subcommand -> detect it here at the
|
||||
// beginning.
|
||||
if len(os.Args) > 1 && os.Args[1] == "cli" {
|
||||
if err := cli.Run(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// First some massaging of the raw command line to fit the new model.
|
||||
// Basically this means adding the default command at the front, and
|
||||
// converting -options to --options.
|
||||
@@ -249,11 +231,20 @@ func main() {
|
||||
|
||||
// Create a parser with an overridden help function to print our extra
|
||||
// help info.
|
||||
parser, err := kong.New(&entrypoint, kong.Help(helpHandler), defaultVars())
|
||||
parser, err := kong.New(
|
||||
&entrypoint,
|
||||
kong.ConfigureHelp(kong.HelpOptions{
|
||||
NoExpandSubcommands: true,
|
||||
Compact: true,
|
||||
}),
|
||||
kong.Help(helpHandler),
|
||||
defaultVars(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
kongplete.Complete(parser)
|
||||
ctx, err := parser.Parse(args)
|
||||
parser.FatalIfErrorf(err)
|
||||
ctx.BindTo(l, (*logger.Logger)(nil)) // main logger available to subcommands
|
||||
@@ -626,7 +617,6 @@ func syncthingMain(options serveOptions) {
|
||||
}
|
||||
|
||||
appOpts := syncthing.Options{
|
||||
DeadlockTimeoutS: options.DebugDeadlockTimeout,
|
||||
NoUpgrade: options.NoUpgrade,
|
||||
ProfilerAddr: options.DebugProfilerListen,
|
||||
ResetDeltaIdxs: options.DebugResetDeltaIdxs,
|
||||
@@ -637,10 +627,6 @@ func syncthingMain(options serveOptions) {
|
||||
if options.Audit {
|
||||
appOpts.AuditWriter = auditWriter(options.AuditFile)
|
||||
}
|
||||
if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {
|
||||
secs, _ := strconv.Atoi(t)
|
||||
appOpts.DeadlockTimeoutS = secs
|
||||
}
|
||||
if dur, err := time.ParseDuration(os.Getenv("STRECHECKDBEVERY")); err == nil {
|
||||
appOpts.DBRecheckInterval = dur
|
||||
}
|
||||
@@ -660,10 +646,6 @@ func syncthingMain(options serveOptions) {
|
||||
|
||||
setupSignalHandling(app)
|
||||
|
||||
if os.Getenv("GOMAXPROCS") == "" {
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
}
|
||||
|
||||
if options.DebugProfileCPU {
|
||||
f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
|
||||
if err != nil {
|
||||
@@ -874,6 +856,7 @@ func cleanConfigDirectory() {
|
||||
"backup-of-v0.8": 30 * 24 * time.Hour, // these neither
|
||||
"tmp-index-sorter.*": time.Minute, // these should never exist on startup
|
||||
"support-bundle-*": 30 * 24 * time.Hour, // keep old support bundle zip or folder for a month
|
||||
"csrftokens.txt": 0, // deprecated, remove immediately
|
||||
}
|
||||
|
||||
for pat, dur := range patterns {
|
||||
|
||||
@@ -241,22 +241,6 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
|
||||
if panicFd == nil {
|
||||
dst.Write([]byte(line))
|
||||
|
||||
if strings.Contains(line, "SIGILL") {
|
||||
l.Warnln(`
|
||||
*******************************************************************************
|
||||
* Crash due to illegal instruction detected. This is most likely due to a CPU *
|
||||
* incompatibility with the high performance hashing package. Switching to the *
|
||||
* standard hashing package instead. Please report this issue at: *
|
||||
* *
|
||||
* https://github.com/syncthing/syncthing/issues *
|
||||
* *
|
||||
* Include the details of your CPU. *
|
||||
*******************************************************************************
|
||||
`)
|
||||
os.Setenv("STHASHING", "standard")
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
|
||||
panicFd, err = os.Create(locations.GetTimestamped(locations.PanicLog))
|
||||
if err != nil {
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
- runtime: go1.21
|
||||
requirements:
|
||||
# See https://en.wikipedia.org/wiki/MacOS_version_history#Releases
|
||||
#
|
||||
# macOS 10.15 (Catalina) per https://go.dev/doc/go1.22#darwin
|
||||
darwin: "19"
|
||||
# Per https://go.dev/doc/go1.23#linux
|
||||
linux: "2.6.32"
|
||||
# Windows 10's initial release was 10.0.10240.16405, per
|
||||
# https://learn.microsoft.com/en-us/windows/release-health/release-information
|
||||
# and Windows 11's initial release was 10.0.22000.194 per
|
||||
# https://learn.microsoft.com/en-us/windows/release-health/windows11-release-information
|
||||
#
|
||||
# Windows 10/Windows Server 2016 per https://go.dev/doc/go1.21#windows
|
||||
windows: "10.0"
|
||||
|
||||
- runtime: go1.22
|
||||
requirements:
|
||||
darwin: "19"
|
||||
linux: "2.6.32"
|
||||
windows: "10.0"
|
||||
|
||||
- runtime: go1.23
|
||||
requirements:
|
||||
# macOS 11 (Big Sur) per https://tip.golang.org/doc/go1.23#darwin
|
||||
darwin: "20"
|
||||
linux: "2.6.32"
|
||||
windows: "10.0"
|
||||
@@ -2,7 +2,7 @@
|
||||
Name=Syncthing Web UI
|
||||
GenericName=File synchronization UI
|
||||
Comment=Opens Syncthing's Web UI in the default browser (Syncthing must already be started).
|
||||
Exec=/usr/bin/syncthing -browser-only
|
||||
Exec=/usr/bin/syncthing --browser-only
|
||||
Icon=syncthing
|
||||
Terminal=false
|
||||
Type=Application
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Increase maximum socket buffer sizes to 2.5MiB for QUIC connections
|
||||
# Increase maximum socket buffer sizes to 7MiB for QUIC connections
|
||||
# see https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes
|
||||
net.core.rmem_max = 2621440
|
||||
net.core.wmem_max = 2621440
|
||||
net.core.rmem_max = 7340032
|
||||
net.core.wmem_max = 7340032
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
[Unit]
|
||||
Description=Restart Syncthing after resume
|
||||
Documentation=man:syncthing(1)
|
||||
After=sleep.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=-/usr/bin/pkill -HUP -x syncthing
|
||||
|
||||
[Install]
|
||||
WantedBy=sleep.target
|
||||
@@ -1,81 +1,101 @@
|
||||
module github.com/syncthing/syncthing
|
||||
|
||||
go 1.20
|
||||
go 1.22.0
|
||||
|
||||
require (
|
||||
github.com/AudriusButkevicius/recli v0.0.7-0.20220911121932-d000ce8fbf0f
|
||||
github.com/alecthomas/kong v0.8.1
|
||||
github.com/calmh/incontainer v0.0.0-20221224152218-b3e71b103d7a
|
||||
github.com/alecthomas/kong v1.2.1
|
||||
github.com/aws/aws-sdk-go v1.55.5
|
||||
github.com/calmh/incontainer v1.0.0
|
||||
github.com/calmh/xdr v1.1.0
|
||||
github.com/ccding/go-stun v0.1.4
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/ccding/go-stun v0.1.5
|
||||
github.com/chmduquesne/rollinghash v4.0.0+incompatible
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1
|
||||
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
|
||||
github.com/getsentry/raven-go v0.2.0
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-ldap/ldap/v3 v3.4.8
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/greatroar/blobloom v0.7.2
|
||||
github.com/greatroar/blobloom v0.8.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/jackpal/gateway v1.0.10
|
||||
github.com/jackpal/gateway v1.0.15
|
||||
github.com/jackpal/go-nat-pmp v1.0.2
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/maruel/panicparse/v2 v2.3.1
|
||||
github.com/maxbrunsfeld/counterfeiter/v6 v6.5.0
|
||||
github.com/minio/sha256-simd v1.0.1
|
||||
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1
|
||||
github.com/maxmind/geoipupdate/v6 v6.1.0
|
||||
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
|
||||
github.com/oschwald/geoip2-golang v1.9.0
|
||||
github.com/pierrec/lz4/v4 v4.1.18
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/prometheus/client_golang v1.17.0
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.12.0 // indirect
|
||||
github.com/quic-go/quic-go v0.39.1
|
||||
github.com/oschwald/geoip2-golang v1.11.0
|
||||
github.com/pierrec/lz4/v4 v4.1.21
|
||||
github.com/prometheus/client_golang v1.20.4
|
||||
github.com/puzpuzpuz/xsync/v3 v3.4.0
|
||||
github.com/quic-go/quic-go v0.47.0
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
|
||||
github.com/sasha-s/go-deadlock v0.3.1
|
||||
github.com/shirou/gopsutil/v3 v3.23.9
|
||||
github.com/shirou/gopsutil/v4 v4.24.8
|
||||
github.com/syncthing/notify v0.0.0-20210616190510-c6b7342338d2
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d
|
||||
github.com/thejerf/suture/v4 v4.0.2
|
||||
github.com/urfave/cli v1.22.14
|
||||
github.com/thejerf/suture/v4 v4.0.5
|
||||
github.com/urfave/cli v1.22.15
|
||||
github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0
|
||||
golang.org/x/crypto v0.14.0
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
|
||||
golang.org/x/mod v0.13.0 // indirect
|
||||
golang.org/x/net v0.17.0
|
||||
golang.org/x/sys v0.13.0
|
||||
golang.org/x/text v0.13.0
|
||||
golang.org/x/time v0.3.0
|
||||
golang.org/x/tools v0.14.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
github.com/willabides/kongplete v0.4.0
|
||||
go.uber.org/automaxprocs v1.5.3
|
||||
golang.org/x/crypto v0.27.0
|
||||
golang.org/x/net v0.29.0
|
||||
golang.org/x/sys v0.25.0
|
||||
golang.org/x/text v0.18.0
|
||||
golang.org/x/time v0.6.0
|
||||
golang.org/x/tools v0.25.0
|
||||
google.golang.org/protobuf v1.34.2
|
||||
sigs.k8s.io/yaml v1.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
|
||||
github.com/google/uuid v1.3.1 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
|
||||
github.com/oschwald/maxminddb-golang v1.12.0 // indirect
|
||||
github.com/petermattis/goid v0.0.0-20230904192822-1876fd5063bc // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
|
||||
github.com/prometheus/client_model v0.5.0 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.3.4 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/gofrs/flock v0.12.1 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nxadm/tail v1.4.11 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.20.2 // indirect
|
||||
github.com/oschwald/maxminddb-golang v1.13.1 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/posener/complete v1.2.3 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.59.1 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
go.uber.org/mock v0.3.0 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/stretchr/testify v1.9.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.14 // indirect
|
||||
github.com/tklauser/numcpus v0.8.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.uber.org/mock v0.4.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
|
||||
golang.org/x/mod v0.21.0 // indirect
|
||||
golang.org/x/sync v0.8.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
// https://github.com/gobwas/glob/pull/55
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user