Compare commits

..

428 Commits

Author SHA1 Message Date
Cedric Staniewski 343d003f7a conf: Update version to 0.14
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/206
2016-07-17 12:34:58 +00:00
Frank Sachsenheim 5738ea240d users/config: Correct username element -> user
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/205
2016-07-17 06:51:32 +00:00
Jakob Borg 57aa23d29c index: discosrv & relaysrv were renamed 2016-07-05 15:00:15 +02:00
Jakob Borg 22fea7df4e specs: Local discovery version was bumped 2016-07-05 14:58:24 +02:00
Jakob Borg 77f8d085e5 deploy: Script must have bash header 2016-07-05 14:56:12 +02:00
aviau aae1a525f1 users: create stdiscosrv manpage
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/203
2016-07-04 14:28:49 +00:00
Jakob Borg 76236e1797 specs: Document instance_id in local discovery 2016-07-04 15:23:18 +02:00
Jakob Borg 91818c9890 dev/issues: Clarify labels and add milestone info
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/199
2016-07-04 13:18:43 +00:00
aviau a1ac0aa446 users: create strelaysrv manpage
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/202
2016-07-04 13:17:20 +00:00
Jakob Borg a9b3fef697 specs: Update specs for protocol buffer formats
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/190
2016-07-04 11:56:58 +00:00
Jakob Borg 3b76ff6329 users/config: Empty is also an option for folder labels 2016-06-30 13:31:09 +02:00
Cedric Staniewski 6307a1b1db users/config: Folder label are not unique
See https://github.com/syncthing/syncthing/issues/3367

GitHub-Pull-Request: https://github.com/syncthing/docs/pull/200
2016-06-30 11:29:55 +00:00
aviau f1808d4aa2 typo: interchangably -> interchangeably
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/198
2016-06-29 13:41:59 +00:00
Peter Dave Hello 2a48e397da optimize png images using zopflipng
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/197
2016-06-29 13:40:51 +00:00
Cedric Staniewski 60c7245e44 Update syncthing major version
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/195
2016-06-27 20:55:33 +00:00
Cedric Staniewski 43bae4005c rest/system/ping: Remove obsolete note
A GET request to the /rest/system/ping endpoint also requires an API key or CSRF token.

GitHub-Pull-Request: https://github.com/syncthing/docs/pull/194
2016-06-27 20:47:00 +00:00
Alexander Graf edf2498b76 rest: Document override call (fixes #39)
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/193
2016-06-26 09:20:24 +00:00
twomice 4a4566570e spellcheck "remove" => "remote"
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/192
2016-06-17 08:52:08 +00:00
Jerry Jacobs 44553d2048 Mention and deprecate user-contrib OS X application (fixes #188)
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/189
2016-06-15 16:38:23 +00:00
Jakob Borg 8241da8264 Note -gui-apikey as deprecated (fixes #186) 2016-06-08 14:16:10 +02:00
Zillode cbdc571b8c rest/system-status-get.rst: added new fields
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/187
2016-06-07 05:50:21 +00:00
imsodin c6b994a815 Update /rest/db/ignores for 0.13 changes
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/174
2016-06-03 13:09:36 +00:00
uglygus 8748339201 (?d) example
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/182
2016-06-03 11:07:17 +00:00
Jakob Borg 899b9fc6c7 dev/mergebot: Emphasize the importance of getting the "Skip-check: authors" part right
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/185
2016-06-03 10:32:08 +00:00
Majed Abdulaziz 88f88a948d Document lastScan field in /rest/stats/folder API endpoint
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/183
2016-06-03 09:58:38 +00:00
Jakob Borg 936cef6ddd dev/mergebot: Describe LGTM flow
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/184
LGTM: AudriusButkevicius, rumpelsepp
2016-06-03 09:37:49 +00:00
Zillode 3b286b9362 docs/rest: updated connection types
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/176
2016-05-31 06:23:43 +00:00
Zillode 0f5701f175 events/deviceconnected.rst: Added new fields
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/180
2016-05-31 06:23:08 +00:00
Zillode 976870ec0e events/devicepaused.rst: document DevicePaused
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/179
2016-05-31 06:22:54 +00:00
Zillode 6ff9af7872 events/deviceconnected.rst: Added new fields
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/178
2016-05-31 06:22:34 +00:00
Zillode 2b1bcd5bfc events/folderrejected.rst: Added folderLabel field
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/181
2016-05-31 06:22:11 +00:00
Zillode 34b34c7b77 rest/system-config-get: Added exemplary data
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/177
2016-05-30 21:12:12 +00:00
Arthur Lutz 47a88349ff users/contrib: Add Saltstack contribution
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/175
2016-05-30 08:09:50 +00:00
Scott Hansen 699786cee1 users/discosrv: Update for /v2/ path component
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/173
2016-05-26 14:08:09 +00:00
Audrius Butkevicius c4a1600950 Update discovery protocol specs for v0.13
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/171
2016-05-21 02:08:46 +00:00
Audrius Butkevicius ddd8042732 Document need pagination (fixes #81)
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/172
2016-05-21 01:35:37 +00:00
Gavrilov Aleksej 28f626f6d4 users/contrib: Add ansible
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/170
2016-05-19 00:01:33 +00:00
Jakob Borg 0bef7754bb users/config: Update config article for current config format (v13)
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/169
2016-05-17 07:10:55 +00:00
Benjamin Masters fd92efc2f6 users/syncing: Minor spelling correction
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/168
2016-05-12 05:58:39 +00:00
Jakob Borg 076ac1aa14 users/faq: Mention scanning and syncthing-inotify in relation to CPU usage
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/167
2016-05-11 11:44:45 +00:00
Lars K.W. Gohlke 0231d12897 users/config: Document folder labels
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/162
2016-05-11 10:48:38 +00:00
Lars K.W. Gohlke c5796b993f readme: Getting started building docs locally
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/164
2016-05-11 10:47:49 +00:00
Jakob Borg 5ac7ff99da dev: Document using the merge bot (#166)
* dev: Document using the merge bot
2016-05-08 13:53:15 +01:00
ka7 7eacdaa11b spelling-fixes 2016-05-08 10:20:20 +01:00
Lars K.W. Gohlke bdafe9f5d4 Updated documentation for foldertype 2016-05-08 07:20:16 +02:00
Audrius Butkevicius de1d3792c1 Document temporary indexes
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/161
2016-05-01 11:21:12 +00:00
Audrius Butkevicius 69cdae0149 Document (?d) prefix
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/160
2016-04-30 18:51:14 +00:00
Audrius Butkevicius 8d245533a7 specs/bep-v1: Add Hello message, changed ClusterConfig
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/159
2016-04-30 12:44:20 +00:00
Jakob Borg 0c91ae2743 users/faq: Mention GOMAXPROCS in question about CPU usage 2016-04-29 16:49:16 +02:00
Jakob Borg 32a16746e5 users/faq: Link to firewall setup from relay question 2016-04-29 16:40:42 +02:00
Jakob Borg 630ec0cd28 users/faq: Add Why is the sync so slow? 2016-04-29 16:33:32 +02:00
Jakob Borg 09150a77ce users/faq: Update and clarify some FAQ answers 2016-04-29 15:56:18 +02:00
Jakob Borg f5ec46e66d users/syncing: Add article describing the underlying mechanisms 2016-04-29 15:34:25 +02:00
Jakob Borg 0e21c6fc0d dev/contributing: Describe new commit message and merge process 2016-04-29 12:11:58 +02:00
Jakob Borg c648d3bceb index: Logically group articles, undo renaming discosrv 2016-04-29 12:05:17 +02:00
Jakob Borg 7c8c38e0be users: Homogenize titling for Custom {Discovery, Relay, Upgrade} Server 2016-04-29 11:54:46 +02:00
Jakob Borg 43699de504 users/firewall: Link to proxy setup 2016-04-29 11:51:13 +02:00
Jakob Borg c50ade832b users/proxying: Add article describing SOCKS proxies 2016-04-29 11:34:57 +02:00
Scott Hansen 5ebce76524 users/discosrv: Document running discosrv behind nginx
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/149
2016-04-29 09:13:57 +00:00
Jakob Borg 7ad9ba5a70 users/autostart: Simplify brew instructions 2016-04-29 11:10:45 +02:00
Edd Barrett c31350e80d Link to OpenBSD port. (#157) 2016-04-21 21:08:21 +01:00
Iliyan ab0a7268f6 Deleted my app
App's not finished

GitHub-Pull-Request: https://github.com/syncthing/docs/pull/155
2016-04-18 20:20:26 +00:00
Iliyan 1a8dbc1058 Update contrib.rst
Added my "SyncthingLauncher" app

GitHub-Pull-Request: https://github.com/syncthing/docs/pull/154
2016-04-18 19:56:47 +00:00
Louis Sautier f6b4c34187 Add a link to the official Gentoo package
GitHub-Pull-Request: https://github.com/syncthing/docs/pull/151
2016-04-13 11:10:17 +00:00
Jakob Borg b8bf3411cc Merge pull request #150 from syncthing/flagdir
Add directory flag to Index message and correct bit assignments
2016-04-10 11:47:37 +01:00
djtm cc5746632a docs: contrib.rst/docker: add syncthing-docker-scratch 2016-04-06 15:48:38 +02:00
ProactiveServices 36d399c3cb Windows service/NSSM improvements
Cleaned up style, grammar and added steps to enable logging and support
Syncthing exit/restart/upgrade actions. Additional note regarding
security concerns. Removed superfluous final paragraph. Somewhat more
modern and tidier screenshot, renamed it to make it less generic
2016-03-30 11:18:41 +00:00
Laurent Etiemble f27f6dff3c Indent URL under unofficial CentOS packages 2016-03-30 11:17:39 +00:00
Adam Piggott cb62b52a30 Windows does not support escaping (ref #2822) 2016-03-30 11:15:54 +00:00
Martin Freund dec9d4c26d Added Windows example for external file versioning
The example uses native batch scripting and is based on the already provided Unix example.
2016-03-30 11:14:44 +00:00
Audrius Butkevicius bd3ce4a771 Merge pull request #145 from sodacode/patch-1
Add a Windows installer
2016-03-30 01:07:41 +01:00
Audrius Butkevicius 28da8242b2 Merge pull request #138 from lkwg82/master
updated spec ...
2016-03-28 15:58:27 +01:00
Dakota abf0124e28 Update contrib.rst
Not sure why nobody has done this yet, maybe I'm missing something?
Wrote a small script for a Windows installer.
2016-03-27 10:34:11 -04:00
Audrius Butkevicius 6b2cddc6d6 Merge pull request #144 from vhbit/patch-1
Fix docs for join session flow
2016-03-26 13:37:09 +00:00
Valerii Hiora 186b96be64 Fix docs for join session flow
`JoinSessionRequest` sounds more appropriate and actually works.
2016-03-26 09:05:32 +02:00
Audrius Butkevicius 518dffae28 Merge pull request #143 from lkwg82/patch-1
Update syncthing.rst
2016-03-25 22:25:21 +00:00
Lars K.W. Gohlke 7fd012b875 Update syncthing.rst
fixes #102
2016-03-25 23:10:04 +01:00
Lars K.W. Gohlke e36eacb046 Update bep-v1.rst 2016-03-25 22:56:12 +01:00
Audrius Butkevicius c06e202121 Merge pull request #142 from lkwg82/patch-3
Update syncthing.rst
2016-03-25 18:03:00 +00:00
Lars K.W. Gohlke 0ddcafa6c7 Update syncthing.rst
fixes #15
2016-03-25 18:53:08 +01:00
Audrius Butkevicius 67b1a291c6 Merge pull request #140 from lkwg82/patch-1
Update syncthing.rst
2016-03-25 17:41:11 +00:00
Lars K.W. Gohlke 65c2a85971 Update syncthing.rst
added 

-browser-only
-paused

fixes #95
2016-03-25 18:39:37 +01:00
Audrius Butkevicius 82fb189d1d Merge pull request #103 from tuathail/patch-1
autostart.rst Mac OS X Updates
2016-03-25 17:34:30 +00:00
Lars K.W. Gohlke c260c16cf5 updated spec according to syncthing/syncthing/pull/2668 2016-03-25 18:11:50 +01:00
Audrius Butkevicius 9adb78c4a6 Merge pull request #137 from syncthing/ProactiveServices-docs-PR2880
Document use of tcp4 and tcp6
2016-03-25 13:23:05 +00:00
Adam Piggott 7c5b00deb1 Add GUI Element address examples 2016-03-25 10:15:42 +00:00
Adam Piggott 609a95318e Document tcp4:// tcp6:// prefixes
Documenting https://github.com/syncthing/syncthing/pull/2880
2016-03-25 09:46:47 +00:00
Jakob Borg c0820c8a0c Delete followsymlinks.rst
For now, as it causes endless confusion.
2016-03-24 16:46:50 +01:00
Jakob Borg 279fd90d57 Update release process 2016-03-23 10:21:21 +01:00
Jakob Borg be75b6c28a Merge pull request #136 from kluppy/patch-4
Clarify remote states when ignoreDelete selected
2016-03-21 12:33:09 +01:00
Scott Klupfel 1941ddf320 Clarify remote states when ignoreDelete selected
As discussed on the forum: https://forum.syncthing.net/t/stuck-syncing-at-90/6991
2016-03-21 19:29:31 +10:00
Jakob Borg 616a1d5d3c Merge pull request #135 from wweich/patch-1
Correct unit for rate limit (fixes syncthing/syncthing#2844)
2016-03-17 07:54:18 +01:00
Wulf Weich 0cd687b714 Correct unit for rate limit (fixes syncthing/syncthing#2844)
Correct unit for `maxSendKbps` and `maxRecvKbps` from kibibits to kibibytes.
2016-03-17 06:13:05 +01:00
Jakob Borg c2a514250f Infrastructure updates 2016-03-14 08:46:33 +01:00
Audrius Butkevicius eb681e3252 Merge pull request #133 from syncthing/revert-132-patch-1
Revert "adjust path to theme path"
2016-03-09 13:04:52 +00:00
Audrius Butkevicius 0bdfaabceb Revert "adjust path to theme path" 2016-03-09 13:04:42 +00:00
Audrius Butkevicius 218001c902 Merge pull request #132 from lkwg82/patch-1
adjust path to theme path
2016-03-09 00:37:28 +00:00
Lars K.W. Gohlke d2a7faed49 adjust path to theme path 2016-03-08 22:39:00 +01:00
Jakob Borg 7079b3f0c9 Add note about compiler versions and vendoring 2016-03-08 08:36:05 +01:00
Jakob Borg b620e636cd Merge pull request #131 from syncthing/revert-130-patch-2
Revert "updated compiler version"
2016-03-08 08:29:00 +01:00
Jakob Borg 4883e834e9 Revert "updated compiler version" 2016-03-08 08:28:50 +01:00
Audrius Butkevicius abecfcd1b1 Merge pull request #130 from lkwg82/patch-2
updated compiler version
2016-03-07 22:14:06 +00:00
Lars K.W. Gohlke 0a251931cb updated compiler version
see compiler change to version 1.6 in https://github.com/syncthing/syncthing/releases/tag/v0.12.20
2016-03-07 22:55:34 +01:00
Audrius Butkevicius 8ee999962e Merge pull request #128 from bugith/patch-1
Update autostart.rst
2016-03-04 20:29:36 +00:00
bugith 7a74caea05 Update autostart.rst
AFAIK the Windows startup folder is parsed on user login, not before.
2016-03-04 21:17:30 +01:00
Jakob Borg 72ab46b379 Merge pull request #127 from letiemble/patch-2
Companion PR for the syncthing/syncthing#2791
2016-02-28 20:06:18 +07:00
Laurent Etiemble cef5fd6282 Companion PR for the syncthing/syncthing#2791
Modification of the specifications to match this [pull request](https://github.com/syncthing/syncthing/pull/2791)
2016-02-28 12:21:41 +01:00
Stefan Tatschner 7c4f1597e2 Add semicolons to XDR code
This ensures that the documented XDR code compiles using
rpcgen.
2016-02-23 12:39:43 +01:00
Audrius Butkevicius 2a4b5025a5 Merge pull request #125 from dinosore/patch-1
spelling
2016-02-20 19:02:55 +00:00
dinosore b108d35f5d spelling 2016-02-20 18:52:56 +00:00
Stefan Tatschner 648e3b77d7 Merge pull request #124 from MingweiSamuel/patch-1
documentation replace "|" with "+" for box
2016-02-19 08:25:25 +01:00
Mingwei Samuel 552d4e0a32 Update bep-v1.rst 2016-02-18 21:57:19 -08:00
Audrius Butkevicius 5873e3d3f7 Merge pull request #119 from syncthing/Proactive-Services-faq-localsync
Add FAQ for syncing folders on same system
2016-02-09 15:13:31 +00:00
ProactiveServices d159b1ec9a Fixed underline 2016-02-09 14:38:09 +00:00
ProactiveServices dec3abaaf7 Add FAQ for syncing between folders on same system 2016-02-09 13:22:41 +00:00
Audrius Butkevicius bb7a50ea03 Merge pull request #118 from syncthing/Proactive-Services-pathdocs
Document -paths
2016-02-08 22:28:28 +00:00
ProactiveServices aabc64b257 Document -paths
Document -paths parameter (fixes #116)
2016-02-08 22:15:50 +00:00
Audrius Butkevicius 2adef7ebfb Merge pull request #117 from mlazarov/patch-1
Adding CentOS package
2016-02-08 19:53:29 +00:00
Martin Lazarov 5abe956ebe Update contrib.rst 2016-02-08 19:47:26 +02:00
Martin Lazarov b92361e45e Adding CentoOS package 2016-02-08 19:39:06 +02:00
Jakob Borg 708ea1f84a Merge pull request #115 from syncthing/ProactiveServices-device-docs-prefixes
Fix device config re: prefixes
2016-02-08 02:05:37 -05:00
Adam Piggott 544b7935fd Update config.rst
Re-added line that was unintentionally removed. Further clarify detail before example section.
2016-02-08 03:43:21 +00:00
Adam Piggott b0c16a631c Fix device config re: prefixes
Added tcp:// prefixes in docs and examples, added host name detail, fixes to IPv6 examples.
2016-02-08 02:16:39 +00:00
Audrius Butkevicius a08d09fca3 Merge pull request #113 from Morphy99/patch-1
Update autostart.rst
2016-02-01 15:21:45 +00:00
Morphy99 51c935d6e1 Update autostart.rst
Added configuration directory location under windows service method as this differs to other methods.
2016-02-01 15:16:15 +00:00
Jakob Borg 845bc0363c Merge pull request #111 from camsteffen/master
add -no-console -no-browser options to Task Scheduler
2016-01-30 09:49:38 +01:00
Jakob Borg 4e9c5102bb Remove docker instructions 2016-01-30 09:14:40 +01:00
Cameron Steffen 59423ddf86 add -no-console -no-browser options to Task Scheduler 2016-01-29 23:53:01 -06:00
simplypeachy 3b25e2ed2f Fix error in unprivileged user section
Thanks to emdete on IRC for pointing out the error.
2016-01-24 16:50:58 +00:00
Jakob Borg 4a792c572e Merge pull request #108 from simplypeachy/patch-1
Update config.rst to include ignoreDevice element
2016-01-19 15:16:15 -08:00
simplypeachy f02eadc886 Update config.rst to include ignoreDevice element 2016-01-19 23:07:24 +00:00
Jakob Borg 1a2f99efe8 Correct internal->lib in dev intro 2016-01-19 14:14:28 -08:00
Stefan Tatschner 3a5ba9f822 Merge pull request #107 from ojongerius/patch-2
Remove links which are not visible to non team members.
2016-01-17 14:23:09 +01:00
Otto Jongerius 6f97972956 Remove links
They are either dead or not visible to non team members.
2016-01-17 23:28:14 +11:00
Audrius Butkevicius 74eb5e2fec Merge pull request #106 from ojongerius/patch-1
Correct example
2016-01-17 00:59:46 +00:00
Otto Jongerius a0e1f38fe2 Correct example 2016-01-17 11:51:50 +11:00
Jakob Borg 4d1c1143c2 Retract hash algo proposal for the moment 2016-01-12 12:01:07 +01:00
Jakob Borg e59d845978 Correct statechanged event states 2016-01-12 11:59:39 +01:00
tuathail 7c3ee6f441 autostart.rst Mac OS X Updates
Some clarifications to the Mac OS X install including:
Spelled out the home directory (/home/<username>/bin) in step one. Some people might gloss over it and just put it in /bin otherwise.
4 replacements of username are required, not 2.
Replaced the ~ with /Users/<username> for consistency, but also if the user switched to root user to avoid it being copied to /var/root/Library.. instead of /Users/<username>/Library...
2016-01-11 20:52:00 +00:00
Audrius Butkevicius f16ab3bdcf Merge pull request #101 from kluppy/master
Clarify Direct Connection Prefered Over Relay
2016-01-07 11:11:40 +00:00
Scott Klupfel 1694117003 Clarify Direct Connection Prefered Over Relay 2016-01-07 21:02:11 +10:00
Scott Klupfel e3eaa4110e Merge pull request #1 from syncthing/master
Bringing Up to Date
2016-01-07 20:51:22 +10:00
Audrius Butkevicius 8ad18f22b0 Merge pull request #99 from kluppy/patch-2
Add description for progressUpdateIntervalS
2016-01-06 08:37:48 +00:00
Scott Klupfel 4175c3a93c Reverted kibibits change
Sorry, my bad.
2016-01-06 15:15:33 +10:00
Scott Klupfel dbd2a292e4 Add description for progressUpdateIntervalS & tidy 2016-01-06 15:08:34 +10:00
Jakob Borg bb87d0f18c Merge pull request #78 from syncthing/hashalgos
Add hash algos as flags on the folder in cluster config
2016-01-01 21:48:37 +01:00
Jakob Borg 2eea0f9799 Add hash algos as flags on the folder in cluster config 2016-01-01 21:47:26 +01:00
Jakob Borg be0715676c Merge pull request #98 from jimmyjones2/patch-1
Support space in directory
2015-12-30 15:42:32 +01:00
Jimmy Jones 0a72691cc4 Support space in directory
Currently the script breaks if versionspath or filepath contain a space, this fixes it.
2015-12-30 14:34:12 +00:00
Audrius Butkevicius bf2eb524f4 Merge pull request #97 from mkantor/patch-1
Syncthing is not beta anymore in SynoCommunity.
2015-12-28 02:14:55 +00:00
Matt Kantor dc1b6d3ecc Syncthing is not beta anymore in SynoCommunity.
The Syncthing package is not marked as beta these days, so there is no 
need to change the package manager settings.
2015-12-27 17:57:40 -08:00
Jakob Borg 0cfbceb428 Add other build servers 2015-12-21 12:04:47 +01:00
Jakob Borg 203ff4a7b1 Merge branch 'pr/96'
* pr/96:
  Add note about running syncthing instance on discovery server
2015-12-18 15:32:32 +01:00
Scott Klupfel c7795aa858 Add note about running syncthing instance on discovery server 2015-12-18 15:32:03 +01:00
Jakob Borg 008e5a0ac0 Merge pull request #93 from sbraz/master
Fix typos
2015-12-14 17:21:55 +01:00
Louis Sautier 650a37b74c Fix typos. 2015-12-14 17:15:18 +01:00
Audrius Butkevicius bb83bd87b9 Merge pull request #92 from devste/patch-1
Master mode: rephrasing how data flows.
2015-12-10 23:14:58 +00:00
Stephen 309829692b Typo 2015-12-10 22:20:39 +01:00
Stephen 053208d3c4 Update bep-v1.rst
I had to read this section about three times, before realizing that the wording did actually describe what's in the picture. Changed the wording slightly to highlight in which direction the data does flow.
2015-12-10 22:16:30 +01:00
Audrius Butkevicius 24d93808ad Merge pull request #91 from 4cdn/patch-1
typo
2015-12-06 15:29:10 +00:00
4cdn 8eec93a6ec typo 2015-12-06 10:23:31 -05:00
Audrius Butkevicius 6b11bc7625 Merge pull request #89 from canton7/feature/relay-docs
Add to the Relays page
2015-12-02 19:29:49 +00:00
Antony Male 77c8906217 Add to the Relays page
- Minimum recommended specifications
 - More detail on how to run a relay on port 443
2015-12-02 17:21:22 +00:00
Stefan Tatschner 6d94f321cd Add syncthing-relay(7) 2015-12-01 15:33:19 +01:00
NickPyz e1f51b2973 Update security.rst 2015-12-01 07:45:19 +01:00
Audrius Butkevicius 8946b20902 Merge pull request #90 from canton7/feature/debug-port
Fix port in example in rest/system-debug-post
2015-11-30 08:34:56 +00:00
Antony Male 7c541140d0 Fix port in example in rest/system-debug-post 2015-11-30 08:20:06 +00:00
Jakob Borg f2c244b761 Draft hash algo doc 2015-11-28 21:44:42 +01:00
Jakob Borg 0aeffe7713 Merge pull request #88 from andersonvom/master
Fix formatting and typo on ignoring.rst
2015-11-26 08:06:46 +01:00
Anderson Mesquita ef0f34b0df Fix formatting and typo on ignoring.rst
This fixes a typo at the end of the ignoring.rst file and takes the
opportunity to improve the formatting of the RST file a bit and add some
missing commas.
2015-11-25 20:25:19 -05:00
Audrius Butkevicius 7d8fa4342b Merge pull request #87 from kc1212/patch-1
Missing lib/protocol in source layout
2015-11-24 23:13:24 +00:00
alex2108 06b3cf4cfb Remove options to externally set username and password (fixes #82) 2015-11-24 17:05:14 +01:00
kc1212 61c0db1ecf Missing lib/protocol in source layout
Not sure whether the description is correct.
2015-11-24 00:03:42 +01:00
Jakob Borg 68d4760f9f Merge pull request #85 from Lockszmith/patch-1
Added Windows Task Scheduler solution.
2015-11-23 19:23:05 +01:00
Gal Szkolnik 8489f7bc88 Update autostart.rst
typos fixed, changed brackets to double quotes
2015-11-23 10:44:57 -05:00
Gal Szkolnik 538aaa8ad3 Added Windows Task Scheduler solution.
This solution does not require 3rd Party tools.
2015-11-23 10:26:00 -05:00
Audrius Butkevicius 801133628c Merge pull request #84 from mess110/patch-1
add SUSE Studio appliance link
2015-11-21 17:22:49 +00:00
Cristian Mircea Messel 864961978c add SUSE Studio appliance link 2015-11-21 19:15:39 +02:00
Jakob Borg 1bdec60772 Better example URLs for upgrade JSON 2015-11-17 08:27:28 +01:00
Audrius Butkevicius ad919c9162 Merge pull request #83 from xor-gate/master
Update user contributed Mac OS X app/dmg package
2015-11-16 21:42:02 +00:00
Jerry Jacobs b928f42827 * OSX app new username 2015-11-16 21:29:11 +01:00
Jerry Jacobs 53d472f1a0 * Update syncthing-mac app/dmg 2015-11-16 21:27:56 +01:00
Jakob Borg b88b1bf76c Note that POST /rest/system/discovery no longer exists in v0.12 2015-11-16 19:43:12 +01:00
Jakob Borg 6e4828ff96 The build slaves are offline for the moment 2015-11-16 12:14:40 +01:00
alex2108 6d5cbaaa5f Usage reort update for v0.12 2015-11-14 17:05:36 +01:00
Jakob Borg b4d3ad8eaa The need call is also expensive 2015-11-13 08:17:46 +01:00
Audrius Butkevicius 04b5f362f4 Merge pull request #77 from syncthing/revert-76-patch-1
Revert 1c5690176b
2015-11-09 08:30:22 -05:00
Audrius Butkevicius da64747ee3 Revert 1c5690176b 2015-11-09 08:30:16 -05:00
Audrius Butkevicius 1c5690176b Merge pull request #76 from mlda065/patch-1
Made "https://localhost:8384/" clickable
2015-11-09 07:37:10 -05:00
Matthew Davis ffa1f28f75 Made "https://localhost:8384/" clickable 2015-11-09 20:03:41 +11:00
Jakob Borg 8c3741412f Merge pull request #75 from alex2108/master
/rest/system/connections update for v0.12
2015-11-08 18:12:42 +01:00
Jakob Borg 1136cbe27e Clarify discovery server ID in URL 2015-11-08 14:38:01 +01:00
alex2108 8c76d69c46 /rest/system/connections update for v0.12 2015-11-07 17:46:32 +01:00
Jakob Borg a7248dfdcd Merge pull request #70 from syncthing/discosrv
Procedure for v0.12 discovery server
2015-11-07 14:47:10 +01:00
Audrius Butkevicius cad2102853 Merge pull request #64 from burkemw3/syncing-configuration-files
Add information about syncing config files safely
2015-11-06 23:31:14 -05:00
Matt Burke ccb488790b Add information about syncing config files safely 2015-11-06 15:49:50 -05:00
Audrius Butkevicius fee49d07cf Merge pull request #74 from syncthing/review
Clarify team membership and review policy
2015-11-06 18:17:39 +00:00
Jakob Borg 5244ac68ad Clarify team membership and review policy 2015-11-06 19:05:10 +01:00
Audrius Butkevicius 2eecee3d67 Merge pull request #73 from Stefan-Code/master
added HOME variable to fix #72
2015-11-05 17:39:46 +00:00
Stefan-Code b201abb0f1 added missing slash 2015-11-05 18:32:54 +01:00
Stefan-Code 353eeddfe1 added HOME variable to fix #72 2015-11-05 18:29:16 +01:00
Audrius Butkevicius 009e78bcca Merge pull request #71 from syncthing/firewall
Firewall update for v0.12
2015-11-04 18:09:16 +00:00
Audrius Butkevicius 1b30be1f8d Merge pull request #68 from syncthing/v12
REST and config file updates for v0.12
2015-11-04 17:35:46 +00:00
Jakob Borg b2f00c5994 Fix compiler error/warnings 2015-11-04 17:33:49 +00:00
Jakob Borg 11d0b1a609 Firewall update for v0.12 2015-11-04 17:28:55 +00:00
Jakob Borg d5d0bd946b Update infrastructure for v0.12 2015-11-04 17:22:21 +00:00
Jakob Borg bdad1afa4c We're now documenting v0.12 2015-11-04 17:13:30 +00:00
Jakob Borg ecb7d0bb51 REST changes around logging 2015-11-04 17:06:11 +00:00
Jakob Borg a40ee17ea3 Update config file docs 2015-11-04 17:06:11 +00:00
Jakob Borg b1395a5350 Procedure for v0.12 discovery server 2015-11-04 17:01:44 +00:00
Audrius Butkevicius 6428ece8c6 Merge pull request #69 from syncthing/relaysrv
Procedure to run a relay server
2015-11-04 16:56:40 +00:00
Jakob Borg 929bebb70c Procedure to run a relay server 2015-11-04 16:44:54 +00:00
Audrius Butkevicius e13aaff239 Merge pull request #67 from AudriusButkevicius/relay
Add relay protocol definition
2015-11-03 18:54:39 +00:00
Audrius Butkevicius dea7fd0504 Add relay protocol definition 2015-11-03 18:53:55 +00:00
Audrius Butkevicius 3324a3f4e6 Merge pull request #65 from vhbit/local-disco-v3-magic
Added a expected value for `Announce.Magic`
2015-10-27 10:25:11 +00:00
Valerii Hiora 0c46aa6ff4 Added a expected value for Announce.Magic 2015-10-27 11:42:25 +02:00
Stefan Tatschner e64564c9ae Add syncthing-globaldisco.7 2015-10-20 14:03:36 +02:00
Jakob Borg a9a26be859 Global discovery specs 2015-10-20 13:50:39 +02:00
Jakob Borg de207e4f92 Update specs for v0.12 2015-10-20 13:35:55 +02:00
Jakob Borg 7c2bcdd069 Merge pull request #63 from burkemw3/fix-firewall-link-on-getting-started
Fix Firewall Setup link on Getting Started page
2015-10-20 06:14:36 +02:00
Matt Burke d2b9c7117f Fix Firewall Setup link on Getting Started page 2015-10-19 17:33:26 -04:00
Stefan Tatschner 5790a0f728 Merge pull request #62 from terrycloth/proofreading
Clarify things which confused me my first time trying to set up autostart
2015-10-17 10:18:09 +02:00
terrycloth 259dd17097 Clarify things which confused me my first time trying to set up autostart.
1. For Ubuntu-like systems, I removed the line about clicking to start the
dashboard because it was unnecessarily specific to Unity. There are plenty of
Ubuntu flavors and derivative distros which use other desktops.

2. For the systemd sections about setting up user and system services, I thought
it would make more sense to explain the difference between user- and
system-level services *before* telling the user how to enable them.

3. Pairs of systemd commands where you would run one or the other depending on
how you set up the service should be labeled as such. Otherwise, inexperienced
readers might think you should run the commands in succession.

4. umask is a tricky subject for people who haven't used it before, so I thought
linking to an article that explains it would be helpful.
2015-10-10 23:16:49 -07:00
Audrius Butkevicius b9c60f6e46 Merge pull request #61 from graboluk/patch-1
added stiko to GUI wrappers
2015-10-10 18:29:56 +01:00
graboluk 0a032f1d90 added stiko to GUI wrappers 2015-10-10 18:19:12 +01:00
Jakob Borg 26232729ec Merge pull request #59 from phil-davis/r1
General doc grammar and typos
2015-10-09 02:19:20 +09:00
Phil Davis 9d40bbe3fd General doc grammar and typos 2015-10-08 22:06:15 +05:45
Stefan Tatschner eb00f9e1de Properly format Permissions section 2015-10-06 17:35:53 +02:00
Stefan Tatschner 20f11b8db0 syncthing-inotify is in the official arch linux repos now 2015-10-01 08:37:55 +02:00
Audrius Butkevicius aeddb98e7b Merge pull request #57 from syncthing/obsolete
Move dead or dying projects to the bottom.
2015-09-23 19:50:48 +01:00
Jakob Borg 2c7e4c7abf Merge pull request #45 from simplypeachy/patch-1
Update ignoring.rst
2015-09-23 20:45:33 +02:00
Jakob Borg 5dd66fd7b3 Move dead or dying projects to the bottom.
These are things not updated in the last six months and which don't look
like they are still viable. I don't want to kill them outright, but I
don't want to point users to them either. Other opinions?
2015-09-23 20:38:52 +02:00
Jakob Borg 1871865f25 Re-add license information 2015-09-23 14:47:35 +02:00
Stefan Tatschner 6470d651bb Revert "Add version information to manpage filename"
This reverts commit 3949222974.

Sorry for the noise. :/
2015-09-23 10:20:39 +02:00
Stefan Tatschner 3949222974 Add version information to manpage filename 2015-09-23 10:17:58 +02:00
Jakob Borg 45bc344838 Merge pull request #55 from syncthing/man-updates
Enable specs manpages
2015-09-23 10:15:00 +02:00
Stefan Tatschner caf42ad394 Enable specs manpages 2015-09-23 09:26:53 +02:00
Audrius Butkevicius 9dc75d7f8f Merge pull request #54 from akissa/master
* Link to new Python REST API Bindings
2015-09-22 23:37:49 +01:00
Andrew Colin Kissa cd659a5a3c * Link to new Python REST API Bindings 2015-09-23 00:24:22 +02:00
Jakob Borg ebb98a13e2 Add BEP & discovery protocols 2015-09-22 23:30:39 +02:00
Jakob Borg afce1b8cd3 Fix minor formatting errors 2015-09-22 22:27:45 +02:00
Jakob Borg 53930687d6 Merge pull request #53 from bigscoop/patch-1
fix typo
2015-09-21 22:04:35 +02:00
bigscoop 8db2533c4c fix typo
Fixed typo
2015-09-21 21:51:02 +02:00
Audrius Butkevicius c2377d0fbf Merge pull request #52 from sieren/patch-2
Update contrib.rst
2015-09-17 21:09:19 +01:00
Matt Sieren d040ba7713 Update contrib.rst
Updated contributions and moved QSyncthingTray to cross-platform section since it now supports Mac, Windows and Linux.
2015-09-17 22:00:06 +02:00
Jakob Borg 99de7e0906 Link to official releases 2015-09-10 14:45:12 +02:00
Jakob Borg 0be393d045 Spelling, prerelease notes 2015-09-10 14:36:14 +02:00
Jakob Borg 015746163f Document custom upgrade server 2015-09-10 14:28:50 +02:00
Stefan Tatschner 9af1eeb559 Rewrap synopsis verbatim part 2015-09-07 21:20:52 +02:00
Stefan Tatschner 134131c04b Merge pull request #48 from cbhushan/patch-1
Added the newly added -logfile option
2015-09-07 21:18:58 +02:00
Stefan Tatschner 44b4bbeef6 Merge pull request #49 from cbhushan/patch-2
Mention -logfile option in FAQ
2015-09-07 21:16:02 +02:00
C Bhushan 99dca43050 updated with -logfile option 2015-09-07 11:39:05 -07:00
C Bhushan 9a16073269 Added the newly added -logfile option 2015-09-07 11:26:46 -07:00
Jakob Borg 8634ef23c1 more speling 2015-09-07 08:39:43 +02:00
Jakob Borg 9077d68d46 spling 2015-09-07 08:37:59 +02:00
Stefan Tatschner 317ce61814 Merge pull request #47 from sieren/patch-1
Update contrib.rst

Added QSyncthingTray to list of OSX Contributions.
2015-09-05 15:01:10 +02:00
Matt Sieren a8a72fe88a Update contrib.rst
Added QSyncthingTray
2015-09-05 14:52:45 +02:00
Jakob Borg 214ad8b9b8 Update contrib.rst 2015-09-04 10:10:04 +02:00
Jakob Borg 7ed969feba Update contrib.rst 2015-09-04 10:08:59 +02:00
Stefan Tatschner 134d64616c Fix underline length 2015-09-03 10:10:58 +02:00
Jakob Borg ac6440ab8b Merge pull request #46 from majedev/patch-1
Fix StartupComplete event documentation title
2015-09-03 07:56:33 +02:00
Majed Abdulaziz 70887ac325 Update startupcomplete.rst
Event title is StartupCompleted, while actual event type is StartupComplete (without d). Should be consistent with event type.
2015-09-03 05:58:09 +03:00
simplypeachy d75c5cae25 Update ignoring.rst 2015-09-02 22:06:33 +01:00
simplypeachy 54d0e2201e Update ignoring.rst
Explicitly specify that #include'd files must be within the repo. Note about Mac/Windows always being case-insensitive.
2015-09-02 20:59:51 +01:00
Jakob Borg f66a9046e7 Discovery clarifications 2015-09-02 17:36:15 +02:00
Audrius Butkevicius f7573e99a7 Merge pull request #44 from simplypeachy/patch-4
Update getting-started.rst
2015-09-02 01:21:08 +01:00
Audrius Butkevicius 23d54affea Merge pull request #43 from simplypeachy/patch-3
Minor grammatical clean-up
2015-09-02 01:20:57 +01:00
simplypeachy dc699980c7 Update getting-started.rst
Minor grammatical improvements, cleaned up some sentences and added a few useful sentences. Mention of the requirement of browser cookies as these seem to trip up some new users, looking at the forum and reddit.
2015-09-02 00:51:57 +01:00
simplypeachy 7dbf077e29 Minor grammatical clean-up 2015-09-02 00:04:15 +01:00
Jakob Borg fad1014d40 Add infrastructure article 2015-09-01 09:44:47 +02:00
Jakob Borg 33a8093e67 Merge pull request #41 from fearedbliss/patch-1
Including additional repository for acquiring Syncthing on Gentoo
2015-09-01 07:20:55 +02:00
Jonathan Vasquez 99987afab7 Including additional repository for acquiring Syncthing on Gentoo 2015-08-31 17:28:52 -04:00
Jakob Borg d82d20201c Describe new release signatures 2015-08-31 21:18:06 +02:00
Jakob Borg 8f74785064 New release process 2015-08-30 20:59:14 +02:00
Jakob Borg 565322be5d Multiple GUI:s are fine nowadays 2015-08-30 20:50:40 +02:00
Jakob Borg d2413c30eb Update issues.rst 2015-08-20 21:26:24 +02:00
Jakob Borg 7e741ca388 Update source directory structure 2015-08-13 12:39:46 +02:00
Jakob Borg 2761f8db9e Merge pull request #37 from raferobinson/patch-1
Update Command Line Operation: Synopsis & Options
2015-08-09 09:06:44 +02:00
raferobinson b203e8143f Update syncthing.rst
Added "On Windows only" -no-console in Option section.
2015-08-08 22:43:06 -04:00
Audrius Butkevicius 4795922c46 Merge pull request #38 from syncthing/Zillode-api-key-curl
Show example with api-key using curl
2015-08-08 20:07:05 +01:00
Zillode 81ef3e2859 Show example with api-key using curl
I always need to look for this, so I think it might be useful here
2015-08-08 21:02:56 +02:00
raferobinson cf96200290 Update syncthing.rst
Added -no-console and description.
2015-08-08 00:22:39 -04:00
Jakob Borg 3f355d26cd Merge pull request #34 from syncthing/Zillode-21026
Document 21026/UDP
2015-08-06 08:23:07 +02:00
Audrius Butkevicius 4fe8af25b7 Merge pull request #36 from simplypeachy/patch-2
Minor typo/grammatical fixes.
2015-08-05 21:49:15 +01:00
simplypeachy 33000e8dea Minor typo/grammatical fixes. 2015-08-05 21:45:00 +01:00
Zillode b171371b2a Document 21026/UDP 2015-07-31 23:40:48 +02:00
Audrius Butkevicius 8d0a22c9ce Merge pull request #33 from cron410/patch-1
Corrected mistype of "have"
2015-07-27 19:38:03 +01:00
cron410 5ed23f7990 Corrected mistype of "have" 2015-07-27 14:27:49 -04:00
Audrius Butkevicius a46c24dd23 Merge pull request #32 from theincogtion/patch-2
Update autostart.rst -> permissions
2015-07-24 23:16:13 +01:00
theincogtion 6a91e70f8f Update autostart.rst 2015-07-25 00:12:56 +02:00
Brian 4ab2357276 Fixed versioning seealso 2015-07-23 23:27:53 +02:00
Jakob Borg 2d886db1a4 Merge pull request #28 from Canisaur/patch-1
Small fixes to config.rst
2015-07-22 15:05:37 +02:00
Brian bd99796c8a Fix bracket description, add versioning seealso
IPv6 will not work with angle <> brackets.  It uses [] square brackets.

Add section label to versioning

Added section label to versioning for cross-ref
2015-07-22 08:44:49 -04:00
Jakob Borg 12770ac455 Document autoNormalize, default value for ignoreDelete 2015-07-21 21:56:30 +02:00
Jakob Borg 5c3e5b8ea8 Merge pull request #29 from syncthing/ignoredelete
Document ignoreDelete
2015-07-21 20:51:25 +02:00
Jakob Borg eefb289e86 Document ignoreDelete 2015-07-21 20:38:23 +02:00
Jakob Borg 2f6e2a585c Add documentation for running a discovery server 2015-07-19 11:49:37 +02:00
Stefan Tatschner f9077d4db0 Merge pull request #27 from kreischweide/patch-1
Removed metrothing windows client
2015-07-13 12:57:37 +02:00
Adrian Rudnik 0e9ce0f409 Removed metrothing windows client
Had to deprecate the solution, no more time to maintain it due to job and other constraints.
2015-07-13 11:19:31 +02:00
Audrius Butkevicius ae80d192ec Merge pull request #26 from canton7/feature/db-file-get
Add query parameter docs to db-file-get
2015-07-11 14:00:53 +01:00
Antony Male 8a2d360e2e Add query parameter docs to db-file-get 2015-07-11 13:17:22 +01:00
Audrius Butkevicius 72cb8817fe Merge pull request #25 from lfam/master
Fix REST discovery API documentation
2015-07-09 09:22:36 +01:00
Leo Famulari 57c3d34c78 Fix REST discovery API documentation
Replace reference to nonexistent "hint" URI for POSTing to the discovery
cache with correct reference:
/rest/system/discovery/hint -> /rest/system/discovery
2015-07-08 22:57:20 -04:00
Jean-Denis Vauguet 1940d5fbfa Fix link to Community Contributions from dev/intro
[Stefan Tatschner]: Added missing link title
2015-07-06 09:44:00 +02:00
Jakob Borg b5c71c2076 draft 2015-07-01 22:16:54 +02:00
Jakob Borg d36508766d Coding style 2015-07-01 17:30:08 +02:00
Jakob Borg f04b3155a3 draft localver 2015-07-01 15:17:06 +02:00
Jakob Borg 19a7e40456 Grammar 2015-07-01 13:31:24 +02:00
Stefan Tatschner 34aed35949 Rewrap users/config.rst; 2015-07-01 08:33:07 +02:00
Stefan Tatschner 612ada6a09 Add literal backticks 2015-07-01 08:32:54 +02:00
Jakob Borg 0612e1cee5 Merge pull request #22 from syncthing/failed-files
FolderErrors event
2015-06-30 19:25:29 +02:00
Jakob Borg 8de7180338 FolderErrors event 2015-06-30 14:44:16 +02:00
Audrius Butkevicius 8c3d3a3b2a Merge pull request #23 from arneko/patch-1
Update config.rst
2015-06-29 17:45:37 +01:00
arneko 6c71c48613 Update config.rst
Added some minimal documentation for ping-related configuration
2015-06-29 14:26:38 +02:00
Jakob Borg d980bdbbfc Update release.rst 2015-06-28 08:12:29 +02:00
Audrius Butkevicius d26a92e494 Update syncthing.rst 2015-06-26 08:43:12 +01:00
Jakob Borg 29670c85ce Merge pull request #20 from syncthing/AudriusButkevicius-patch-1
Update faq.rst
2015-06-23 20:50:16 +02:00
Audrius Butkevicius bd35b7f006 Merge pull request #21 from simplypeachy/patch-1
Removed minor grammatical error
2015-06-23 11:17:01 +01:00
simplypeachy 9778ffb967 Removed minor grammatical error 2015-06-23 10:54:22 +01:00
Audrius Butkevicius 2214201990 Update faq.rst 2015-06-22 17:00:20 +01:00
Jakob Borg 396524252c Local sessions 2015-06-21 23:35:12 +02:00
Jakob Borg c07414cda5 Local sessions 2015-06-21 23:34:48 +02:00
Jakob Borg 70734b42a1 Merge pull request #11 from syncthing/Zillode-cookies
Explain CSRF tokens & cookies
2015-06-21 23:29:22 +02:00
Jakob Borg dc7c973c42 Followsymlinks tweaks 2015-06-21 23:28:38 +02:00
Jakob Borg 0b00ef65c2 Draft docs for FollowSymlinks 2015-06-20 22:56:37 +02:00
Zillode 6aeb42602f Clarify session 2015-06-20 13:43:30 +02:00
Jakob Borg f0d6e9aeba Improve formatting of build commands 2015-06-15 20:52:38 +02:00
Jakob Borg 8cada19fd3 Minor tweaks... 2015-06-15 00:16:48 +02:00
Jakob Borg 22f3e392ad Link syntax 2015-06-15 00:06:19 +02:00
Jakob Borg 8b2d790131 Update configuration description 2015-06-14 23:55:40 +02:00
Jakob Borg d07c5b07b1 ItemFinished/ItemStarted metadata 2015-06-14 23:03:02 +02:00
Jakob Borg 2ffe2b7226 Introduction to file versioning (fixes #17) 2015-06-14 13:48:39 +02:00
Jakob Borg 76a8ef14f8 Correct contribution guidelines link 2015-06-13 22:40:09 +02:00
Jakob Borg ec9ade33c2 Update /rest/system/reset to new behavior 2015-06-13 22:34:37 +02:00
Jakob Borg 000536ca76 Describe trash can versioning 2015-06-12 16:29:04 +02:00
Jakob Borg 86647d6ebb Merge pull request #16 from AudriusButkevicius/AudriusButkevicius-patch-1
Update selective.rst
2015-06-11 11:27:24 +02:00
Audrius Butkevicius eaa1663d9d Update selective.rst 2015-06-11 10:13:21 +01:00
Jakob Borg dd74dffe3a selective sync draft 2015-06-11 09:15:53 +02:00
Jakob Borg 61ae7c3100 selective sync draft 2015-06-11 08:19:03 +02:00
Jakob Borg d707643ce6 Fix lists in gui intro 2015-06-10 17:43:07 +02:00
Jakob Borg 9f4325980f Add draft selective sync doc 2015-06-10 17:39:47 +02:00
Jakob Borg 2a09fa5ce4 Reflow it back. :) 2015-06-10 14:01:52 +02:00
Jakob Borg bb04cbbf8b Reformat faq entry 2015-06-10 13:59:40 +02:00
Stefan Tatschner e59772d3df Definition lists everywhere
I just found out that they can be nested, which is useful in this case.
2015-06-10 01:01:59 +02:00
Stefan Tatschner c62fd6af98 Use .. cmdoption:: to document the cli 2015-06-10 00:59:58 +02:00
Jakob Borg 716a75645e Add contribution guidelines 2015-06-10 00:00:12 +02:00
Jakob Borg 605e8d04c6 Remove spurious space in device ID 2015-06-09 23:38:39 +02:00
Jakob Borg 0d946a3401 Clean up the dev intro a little, add source code tree 2015-06-09 23:36:14 +02:00
Jakob Borg 10834febdd Merge pull request #14 from rumpelsepp/apache-24
reverseproxy: Apache 2.2 is rather old, switch to 2.4
2015-06-07 00:10:44 +02:00
Stefan Tatschner fd865c3d14 device-ids: Use proper code blocks 2015-06-06 22:43:07 +02:00
Stefan Tatschner fc70cf4e6a issues.rst: Use definition lists instead of plain lists 2015-06-06 14:06:03 +02:00
Stefan Tatschner 6e14b63175 reverseproxy: Fix obsolete port 2015-06-06 12:01:47 +02:00
Stefan Tatschner 02a9bf7a19 reverseproxy: Apache 2.2 is rather old, switch to 2.4
Apache 2.2 is rather old, IIRC almost debian now ships with 2.4. So
let's document the most recent one.
2015-06-06 11:56:03 +02:00
Stefan Tatschner ba04d232e8 Fix typos and formatting in reverseproxy.rst
It is:

  .. code-block:: LANGUAGE

      Awesome Sourcecode
2015-06-06 11:51:28 +02:00
Jakob Borg 9f2b003c76 Update reverseproxy.rst 2015-06-06 07:18:58 +02:00
Jakob Borg 108b645079 Update reverseproxy.rst 2015-06-06 07:17:18 +02:00
Jakob Borg 8c4454ee64 Update reverseproxy.rst 2015-06-06 07:15:33 +02:00
Jakob Borg 5ec61b3712 Merge pull request #13 from zukoo/patch-1
Create reverseproxy.rst
2015-06-06 07:12:08 +02:00
François-Xavier Gsell ccc9f644ef Create reverseproxy.rst 2015-06-06 10:24:01 +08:00
Jakob Borg 76a1876d10 Merge pull request #10 from vikbez/patch-1
Added an osx 10.9 compatible client link
2015-06-04 19:29:01 +02:00
Zillode a3c3203ab1 Explain CSRF tokens & cookies 2015-06-04 15:08:26 +02:00
Romain Gay 2445726123 Update contrib.rst
added osx 10.9 client link
2015-06-04 14:43:25 +02:00
Jakob Borg 2c31f04547 Clean away .ropeproject 2015-06-03 19:46:27 +02:00
Jakob Borg f9fda9eca4 Fix blockquote and li font sizes 2015-06-03 19:46:27 +02:00
Jakob Borg 08e652b2a2 Add development content from old wiki 2015-06-03 19:46:27 +02:00
Jakob Borg 7bb9daec42 Merge pull request #9 from timboudreau/master
Add Gentoo Linux instructions
2015-06-01 10:25:55 +02:00
Tim Boudreau 3aeb01235e Add Gentoo Linux instructions 2015-06-01 04:12:56 -04:00
Jakob Borg e62be987f9 Generate selected PDF:s 2015-06-01 09:14:27 +02:00
Jakob Borg 033286ab6f Figures are in fact images 2015-06-01 09:04:14 +02:00
Jakob Borg 70d0b9d201 Rudimentary docs for ext file versioning 2015-06-01 08:50:06 +02:00
Jakob Borg 914fef83e9 Slightly clarify versioning 2015-06-01 08:29:59 +02:00
Stefan Tatschner 4f2b1e5d04 Add a TODO, since the debugging page duplicates syncthing(1) 2015-05-31 10:31:07 +02:00
Stefan Tatschner a0aef6e94d Add :user:`` links 2015-05-31 10:26:49 +02:00
Stefan Tatschner 0b4ee46f13 Remove Syncthing prefix from syncthing-config(5) in html toc 2015-05-31 10:15:50 +02:00
Jakob Borg fe9341dc28 Title case See Also 2015-05-31 10:13:01 +02:00
Jakob Borg 4ae12a57c0 Fix toc title for man page 2015-05-31 10:09:25 +02:00
Jakob Borg e2467731a3 Simplify footer 2015-05-31 10:06:57 +02:00
Jakob Borg c74d212da0 De-retinize images 2015-05-31 10:03:09 +02:00
Jakob Borg 1aeb5516d7 Build pdf 2015-05-30 11:29:21 +02:00
Jakob Borg 425d2ed163 Build man pages 2015-05-30 10:41:13 +02:00
Jakob Borg 84e2295834 Typos 2015-05-30 08:31:21 +02:00
Jakob Borg e29443e5dc Update building.rst 2015-05-30 08:29:22 +02:00
Jakob Borg 665ce7aa58 Note about GOPATH 2015-05-30 08:24:24 +02:00
Stefan Tatschner debef5cf6e Merge exit codes page into syncthing(1) 2015-05-30 00:30:01 +02:00
Stefan Tatschner 428c3638c1 Enable syncthing(1) page 2015-05-30 00:15:11 +02:00
Stefan Tatschner 6ef06bb854 Create syncthing(1) 2015-05-30 00:15:02 +02:00
Stefan Tatschner 5d5d5394a0 Fix broken .. seealso:: 2015-05-29 23:34:41 +02:00
Stefan Tatschner 348cd30fc2 Reformat project-presentation.rst and use external links ext 2015-05-29 23:34:41 +02:00
Stefan Tatschner a9afba6c21 Wrap text and use proper rst links on index.rst 2015-05-29 23:34:41 +02:00
Stefan Tatschner 385c46fd4b Use sphinx.ext.extlinks to generate frequently used links
Documentation see here:
http://sphinx-doc.org/ext/extlinks.html
2015-05-29 23:08:17 +02:00
Stefan Tatschner 6ad32bfe1b Add a favicon
Although there is a warning about not being an *.ico, I think the
warning can be safely ignored. Modern browsers support pngs well.
2015-05-29 22:55:47 +02:00
Stefan Tatschner 0c2d769dee Update the README
Since we are using rst now, let's convert the README to rst as well. I
also removed some obsolete Jekyll instructions and replaced them with
the proper Sphinx ones.
2015-05-29 22:48:26 +02:00
Jakob Borg 5f7f96bed3 Fix *most* of monospaced font sizes
grep -lr 'font-size: 12px' bower_components/wyrm | xargs sed -i "" 's/font-size: 12px/font-size: 15px/'

There's still a font-size: 75% in there that I haven't got to.
2015-05-29 18:02:43 +02:00
Jakob Borg 31a396bc86 Merge pull request #7 from rumpelsepp/man
Create and enable manpages
2015-05-29 17:04:26 +02:00
Jakob Borg 5b37d1b497 Add patched sphinx_rtd_theme
There must be a cleaner way to do this... Currently, this theme is built
with:

--- a/sass/_theme_layout.sass
+++ b/sass/_theme_layout.sass
@@ -292,7 +292,7 @@
 .wy-nav-content
   padding: $gutter $gutter * 2
   height: 100%
-  max-width: 800px
+  max-width: 1000px
   margin: auto

 .wy-body-mask
@@ -354,13 +354,6 @@ footer
       height: 100%
       overflow: hidden

-+media($desktop-wider)
-  .wy-nav-content-wrap
-    background: rgba(0,0,0,.05)
-  .wy-nav-content
-    margin: 0
-    background: $section-background-color
-
 @media print
   .rst-versions, footer, .wy-nav-side
     display: none

and this on bower_components/wyrm/sass/wyrm_core/_wy_variables.sass

- $base-font-size:                      16px !default
+ $base-font-size:                      18px !default
  $base-font-family:                    "Helvetica Neue", Arial, sans-serif !default
  $base-font-family-bold:               bold
- $base-line-height:                    24px !default
+ $base-line-height:                    27px !default
2015-05-29 16:58:12 +02:00
Stefan Tatschner ef76180205 Enable the manpages in the configuration file
This patch enables the previously created manpages in the sphinx
configuration file. It also enables "man_show_urls" because the
manpages include several external links.
2015-05-29 16:45:03 +02:00
Stefan Tatschner b7c37f330d Create syncthing-networking(7) 2015-05-29 16:45:03 +02:00
Stefan Tatschner 0e884a9b65 Create syncthing-security(7) 2015-05-29 16:45:03 +02:00
Stefan Tatschner dada1daded Create syncthing-versioning(7) 2015-05-29 16:45:03 +02:00
Stefan Tatschner e6d0b370d2 Create syncthing-faq(7) 2015-05-29 16:44:16 +02:00
Stefan Tatschner a979a50124 Create syncthing-rest-api(7) 2015-05-29 16:43:13 +02:00
Stefan Tatschner bcee82cdde Create syncthing-device-ids(7) 2015-05-29 16:42:28 +02:00
Jakob Borg 3d76603ece Merge pull request #5 from rumpelsepp/cleanups
Some cleanups
2015-05-29 16:39:28 +02:00
Stefan Tatschner 0215ab696a Create syncthing-event-api(7) 2015-05-29 16:39:22 +02:00
Stefan Tatschner 855a77234f Create syncthing-stignore(5) 2015-05-29 16:39:22 +02:00
Stefan Tatschner 0d06e47acf Create syncthing-config(5)
Convert users/config.rst to a properly formatted manpage
2015-05-29 16:39:22 +02:00
Jakob Borg 2e407d3cf7 Merge pull request #3 from rumpelsepp/master
Some rst fixes
2015-05-29 16:37:44 +02:00
Jakob Borg ff64e58b41 Merge pull request #8 from rumpelsepp/todos
Enable TODO extension
2015-05-29 16:34:08 +02:00
Stefan Tatschner dd410c92f1 Fix some rst styling things in autostart.rst 2015-05-29 16:26:09 +02:00
Stefan Tatschner e3cb19f311 Do not count by hand
Humans are too stupid to count, so let the computer do the actual work!
2015-05-29 16:24:58 +02:00
Stefan Tatschner e5242d68b4 Change .. code:: to .. code-block::
Sphinx uses .. code-block:: directives to enable highlighting:
http://sphinx-doc.org/markup/code.html#directive-code-block

Use

  for i in *.rst; do
        sed -i 's/code::/code-block::/g' $i
    done

to update these strings to enable proper highlighting.

Note: A have disabled the windows "bash" one, because the highlighting
is broken anyway...
2015-05-29 16:24:58 +02:00
Jakob Borg 892ad9de74 Merge pull request #2 from alexwlchan/master
A handful of spelling fixes
2015-05-29 16:19:50 +02:00
Stefan Tatschner 89dc32dca0 Enable TODO extension
This extension enables the usage of .. todo:: blocks. This is quite
useful because we can encourage people to work on particular sections
of the documentation.
2015-05-29 14:27:10 +02:00
Jakob Borg 1c45b78ecb Update Python bindings 2015-05-29 13:18:29 +02:00
Stefan Tatschner 635b3f6db5 Remove the font size adjustments
Increasing the font size is *not* useful due to a couple of reasons:

  - It breaks the visual seperation between the headlines. It is very
    hard to determine the headline levels when the font size is that big.

  - It breaks the definition lists which are useful for manpages.

  - It extends the line width to an unnatural value. It is not narrow
    any more, but it is also not big. It just looks a bit odd.

  - The regular font (on linux) renders as almost a light bold type. It
    generates a "Comic Sans" experience. That does not look
    professional.

Since I am not a designer and I don't know how to fix these problems in
the right way, I think the best solution is switching back to the very
sane defaults.
2015-05-29 12:59:37 +02:00
Stefan Tatschner f59300023e Cleanup conf.py
- Move the configuration values to the appropriate position in the file.
- Setting "html_theme_path" is not needed.
- Enable "html_last_updated_fmt". It is quite useful to determine the
  last documentation update. IMO it does not harm at all.
2015-05-29 12:51:35 +02:00
Alex Chan a529eb7fd4 Fix a capitalisation of Syncthing 2015-05-29 09:40:48 +01:00
Alex Chan f6b264275b Spelling fixes 2015-05-29 09:40:00 +01:00
Jakob Borg 5aca789fea Update faq.rst 2015-05-29 09:43:11 +02:00
Jakob Borg afe9f4a845 Update faq.rst 2015-05-29 09:42:47 +02:00
Jakob Borg 8da54bd187 Strikeout 2015-05-29 09:40:38 +02:00
Jakob Borg eb3ec76215 Update project-presentation.rst 2015-05-29 09:29:50 +02:00
Jakob Borg 567d3dd0f5 Sync with latest Wiki content, syncthing -> Syncthing 2015-05-29 09:10:37 +02:00
Jakob Borg cbf0766e86 Formatting fixes 2015-05-28 15:11:12 +02:00
Jakob Borg 040a7f1cf4 Spacing, deploy 2015-05-28 14:19:05 +02:00
Jakob Borg 997d586c5e Style & cleanups 2015-05-28 14:06:57 +02:00
Jakob Borg d0c6272a90 rm -rf 2015-05-28 12:43:56 +02:00
Jakob Borg fd7fe795ad Initial 2015-05-28 12:40:43 +02:00
706 changed files with 14890 additions and 110681 deletions
-9
View File
@@ -1,9 +0,0 @@
# Text files use LF line endings in this repository
* text=auto
# Except the dependencies, which we leave alone
Godeps/** -text=auto
# Diffs on these files are meaningless
gui.files.go -diff
*.svg -diff
+3 -18
View File
@@ -1,18 +1,3 @@
./syncthing
syncthing.exe
*.tar.gz
*.zip
*.asc
*.sublime*
.idea/
.jshintrc
coverage.out
files/pidx
bin
perfstats*.csv
coverage.xml
!gui/scripts/syncthing
.DS_Store
syncthing.md5
syncthing.exe.md5
RELEASE
_build/
_deployed
_deployed.old
-1
View File
@@ -1 +0,0 @@
NICKS
-55
View File
@@ -1,55 +0,0 @@
# This is the official list of Syncthing authors for copyright purposes.
Aaron Bieber <qbit@deftly.net>
Alexander Graf <register-github@alex-graf.de>
Andrew Dunham <andrew@du.nham.ca>
Audrius Butkevicius <audrius.butkevicius@gmail.com>
Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
Ben Curthoys <ben@bencurthoys.com>
Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
Ben Sidhom <bsidhom@gmail.com>
Brandon Philips <brandon@ifup.org>
Brendan Long <self@brendanlong.com>
Caleb Callaway <enlightened.despot@gmail.com>
Carsten Hagemann <moter8@gmail.com>
Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
Chris Joel <chris@scriptolo.gy>
Colin Kennedy <moshen.colin@gmail.com>
Daniel Martí <mvdan@mvdan.cc>
Dennis Wilson <dw@risu.io>
Dominik Heidler <dominik@heidler.eu>
Elias Jarlebring <jarlebring@gmail.com>
Emil Hessman <emil@hessman.se>
Federico Castagnini <federico.castagnini@gmail.com>
Felix Ableitner <me@nutomic.com>
Felix Unterpaintner <bigbear2nd@gmail.com>
Francois-Xavier Gsell <fxgsell@gmail.com>
Gilli Sigurdsson <gilli@vx.is>
Jakob Borg <jakob@nym.se>
James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
Jaroslav Malec <dzardacz@gmail.com>
Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
Jochen Voss <voss@seehuhn.de>
Johan Vromans <jvromans@squirrel.nl>
Karol Różycki <rozycki.karol@gmail.com>
Ken'ichi Kamada <kamada@nanohz.org>
Lode Hoste <zillode@zillode.be>
Marcin Dziadus <dziadus.marcin@gmail.com>
Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
Marc Pujol <kilburn@la3.org>
Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
Michael Tilli <pyfisch@gmail.com>
Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
Peter Hoeg <peter@speartail.com>
Philippe Schommers <philippe@schommers.be>
Phill Luby <phill.luby@newredo.com>
Piotr Bejda <piotrb10@gmail.com>
Ryan Sullivan <kayoticsully@gmail.com>
Sergey Mishin <ralder@yandex.ru>
Stefan Tatschner <stefan@sevenbyte.org>
Tim Abell <tim@timwise.co.uk>
Tobias Nygren <tnn@nygren.pp.se>
Tomas Cerveny <kozec@kozec.com>
Tully Robinson <tully@tojr.org>
Veeti Paananen <veeti.paananen@rojekti.fi>
Vil Brekin <vilbrekin@gmail.com>
-92
View File
@@ -1,92 +0,0 @@
## Conduct
* We are committed to providing a friendly, safe and welcoming
environment for all, regardless of gender, sexual orientation,
disability, ethnicity, religion, or similar personal characteristic.
* On IRC, please avoid using overtly sexual nicknames or other nicknames
that might detract from a friendly, safe and welcoming environment for
all.
* Please be kind and courteous. There's no need to be mean or rude.
* Respect that people have differences of opinion and that every design
or implementation choice carries a trade-off and numerous costs. There
is seldom a right answer.
* Please keep unstructured critique to a minimum. If you have solid
ideas you want to experiment with, make a fork and see how it works.
* We will exclude you from interaction if you insult, demean or harass
anyone. That is not welcome behaviour. We interpret the term
"harassment" as including the definition in the <a
href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
if you have any lack of clarity about what might be included in that
concept, please read their definition. In particular, we don't
tolerate behavior that excludes people in socially marginalized
groups.
* Private harassment is also unacceptable. No matter who you are, if you
feel you have been or are being harassed or made uncomfortable by a
community member, please contact one of the channel ops or any of the
Syncthing core team immediately. Whether you're a regular contributor
or a newcomer, we care about making this community a safe place for
you and we've got your back.
* Likewise any spamming, trolling, flaming, baiting or other
attention-stealing behaviour is not welcome.
## Moderation
These are the policies for upholding our community's standards of
conduct in our communication channels, most notably in Syncthing-related
IRC channels and on the web forum.
1. Remarks that violate the Syncthing standards of conduct, including
hateful, hurtful, oppressive, or exclusionary remarks, are not
allowed. (Cursing is allowed, but never targeting another user, and
never in a hateful manner.)
2. Remarks that moderators find inappropriate, whether listed in the
code of conduct or not, are also not allowed.
3. Moderators will first respond to such remarks with a warning.
4. If the warning is unheeded, the user will be "kicked," i.e., kicked
out of the communication channel to cool off.
5. If the user comes back and continues to make trouble, they will be
banned, i.e., indefinitely excluded.
6. Moderators may choose at their discretion to un-ban the user if it
was a first offense and they offer the offended party a genuine
apology.
7. If a moderator bans someone and you think it was unjustified, please
take it up with that moderator, or with a different moderator, **in
private**. Complaints about bans in-channel are not allowed.
8. Moderators are held to a higher standard than other community
members. If a moderator creates an inappropriate situation, they
should expect less leeway than others.
In the Syncthing community we strive to go the extra step to look out
for each other. Don't just aim to be technically unimpeachable, try to
be your best self. In particular, avoid flirting with offensive or
sensitive issues, particularly if they're off-topic; this all too
often leads to unnecessary fights, hurt feelings, and damaged trust;
worse, it can drive people away from the community entirely.
And if someone takes issue with something you said or did, resist the
urge to be defensive. Just stop doing what it was they complained about
and apologize. Even if you feel you were misinterpreted or unfairly
accused, chances are good there was something you could've communicated
better — remember that it's your responsibility to make your fellow
community members comfortable. Everyone wants to get along and we are
all here first and foremost because we want to talk about cool
technology. You will find that people will be eager to assume good
intent and forgive as long as you earn their trust.
*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*
-140
View File
@@ -1,140 +0,0 @@
## Reporting Bugs
Please file bugs in the [Github Issue
Tracker](https://github.com/syncthing/syncthing/issues). Include at
least the following:
- What happened
- What did you expect to happen instead of what *did* happen, if it's
not crazy obvious
- What operating system, operating system version and version of
Syncthing you are running
- The same for other connected devices, where relevant
- Screenshot if the issue concerns something visible in the GUI
- Console log entries, where possible and relevant
If you're not sure whether something is relevant, erring on the side of
too much information will never get you yelled at. :)
## Contributing Translations
All translations are done via
[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
wish to contribute to a translation, just head over there and sign up.
Before every release, the language resources are updated from the
latest info on Transifex.
## Contributing Code
Every contribution is welcome. If you want to contribute but are unsure
where to start, any open issues are fair game! Be prepared for a
[certain amount of review](https://github.com/syncthing/syncthing/wiki/FAQ#why-are-you-being-so-hard-on-my-pull-request);
it's all in the name of quality. :) Following the points below will make this
a smoother process.
Individuals making significant and valuable contributions are given
commit-access to the project. If you make a significant contribution and
are not considered for commit-access, please contact any of the
Syncthing core team members.
All nontrivial contributions should go through the pull request
mechanism for internal review. Determining what is "nontrivial" is left
at the discretion of the contributor.
### Authorship
All code authors are listed in the AUTHORS file. Commits must be made
with the same name and email as listed in the AUTHORS file. To
accomplish this, ensure that your git configuration is set correctly
prior to making your first commit;
$ git config --global user.name "Jane Doe"
$ git config --global user.email janedoe@example.com
You must be reachable on the given email address. If you do not wish to
use your real name for whatever reason, using a nickname or pseudonym is
perfectly acceptable.
### Core Team
The Syncthing core team currently consists of the following members;
- Jakob Borg (@calmh)
- Audrius Butkevicius (@AudriusButkevicius)
## Coding Style
- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
as much as makes sense.
- All text files use Unix line endings.
- Each commit should be `go fmt` clean.
- The commit message subject should be a single short sentence
describing the change, starting with a capital letter.
- Commits that resolve an existing issue must include the issue number
as `(fixes #123)` at the end of the commit message subject.
- Imports are grouped per `goimports` standard; that is, standard
library first, then third party libraries after a blank line.
- A contribution solving a single issue or introducing a single new
feature should probably be a single commit based on the current
`master` branch. You may be asked to "rebase" or "squash" your pull
request to make sure this is the case, especially if there have been
amendments during review.
## Licensing
All contributions are made under the same MPLv2 license as the rest of
the project, except documentation, user interface text and translation
strings which are licensed under the Creative Commons Attribution 4.0
International License. You retain the copyright to code you have
written.
When accepting your first contribution, the maintainer of the project
will ensure that you are added to the AUTHORS file. You are welcome to
add yourself as a separate commit in your first pull request.
## Building
[See the documentation](https://github.com/syncthing/syncthing/wiki/Building)
on how to get started with a build environment.
## Branches
- `master` is the main branch containing good code that will end up in
the next release. You should base your work on it. It won't ever be
rebased or force-pushed to.
- `vx.y` branches exist to make patch releases on otherwise obsolete
minor releases. Should only contain fixes cherry picked from master.
Don't base any work on them.
- Other branches are probably topic branches and may be subject to
rebasing. Don't base any work on them unless you specifically know
otherwise.
## Tags
All releases are tagged semver style as `vx.y.z`. Release tags are
signed by GPG key BCE524C7.
## Tests
Yes please!
## Documentation
[Over here!](https://github.com/syncthing/syncthing/wiki)
## License
MPLv2
-77
View File
@@ -1,77 +0,0 @@
{
"ImportPath": "github.com/syncthing/syncthing",
"GoVersion": "go1.4",
"Packages": [
"./cmd/..."
],
"Deps": [
{
"ImportPath": "github.com/bkaradzic/go-lz4",
"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"
},
{
"ImportPath": "github.com/calmh/logger",
"Rev": "4d4e2801954c5581e4c2a80a3d3beb3b3645fd04"
},
{
"ImportPath": "github.com/calmh/luhn",
"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
},
{
"ImportPath": "github.com/calmh/xdr",
"Rev": "5f7208e86762911861c94f1849eddbfc0a60cbf0"
},
{
"ImportPath": "github.com/juju/ratelimit",
"Rev": "c5abe513796336ee2869745bff0638508450e9c5"
},
{
"ImportPath": "github.com/kardianos/osext",
"Rev": "efacde03154693404c65e7aa7d461ac9014acd0c"
},
{
"ImportPath": "github.com/syncthing/protocol",
"Rev": "e7db2648034fb71b051902a02bc25d4468ed492e"
},
{
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
"Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
},
{
"ImportPath": "github.com/syndtr/gosnappy/snappy",
"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
},
{
"ImportPath": "github.com/thejerf/suture",
"Rev": "ff19fb384c3fe30f42717967eaa69da91e5f317c"
},
{
"ImportPath": "github.com/vitrun/qart/coding",
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
},
{
"ImportPath": "github.com/vitrun/qart/gf256",
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
},
{
"ImportPath": "github.com/vitrun/qart/qr",
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
},
{
"ImportPath": "golang.org/x/crypto/bcrypt",
"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
},
{
"ImportPath": "golang.org/x/crypto/blowfish",
"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
},
{
"ImportPath": "golang.org/x/text/transform",
"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
},
{
"ImportPath": "golang.org/x/text/unicode/norm",
"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
}
]
}
Generated
-5
View File
@@ -1,5 +0,0 @@
This directory tree is generated automatically by godep.
Please do not edit.
See https://github.com/tools/godep for more information.
-2
View File
@@ -1,2 +0,0 @@
/pkg
/bin
@@ -1 +0,0 @@
/lz4-example/lz4-example
-7
View File
@@ -1,7 +0,0 @@
language: go
go:
- 1.1
- 1.2
- 1.3
- tip
-24
View File
@@ -1,24 +0,0 @@
Copyright 2011-2012 Branimir Karadzic. All rights reserved.
Copyright 2013 Damian Gryski. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
-71
View File
@@ -1,71 +0,0 @@
go-lz4
======
go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
is located at:
https://code.google.com/p/lz4/
Status
------
[![Build Status](https://secure.travis-ci.org/bkaradzic/go-lz4.png)](http://travis-ci.org/bkaradzic/go-lz4)
[![GoDoc](https://godoc.org/github.com/bkaradzic/go-lz4?status.png)](https://godoc.org/github.com/bkaradzic/go-lz4)
Usage
-----
go get github.com/bkaradzic/go-lz4
import "github.com/bkaradzic/go-lz4"
The package name is `lz4`
Notes
-----
* go-lz4 saves a uint32 with the original uncompressed length at the beginning
of the encoded buffer. They may get in the way of interoperability with
other implementations.
Contributors
------------
Damian Gryski ([@dgryski](https://github.com/dgryski))
Dustin Sallings ([@dustin](https://github.com/dustin))
Contact
-------
[@bkaradzic](https://twitter.com/bkaradzic)
http://www.stuckingeometry.com
Project page
https://github.com/bkaradzic/go-lz4
License
-------
Copyright 2011-2012 Branimir Karadzic. All rights reserved.
Copyright 2013 Damian Gryski. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
-74
View File
@@ -1,74 +0,0 @@
package main
import (
"math/rand"
"github.com/bkaradzic/go-lz4"
// lz4's API matches snappy's, so we can easily see how it performs
// lz4 "code.google.com/p/snappy-go/snappy"
)
var input = `
ADVENTURE I. A SCANDAL IN BOHEMIA
I.
To Sherlock Holmes she is always THE woman. I have seldom heard
him mention her under any other name. In his eyes she eclipses
and predominates the whole of her sex. It was not that he felt
any emotion akin to love for Irene Adler. All emotions, and that
one particularly, were abhorrent to his cold, precise but
admirably balanced mind. He was, I take it, the most perfect
reasoning and observing machine that the world has seen, but as a
lover he would have placed himself in a false position. He never
spoke of the softer passions, save with a gibe and a sneer. They
were admirable things for the observer--excellent for drawing the
veil from men's motives and actions. But for the trained reasoner
to admit such intrusions into his own delicate and finely
adjusted temperament was to introduce a distracting factor which
might throw a doubt upon all his mental results. Grit in a
sensitive instrument, or a crack in one of his own high-power
lenses, would not be more disturbing than a strong emotion in a
nature such as his. And yet there was but one woman to him, and
that woman was the late Irene Adler, of dubious and questionable
memory.
I had seen little of Holmes lately. My marriage had drifted us
away from each other. My own complete happiness, and the
home-centred interests which rise up around the man who first
finds himself master of his own establishment, were sufficient to
absorb all my attention, while Holmes, who loathed every form of
society with his whole Bohemian soul, remained in our lodgings in
Baker Street, buried among his old books, and alternating from
week to week between cocaine and ambition, the drowsiness of the
drug, and the fierce energy of his own keen nature. He was still,
as ever, deeply attracted by the study of crime, and occupied his
immense faculties and extraordinary powers of observation in
following out those clues, and clearing up those mysteries which
had been abandoned as hopeless by the official police. From time
to time I heard some vague account of his doings: of his summons
to Odessa in the case of the Trepoff murder, of his clearing up
of the singular tragedy of the Atkinson brothers at Trincomalee,
and finally of the mission which he had accomplished so
delicately and successfully for the reigning family of Holland.
Beyond these signs of his activity, however, which I merely
shared with all the readers of the daily press, I knew little of
my former friend and companion.
`
func main() {
compressed, _ := lz4.Encode(nil, []byte(input))
modified := make([]byte, len(compressed))
for {
copy(modified, compressed)
for i := 0; i < 100; i++ {
modified[rand.Intn(len(compressed)-4)+4] = byte(rand.Intn(256))
}
lz4.Decode(nil, modified)
}
}
-94
View File
@@ -1,94 +0,0 @@
/*
* Copyright 2011 Branimir Karadzic. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
package main
import (
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"runtime/pprof"
lz4 "github.com/bkaradzic/go-lz4"
)
var (
decompress = flag.Bool("d", false, "decompress")
)
func main() {
var optCPUProfile = flag.String("cpuprofile", "", "profile")
flag.Parse()
if *optCPUProfile != "" {
f, err := os.Create(*optCPUProfile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
args := flag.Args()
var data []byte
if len(args) < 2 {
fmt.Print("Usage: lz4 [-d] <input> <output>\n")
os.Exit(1)
}
input, err := os.OpenFile(args[0], os.O_RDONLY, 0644)
if err != nil {
fmt.Printf("Failed to open input file %s\n", args[0])
os.Exit(1)
}
defer input.Close()
if *decompress {
data, _ = ioutil.ReadAll(input)
data, err = lz4.Decode(nil, data)
if err != nil {
fmt.Println("Failed to decode:", err)
return
}
} else {
data, _ = ioutil.ReadAll(input)
data, err = lz4.Encode(nil, data)
if err != nil {
fmt.Println("Failed to encode:", err)
return
}
}
err = ioutil.WriteFile(args[1], data, 0644)
if err != nil {
fmt.Printf("Failed to open output file %s\n", args[1])
os.Exit(1)
}
}
-63
View File
@@ -1,63 +0,0 @@
package lz4
import (
"bytes"
"io/ioutil"
"testing"
)
var testfile, _ = ioutil.ReadFile("testdata/pg1661.txt")
func roundtrip(t *testing.T, input []byte) {
dst, err := Encode(nil, input)
if err != nil {
t.Errorf("got error during compression: %s", err)
}
output, err := Decode(nil, dst)
if err != nil {
t.Errorf("got error during decompress: %s", err)
}
if !bytes.Equal(output, input) {
t.Errorf("roundtrip failed")
}
}
func TestEmpty(t *testing.T) {
roundtrip(t, nil)
}
func TestLengths(t *testing.T) {
for i := 0; i < 1024; i++ {
roundtrip(t, testfile[:i])
}
for i := 1024; i < 4096; i += 23 {
roundtrip(t, testfile[:i])
}
}
func TestWords(t *testing.T) {
roundtrip(t, testfile)
}
func BenchmarkLZ4Encode(b *testing.B) {
for i := 0; i < b.N; i++ {
Encode(nil, testfile)
}
}
func BenchmarkLZ4Decode(b *testing.B) {
var compressed, _ = Encode(nil, testfile)
b.ResetTimer()
for i := 0; i < b.N; i++ {
Decode(nil, compressed)
}
}
-194
View File
@@ -1,194 +0,0 @@
/*
* Copyright 2011-2012 Branimir Karadzic. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
package lz4
import (
"encoding/binary"
"errors"
"io"
)
var (
// ErrCorrupt indicates the input was corrupt
ErrCorrupt = errors.New("corrupt input")
)
const (
mlBits = 4
mlMask = (1 << mlBits) - 1
runBits = 8 - mlBits
runMask = (1 << runBits) - 1
)
type decoder struct {
src []byte
dst []byte
spos uint32
dpos uint32
ref uint32
}
func (d *decoder) readByte() (uint8, error) {
if int(d.spos) == len(d.src) {
return 0, io.EOF
}
b := d.src[d.spos]
d.spos++
return b, nil
}
func (d *decoder) getLen() (uint32, error) {
length := uint32(0)
ln, err := d.readByte()
if err != nil {
return 0, ErrCorrupt
}
for ln == 255 {
length += 255
ln, err = d.readByte()
if err != nil {
return 0, ErrCorrupt
}
}
length += uint32(ln)
return length, nil
}
func (d *decoder) cp(length, decr uint32) {
if int(d.ref+length) < int(d.dpos) {
copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
} else {
for ii := uint32(0); ii < length; ii++ {
d.dst[d.dpos+ii] = d.dst[d.ref+ii]
}
}
d.dpos += length
d.ref += length - decr
}
func (d *decoder) finish(err error) error {
if err == io.EOF {
return nil
}
return err
}
// Decode returns the decoded form of src. The returned slice may be a
// subslice of dst if it was large enough to hold the entire decoded block.
func Decode(dst, src []byte) ([]byte, error) {
if len(src) < 4 {
return nil, ErrCorrupt
}
uncompressedLen := binary.LittleEndian.Uint32(src)
if uncompressedLen == 0 {
return nil, nil
}
if uncompressedLen > MaxInputSize {
return nil, ErrTooLarge
}
if dst == nil || len(dst) < int(uncompressedLen) {
dst = make([]byte, uncompressedLen)
}
d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
decr := []uint32{0, 3, 2, 3}
for {
code, err := d.readByte()
if err != nil {
return d.dst, d.finish(err)
}
length := uint32(code >> mlBits)
if length == runMask {
ln, err := d.getLen()
if err != nil {
return nil, ErrCorrupt
}
length += ln
}
if int(d.spos+length) > len(d.src) {
return nil, ErrCorrupt
}
for ii := uint32(0); ii < length; ii++ {
d.dst[d.dpos+ii] = d.src[d.spos+ii]
}
d.spos += length
d.dpos += length
if int(d.spos) == len(d.src) {
return d.dst, nil
}
if int(d.spos+2) >= len(d.src) {
return nil, ErrCorrupt
}
back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
if back > d.dpos {
return nil, ErrCorrupt
}
d.spos += 2
d.ref = d.dpos - back
length = uint32(code & mlMask)
if length == mlMask {
ln, err := d.getLen()
if err != nil {
return nil, ErrCorrupt
}
length += ln
}
literal := d.dpos - d.ref
if literal < 4 {
d.cp(4, decr[literal])
} else {
length += 4
}
if d.dpos+length > uncompressedLen {
return nil, ErrCorrupt
}
d.cp(length, 0)
}
}
File diff suppressed because it is too large Load Diff
-188
View File
@@ -1,188 +0,0 @@
/*
* Copyright 2011-2012 Branimir Karadzic. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
package lz4
import "encoding/binary"
import "errors"
const (
minMatch = 4
hashLog = 17
hashTableSize = 1 << hashLog
hashShift = (minMatch * 8) - hashLog
incompressible uint32 = 128
uninitHash = 0x88888888
// MaxInputSize is the largest buffer than can be compressed in a single block
MaxInputSize = 0x7E000000
)
var (
// ErrTooLarge indicates the input buffer was too large
ErrTooLarge = errors.New("input too large")
)
type encoder struct {
src []byte
dst []byte
hashTable []uint32
pos uint32
anchor uint32
dpos uint32
}
// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
func CompressBound(isize int) int {
if isize > MaxInputSize {
return 0
}
return isize + ((isize) / 255) + 16 + 4
}
func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
ln := length
var code byte
if ln > runMask-1 {
code = runMask
} else {
code = byte(ln)
}
if mlLen > mlMask-1 {
e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
} else {
e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
}
e.dpos++
if code == runMask {
ln -= runMask
for ; ln > 254; ln -= 255 {
e.dst[e.dpos] = 255
e.dpos++
}
e.dst[e.dpos] = byte(ln)
e.dpos++
}
for ii := uint32(0); ii < length; ii++ {
e.dst[e.dpos+ii] = e.src[pos+ii]
}
e.dpos += length
}
// Encode returns the encoded form of src. The returned array may be a
// sub-slice of dst if it was large enough to hold the entire output.
func Encode(dst, src []byte) ([]byte, error) {
if len(src) >= MaxInputSize {
return nil, ErrTooLarge
}
if n := CompressBound(len(src)); len(dst) < n {
dst = make([]byte, n)
}
e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
binary.LittleEndian.PutUint32(dst, uint32(len(src)))
e.dpos = 4
var (
step uint32 = 1
limit = incompressible
)
for {
if int(e.pos)+12 >= len(e.src) {
e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
return e.dst[:e.dpos], nil
}
sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
hash := (sequence * 2654435761) >> hashShift
ref := e.hashTable[hash] + uninitHash
e.hashTable[hash] = e.pos - uninitHash
if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
if e.pos-e.anchor > limit {
limit <<= 1
step += 1 + (step >> 2)
}
e.pos += step
continue
}
if step > 1 {
e.hashTable[hash] = ref - uninitHash
e.pos -= step - 1
step = 1
continue
}
limit = incompressible
ln := e.pos - e.anchor
back := e.pos - ref
anchor := e.anchor
e.pos += minMatch
ref += minMatch
e.anchor = e.pos
for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
e.pos++
ref++
}
mlLen := e.pos - e.anchor
e.writeLiterals(ln, mlLen, anchor)
e.dst[e.dpos] = uint8(back)
e.dst[e.dpos+1] = uint8(back >> 8)
e.dpos += 2
if mlLen > mlMask-1 {
mlLen -= mlMask
for mlLen > 254 {
mlLen -= 255
e.dst[e.dpos] = 255
e.dpos++
}
e.dst[e.dpos] = byte(mlLen)
e.dpos++
}
e.anchor = e.pos
}
}
-19
View File
@@ -1,19 +0,0 @@
Copyright (C) 2013 Jakob Borg
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-15
View File
@@ -1,15 +0,0 @@
logger
======
A small wrapper around `log` to provide log levels.
Documentation
-------------
http://godoc.org/github.com/calmh/logger
License
-------
MIT
-179
View File
@@ -1,179 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// Package logger implements a standardized logger with callback functionality
package logger
import (
"fmt"
"log"
"os"
"strings"
"sync"
)
type LogLevel int
const (
LevelDebug LogLevel = iota
LevelVerbose
LevelInfo
LevelOK
LevelWarn
LevelFatal
NumLevels
)
// A MessageHandler is called with the log level and message text.
type MessageHandler func(l LogLevel, msg string)
type Logger struct {
logger *log.Logger
handlers [NumLevels][]MessageHandler
mut sync.Mutex
}
// The default logger logs to standard output with a time prefix.
var DefaultLogger = New()
func New() *Logger {
return &Logger{
logger: log.New(os.Stdout, "", log.Ltime),
}
}
// AddHandler registers a new MessageHandler to receive messages with the
// specified log level or above.
func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
l.mut.Lock()
defer l.mut.Unlock()
l.handlers[level] = append(l.handlers[level], h)
}
// See log.SetFlags
func (l *Logger) SetFlags(flag int) {
l.logger.SetFlags(flag)
}
// See log.SetPrefix
func (l *Logger) SetPrefix(prefix string) {
l.logger.SetPrefix(prefix)
}
func (l *Logger) callHandlers(level LogLevel, s string) {
for _, h := range l.handlers[level] {
h(level, strings.TrimSpace(s))
}
}
// Debugln logs a line with a DEBUG prefix.
func (l *Logger) Debugln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "DEBUG: "+s)
l.callHandlers(LevelDebug, s)
}
// Debugf logs a formatted line with a DEBUG prefix.
func (l *Logger) Debugf(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "DEBUG: "+s)
l.callHandlers(LevelDebug, s)
}
// Infoln logs a line with a VERBOSE prefix.
func (l *Logger) Verboseln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "VERBOSE: "+s)
l.callHandlers(LevelVerbose, s)
}
// Infof logs a formatted line with a VERBOSE prefix.
func (l *Logger) Verbosef(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "VERBOSE: "+s)
l.callHandlers(LevelVerbose, s)
}
// Infoln logs a line with an INFO prefix.
func (l *Logger) Infoln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "INFO: "+s)
l.callHandlers(LevelInfo, s)
}
// Infof logs a formatted line with an INFO prefix.
func (l *Logger) Infof(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "INFO: "+s)
l.callHandlers(LevelInfo, s)
}
// Okln logs a line with an OK prefix.
func (l *Logger) Okln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "OK: "+s)
l.callHandlers(LevelOK, s)
}
// Okf logs a formatted line with an OK prefix.
func (l *Logger) Okf(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "OK: "+s)
l.callHandlers(LevelOK, s)
}
// Warnln logs a formatted line with a WARNING prefix.
func (l *Logger) Warnln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "WARNING: "+s)
l.callHandlers(LevelWarn, s)
}
// Warnf logs a formatted line with a WARNING prefix.
func (l *Logger) Warnf(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "WARNING: "+s)
l.callHandlers(LevelWarn, s)
}
// Fatalln logs a line with a FATAL prefix and exits the process with exit
// code 1.
func (l *Logger) Fatalln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "FATAL: "+s)
l.callHandlers(LevelFatal, s)
os.Exit(1)
}
// Fatalf logs a formatted line with a FATAL prefix and exits the process with
// exit code 1.
func (l *Logger) Fatalf(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "FATAL: "+s)
l.callHandlers(LevelFatal, s)
os.Exit(1)
}
-58
View File
@@ -1,58 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package logger
import (
"strings"
"testing"
)
func TestAPI(t *testing.T) {
l := New()
l.SetFlags(0)
l.SetPrefix("testing")
debug := 0
l.AddHandler(LevelDebug, checkFunc(t, LevelDebug, "test 0", &debug))
info := 0
l.AddHandler(LevelInfo, checkFunc(t, LevelInfo, "test 1", &info))
warn := 0
l.AddHandler(LevelWarn, checkFunc(t, LevelWarn, "test 2", &warn))
ok := 0
l.AddHandler(LevelOK, checkFunc(t, LevelOK, "test 3", &ok))
l.Debugf("test %d", 0)
l.Debugln("test", 0)
l.Infof("test %d", 1)
l.Infoln("test", 1)
l.Warnf("test %d", 2)
l.Warnln("test", 2)
l.Okf("test %d", 3)
l.Okln("test", 3)
if debug != 2 {
t.Errorf("Debug handler called %d != 2 times", debug)
}
if info != 2 {
t.Errorf("Info handler called %d != 2 times", info)
}
if warn != 2 {
t.Errorf("Warn handler called %d != 2 times", warn)
}
if ok != 2 {
t.Errorf("Ok handler called %d != 2 times", ok)
}
}
func checkFunc(t *testing.T, expectl LogLevel, expectmsg string, counter *int) func(LogLevel, string) {
return func(l LogLevel, msg string) {
*counter++
if l != expectl {
t.Errorf("Incorrect message level %d != %d", l, expectl)
}
if !strings.HasSuffix(msg, expectmsg) {
t.Errorf("%q does not end with %q", msg, expectmsg)
}
}
}
-19
View File
@@ -1,19 +0,0 @@
Copyright (C) 2014 Jakob Borg
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-70
View File
@@ -1,70 +0,0 @@
// Copyright (C) 2014 Jakob Borg
// Package luhn generates and validates Luhn mod N check digits.
package luhn
import (
"fmt"
"strings"
)
// An alphabet is a string of N characters, representing the digits of a given
// base N.
type Alphabet string
var (
Base32 Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
)
// Generate returns a check digit for the string s, which should be composed
// of characters from the Alphabet a.
func (a Alphabet) Generate(s string) (rune, error) {
if err := a.check(); err != nil {
return 0, err
}
factor := 1
sum := 0
n := len(a)
for i := range s {
codepoint := strings.IndexByte(string(a), s[i])
if codepoint == -1 {
return 0, fmt.Errorf("Digit %q not valid in alphabet %q", s[i], a)
}
addend := factor * codepoint
if factor == 2 {
factor = 1
} else {
factor = 2
}
addend = (addend / n) + (addend % n)
sum += addend
}
remainder := sum % n
checkCodepoint := (n - remainder) % n
return rune(a[checkCodepoint]), nil
}
// Validate returns true if the last character of the string s is correct, for
// a string s composed of characters in the alphabet a.
func (a Alphabet) Validate(s string) bool {
t := s[:len(s)-1]
c, err := a.Generate(t)
if err != nil {
return false
}
return rune(s[len(s)-1]) == c
}
// check returns an error if the given alphabet does not consist of unique characters
func (a Alphabet) check() error {
cm := make(map[byte]bool, len(a))
for i := range a {
if cm[a[i]] {
return fmt.Errorf("Digit %q non-unique in alphabet %q", a[i], a)
}
cm[a[i]] = true
}
return nil
}
-59
View File
@@ -1,59 +0,0 @@
// Copyright (C) 2014 Jakob Borg
package luhn_test
import (
"testing"
"github.com/calmh/luhn"
)
func TestGenerate(t *testing.T) {
// Base 6 Luhn
a := luhn.Alphabet("abcdef")
c, err := a.Generate("abcdef")
if err != nil {
t.Fatal(err)
}
if c != 'e' {
t.Errorf("Incorrect check digit %c != e", c)
}
// Base 10 Luhn
a = luhn.Alphabet("0123456789")
c, err = a.Generate("7992739871")
if err != nil {
t.Fatal(err)
}
if c != '3' {
t.Errorf("Incorrect check digit %c != 3", c)
}
}
func TestInvalidString(t *testing.T) {
a := luhn.Alphabet("ABC")
_, err := a.Generate("7992739871")
t.Log(err)
if err == nil {
t.Error("Unexpected nil error")
}
}
func TestBadAlphabet(t *testing.T) {
a := luhn.Alphabet("01234566789")
_, err := a.Generate("7992739871")
t.Log(err)
if err == nil {
t.Error("Unexpected nil error")
}
}
func TestValidate(t *testing.T) {
a := luhn.Alphabet("abcdef")
if !a.Validate("abcdefe") {
t.Errorf("Incorrect validation response for abcdefe")
}
if a.Validate("abcdefd") {
t.Errorf("Incorrect validation response for abcdefd")
}
}
-1
View File
@@ -1 +0,0 @@
coverage.out
-19
View File
@@ -1,19 +0,0 @@
language: go
go:
- tip
install:
- export PATH=$PATH:$HOME/gopath/bin
- go get code.google.com/p/go.tools/cmd/cover
- go get github.com/mattn/goveralls
script:
- ./generate.sh
- go test -coverprofile=coverage.out
after_success:
- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
env:
global:
secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=
-19
View File
@@ -1,19 +0,0 @@
Copyright (C) 2014 Jakob Borg.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
-12
View File
@@ -1,12 +0,0 @@
xdr
===
[![Build Status](https://img.shields.io/travis/calmh/xdr.svg?style=flat)](https://travis-ci.org/calmh/xdr)
[![Coverage Status](https://img.shields.io/coveralls/calmh/xdr.svg?style=flat)](https://coveralls.io/r/calmh/xdr?branch=master)
[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)
This is an XDR encoding/decoding library. It uses code generation and
not reflection. It supports the IPDR bastardized XDR format when built
with `-tags ipdr`.
-117
View File
@@ -1,117 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package xdr_test
import (
"io"
"io/ioutil"
"testing"
"github.com/calmh/xdr"
)
type XDRBenchStruct struct {
I1 uint64
I2 uint32
I3 uint16
I4 uint8
Bs0 []byte // max:128
Bs1 []byte
S0 string // max:128
S1 string
}
var res []byte // no to be optimized away
var s = XDRBenchStruct{
I1: 42,
I2: 43,
I3: 44,
I4: 45,
Bs0: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18},
Bs1: []byte{11, 12, 13, 14, 15, 16, 17, 18, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
S0: "Hello World! String one.",
S1: "Hello World! String two.",
}
var e []byte
func init() {
e, _ = s.MarshalXDR()
}
func BenchmarkThisMarshal(b *testing.B) {
for i := 0; i < b.N; i++ {
res, _ = s.MarshalXDR()
}
}
func BenchmarkThisUnmarshal(b *testing.B) {
var t XDRBenchStruct
for i := 0; i < b.N; i++ {
err := t.UnmarshalXDR(e)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkThisEncode(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := s.EncodeXDR(ioutil.Discard)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkThisEncoder(b *testing.B) {
w := xdr.NewWriter(ioutil.Discard)
for i := 0; i < b.N; i++ {
_, err := s.EncodeXDRInto(w)
if err != nil {
b.Fatal(err)
}
}
}
type repeatReader struct {
data []byte
}
func (r *repeatReader) Read(bs []byte) (n int, err error) {
if len(bs) > len(r.data) {
err = io.EOF
}
n = copy(bs, r.data)
r.data = r.data[n:]
return n, err
}
func (r *repeatReader) Reset(bs []byte) {
r.data = bs
}
func BenchmarkThisDecode(b *testing.B) {
rr := &repeatReader{e}
var t XDRBenchStruct
for i := 0; i < b.N; i++ {
err := t.DecodeXDR(rr)
if err != nil {
b.Fatal(err)
}
rr.Reset(e)
}
}
func BenchmarkThisDecoder(b *testing.B) {
rr := &repeatReader{e}
r := xdr.NewReader(rr)
var t XDRBenchStruct
for i := 0; i < b.N; i++ {
err := t.DecodeXDRFrom(r)
if err != nil {
b.Fatal(err)
}
rr.Reset(e)
}
}
-201
View File
@@ -1,201 +0,0 @@
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package xdr_test
import (
"bytes"
"io"
"github.com/calmh/xdr"
)
/*
XDRBenchStruct Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ I1 (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| I2 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| 0x0000 | I3 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ uint8 Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of Bs0 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Bs0 (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of Bs1 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Bs1 (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of S0 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ S0 (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of S1 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ S1 (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct XDRBenchStruct {
unsigned hyper I1;
unsigned int I2;
unsigned int I3;
uint8 I4;
opaque Bs0<128>;
opaque Bs1<>;
string S0<128>;
string S1<>;
}
*/
func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.EncodeXDRInto(xw)
}
func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o XDRBenchStruct) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
_, err := o.EncodeXDRInto(xw)
return []byte(aw), err
}
func (o XDRBenchStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
xw.WriteUint64(o.I1)
xw.WriteUint32(o.I2)
xw.WriteUint16(o.I3)
xw.WriteUint8(o.I4)
if l := len(o.Bs0); l > 128 {
return xw.Tot(), xdr.ElementSizeExceeded("Bs0", l, 128)
}
xw.WriteBytes(o.Bs0)
xw.WriteBytes(o.Bs1)
if l := len(o.S0); l > 128 {
return xw.Tot(), xdr.ElementSizeExceeded("S0", l, 128)
}
xw.WriteString(o.S0)
xw.WriteString(o.S1)
return xw.Tot(), xw.Error()
}
func (o *XDRBenchStruct) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.DecodeXDRFrom(xr)
}
func (o *XDRBenchStruct) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.DecodeXDRFrom(xr)
}
func (o *XDRBenchStruct) DecodeXDRFrom(xr *xdr.Reader) error {
o.I1 = xr.ReadUint64()
o.I2 = xr.ReadUint32()
o.I3 = xr.ReadUint16()
o.I4 = xr.ReadUint8()
o.Bs0 = xr.ReadBytesMax(128)
o.Bs1 = xr.ReadBytes()
o.S0 = xr.ReadStringMax(128)
o.S1 = xr.ReadString()
return xr.Error()
}
/*
repeatReader Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of data |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ data (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct repeatReader {
opaque data<>;
}
*/
func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.EncodeXDRInto(xw)
}
func (o repeatReader) MarshalXDR() ([]byte, error) {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o repeatReader) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
_, err := o.EncodeXDRInto(xw)
return []byte(aw), err
}
func (o repeatReader) EncodeXDRInto(xw *xdr.Writer) (int, error) {
xw.WriteBytes(o.data)
return xw.Tot(), xw.Error()
}
func (o *repeatReader) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.DecodeXDRFrom(xr)
}
func (o *repeatReader) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.DecodeXDRFrom(xr)
}
func (o *repeatReader) DecodeXDRFrom(xr *xdr.Reader) error {
o.data = xr.ReadBytes()
return xr.Error()
}
-467
View File
@@ -1,467 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package main
import (
"bytes"
"flag"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"io"
"log"
"os"
"regexp"
"strconv"
"strings"
"text/template"
)
type fieldInfo struct {
Name string
IsBasic bool // handled by one the native Read/WriteUint64 etc functions
IsSlice bool // field is a slice of FieldType
FieldType string // original type of field, i.e. "int"
Encoder string // the encoder name, i.e. "Uint64" for Read/WriteUint64
Convert string // what to convert to when encoding, i.e. "uint64"
Max int // max size for slices and strings
}
type structInfo struct {
Name string
Fields []fieldInfo
}
var headerTpl = template.Must(template.New("header").Parse(`// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package {{.Package}}
import (
"bytes"
"io"
"github.com/calmh/xdr"
)
`))
var encodeTpl = template.Must(template.New("encoder").Parse(`
func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.EncodeXDRInto(xw)
}//+n
func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
return o.AppendXDR(make([]byte, 0, 128))
}//+n
func (o {{.TypeName}}) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}//+n
func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
_, err := o.EncodeXDRInto(xw)
return []byte(aw), err
}//+n
func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
{{range $fieldInfo := .Fields}}
{{if not $fieldInfo.IsSlice}}
{{if ne $fieldInfo.Convert ""}}
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
{{else if $fieldInfo.IsBasic}}
{{if ge $fieldInfo.Max 1}}
if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
}
{{end}}
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
{{else}}
_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
if err != nil {
return xw.Tot(), err
}
{{end}}
{{else}}
{{if ge $fieldInfo.Max 1}}
if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
}
{{end}}
xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
for i := range o.{{$fieldInfo.Name}} {
{{if ne $fieldInfo.Convert ""}}
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}[i]))
{{else if $fieldInfo.IsBasic}}
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
{{else}}
_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
if err != nil {
return xw.Tot(), err
}
{{end}}
}
{{end}}
{{end}}
return xw.Tot(), xw.Error()
}//+n
func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.DecodeXDRFrom(xr)
}//+n
func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.DecodeXDRFrom(xr)
}//+n
func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
{{range $fieldInfo := .Fields}}
{{if not $fieldInfo.IsSlice}}
{{if ne $fieldInfo.Convert ""}}
o.{{$fieldInfo.Name}} = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
{{else if $fieldInfo.IsBasic}}
{{if ge $fieldInfo.Max 1}}
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Max}})
{{else}}
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
{{end}}
{{else}}
(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
{{end}}
{{else}}
_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
if _{{$fieldInfo.Name}}Size < 0 {
return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
}
{{if ge $fieldInfo.Max 1}}
if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
}
{{end}}
o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
for i := range o.{{$fieldInfo.Name}} {
{{if ne $fieldInfo.Convert ""}}
o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
{{else if $fieldInfo.IsBasic}}
o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
{{else}}
(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
{{end}}
}
{{end}}
{{end}}
return xr.Error()
}`))
var maxRe = regexp.MustCompile(`\Wmax:(\d+)`)
type typeSet struct {
Type string
Encoder string
}
var xdrEncoders = map[string]typeSet{
"int8": typeSet{"uint8", "Uint8"},
"uint8": typeSet{"", "Uint8"},
"int16": typeSet{"uint16", "Uint16"},
"uint16": typeSet{"", "Uint16"},
"int32": typeSet{"uint32", "Uint32"},
"uint32": typeSet{"", "Uint32"},
"int64": typeSet{"uint64", "Uint64"},
"uint64": typeSet{"", "Uint64"},
"int": typeSet{"uint64", "Uint64"},
"string": typeSet{"", "String"},
"[]byte": typeSet{"", "Bytes"},
"bool": typeSet{"", "Bool"},
}
func handleStruct(t *ast.StructType) []fieldInfo {
var fs []fieldInfo
for _, sf := range t.Fields.List {
if len(sf.Names) == 0 {
// We don't handle anonymous fields
continue
}
fn := sf.Names[0].Name
var max = 0
if sf.Comment != nil {
c := sf.Comment.List[0].Text
if m := maxRe.FindStringSubmatch(c); m != nil {
max, _ = strconv.Atoi(m[1])
}
if strings.Contains(c, "noencode") {
continue
}
}
var f fieldInfo
switch ft := sf.Type.(type) {
case *ast.Ident:
tn := ft.Name
if enc, ok := xdrEncoders[tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max,
}
} else {
f = fieldInfo{
Name: fn,
IsBasic: false,
FieldType: tn,
Max: max,
}
}
case *ast.ArrayType:
if ft.Len != nil {
// We don't handle arrays
continue
}
tn := ft.Elt.(*ast.Ident).Name
if enc, ok := xdrEncoders["[]"+tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max,
}
} else if enc, ok := xdrEncoders[tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
IsSlice: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max,
}
} else {
f = fieldInfo{
Name: fn,
IsSlice: true,
FieldType: tn,
Max: max,
}
}
case *ast.SelectorExpr:
f = fieldInfo{
Name: fn,
FieldType: ft.Sel.Name,
Max: max,
}
}
fs = append(fs, f)
}
return fs
}
func generateCode(output io.Writer, s structInfo) {
name := s.Name
fs := s.Fields
var buf bytes.Buffer
err := encodeTpl.Execute(&buf, map[string]interface{}{"TypeName": name, "Fields": fs})
if err != nil {
panic(err)
}
bs := regexp.MustCompile(`(\s*\n)+`).ReplaceAll(buf.Bytes(), []byte("\n"))
bs = bytes.Replace(bs, []byte("//+n"), []byte("\n"), -1)
bs, err = format.Source(bs)
if err != nil {
panic(err)
}
fmt.Fprintln(output, string(bs))
}
func uncamelize(s string) string {
return regexp.MustCompile("[a-z][A-Z]").ReplaceAllStringFunc(s, func(camel string) string {
return camel[:1] + " " + camel[1:]
})
}
func generateDiagram(output io.Writer, s structInfo) {
sn := s.Name
fs := s.Fields
fmt.Fprintln(output, sn+" Structure:")
fmt.Fprintln(output)
fmt.Fprintln(output, " 0 1 2 3")
fmt.Fprintln(output, " 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
line := "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"
fmt.Fprintln(output, line)
for _, f := range fs {
tn := f.FieldType
name := uncamelize(f.Name)
if f.IsSlice {
fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
fmt.Fprintln(output, line)
}
switch tn {
case "bool":
fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
fmt.Fprintln(output, line)
case "int16", "uint16":
fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
fmt.Fprintln(output, line)
case "int32", "uint32":
fmt.Fprintf(output, "| %s |\n", center(name, 61))
fmt.Fprintln(output, line)
case "int64", "uint64":
fmt.Fprintf(output, "| %-61s |\n", "")
fmt.Fprintf(output, "+ %s +\n", center(name+" (64 bits)", 61))
fmt.Fprintf(output, "| %-61s |\n", "")
fmt.Fprintln(output, line)
case "string", "byte": // XXX We assume slice of byte!
fmt.Fprintf(output, "| %s |\n", center("Length of "+name, 61))
fmt.Fprintln(output, line)
fmt.Fprintf(output, "/ %61s /\n", "")
fmt.Fprintf(output, "\\ %s \\\n", center(name+" (variable length)", 61))
fmt.Fprintf(output, "/ %61s /\n", "")
fmt.Fprintln(output, line)
default:
if f.IsSlice {
tn = "Zero or more " + tn + " Structures"
fmt.Fprintf(output, "/ %s /\n", center("", 61))
fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
fmt.Fprintf(output, "/ %s /\n", center("", 61))
} else {
tn = tn + " Structure"
fmt.Fprintf(output, "/ %s /\n", center("", 61))
fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
fmt.Fprintf(output, "/ %s /\n", center("", 61))
}
fmt.Fprintln(output, line)
}
}
fmt.Fprintln(output)
fmt.Fprintln(output)
}
func generateXdr(output io.Writer, s structInfo) {
sn := s.Name
fs := s.Fields
fmt.Fprintf(output, "struct %s {\n", sn)
for _, f := range fs {
tn := f.FieldType
fn := f.Name
suf := ""
l := ""
if f.Max > 0 {
l = strconv.Itoa(f.Max)
}
if f.IsSlice {
suf = "<" + l + ">"
}
switch tn {
case "int16", "int32":
fmt.Fprintf(output, "\tint %s%s;\n", fn, suf)
case "uint16", "uint32":
fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
case "int64":
fmt.Fprintf(output, "\thyper %s%s;\n", fn, suf)
case "uint64":
fmt.Fprintf(output, "\tunsigned hyper %s%s;\n", fn, suf)
case "string":
fmt.Fprintf(output, "\tstring %s<%s>;\n", fn, l)
case "byte":
fmt.Fprintf(output, "\topaque %s<%s>;\n", fn, l)
default:
fmt.Fprintf(output, "\t%s %s%s;\n", tn, fn, suf)
}
}
fmt.Fprintln(output, "}")
fmt.Fprintln(output)
}
func center(s string, w int) string {
w -= len(s)
l := w / 2
r := l
if l+r < w {
r++
}
return strings.Repeat(" ", l) + s + strings.Repeat(" ", r)
}
func inspector(structs *[]structInfo) func(ast.Node) bool {
return func(n ast.Node) bool {
switch n := n.(type) {
case *ast.TypeSpec:
switch t := n.Type.(type) {
case *ast.StructType:
name := n.Name.Name
fs := handleStruct(t)
*structs = append(*structs, structInfo{name, fs})
}
return false
default:
return true
}
}
}
func main() {
outputFile := flag.String("o", "", "Output file, blank for stdout")
flag.Parse()
fname := flag.Arg(0)
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
if err != nil {
log.Fatal(err)
}
var structs []structInfo
i := inspector(&structs)
ast.Inspect(f, i)
var output io.Writer = os.Stdout
if *outputFile != "" {
fd, err := os.Create(*outputFile)
if err != nil {
log.Fatal(err)
}
output = fd
}
headerTpl.Execute(output, map[string]string{"Package": f.Name.Name})
for _, s := range structs {
fmt.Fprintf(output, "\n/*\n\n")
generateDiagram(output, s)
generateXdr(output, s)
fmt.Fprintf(output, "*/\n")
generateCode(output, s)
}
}
-16
View File
@@ -1,16 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package xdr
import (
"log"
"os"
)
var (
debug = len(os.Getenv("XDRTRACE")) > 0
dl = log.New(os.Stdout, "xdr: ", log.Lshortfile|log.Ltime|log.Lmicroseconds)
)
const maxDebugBytes = 32
-5
View File
@@ -1,5 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// Package xdr implements an XDR (RFC 4506) encoder/decoder.
package xdr
-79
View File
@@ -1,79 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package xdr_test
import (
"bytes"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/calmh/xdr"
)
// Contains all supported types
type TestStruct struct {
I int
I8 int8
UI8 uint8
I16 int16
UI16 uint16
I32 int32
UI32 uint32
I64 int64
UI64 uint64
BS []byte // max:1024
S string // max:1024
C Opaque
SS []string // max:1024
}
type Opaque [32]byte
func (u *Opaque) EncodeXDRInto(w *xdr.Writer) (int, error) {
return w.WriteRaw(u[:])
}
func (u *Opaque) DecodeXDRFrom(r *xdr.Reader) (int, error) {
return r.ReadRaw(u[:])
}
func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
var u Opaque
for i := range u[:] {
u[i] = byte(rand.Int())
}
return reflect.ValueOf(u)
}
func TestEncDec(t *testing.T) {
fn := func(t0 TestStruct) bool {
bs, err := t0.MarshalXDR()
if err != nil {
t.Fatal(err)
}
var t1 TestStruct
err = t1.UnmarshalXDR(bs)
if err != nil {
t.Fatal(err)
}
// Not comparing with DeepEqual since we'll unmarshal nil slices as empty
if t0.I != t1.I ||
t0.I16 != t1.I16 || t0.UI16 != t1.UI16 ||
t0.I32 != t1.I32 || t0.UI32 != t1.UI32 ||
t0.I64 != t1.I64 || t0.UI64 != t1.UI64 ||
bytes.Compare(t0.BS, t1.BS) != 0 ||
t0.S != t1.S || t0.C != t1.C {
t.Logf("%#v", t0)
t.Logf("%#v", t1)
return false
}
return true
}
if err := quick.Check(fn, nil); err != nil {
t.Error(err)
}
}
-185
View File
@@ -1,185 +0,0 @@
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package xdr_test
import (
"bytes"
"io"
"github.com/calmh/xdr"
)
/*
TestStruct Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ int Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ int8 Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ uint8 Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| 0x0000 | I16 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| 0x0000 | UI16 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| I32 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| UI32 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ I64 (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ UI64 (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of BS |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ BS (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of S |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ S (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Opaque Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of SS |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of SS |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ SS (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct TestStruct {
int I;
int8 I8;
uint8 UI8;
int I16;
unsigned int UI16;
int I32;
unsigned int UI32;
hyper I64;
unsigned hyper UI64;
opaque BS<1024>;
string S<1024>;
Opaque C;
string SS<1024>;
}
*/
func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.EncodeXDRInto(xw)
}
func (o TestStruct) MarshalXDR() ([]byte, error) {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o TestStruct) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
_, err := o.EncodeXDRInto(xw)
return []byte(aw), err
}
func (o TestStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
xw.WriteUint64(uint64(o.I))
xw.WriteUint8(uint8(o.I8))
xw.WriteUint8(o.UI8)
xw.WriteUint16(uint16(o.I16))
xw.WriteUint16(o.UI16)
xw.WriteUint32(uint32(o.I32))
xw.WriteUint32(o.UI32)
xw.WriteUint64(uint64(o.I64))
xw.WriteUint64(o.UI64)
if l := len(o.BS); l > 1024 {
return xw.Tot(), xdr.ElementSizeExceeded("BS", l, 1024)
}
xw.WriteBytes(o.BS)
if l := len(o.S); l > 1024 {
return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
}
xw.WriteString(o.S)
_, err := o.C.EncodeXDRInto(xw)
if err != nil {
return xw.Tot(), err
}
if l := len(o.SS); l > 1024 {
return xw.Tot(), xdr.ElementSizeExceeded("SS", l, 1024)
}
xw.WriteUint32(uint32(len(o.SS)))
for i := range o.SS {
xw.WriteString(o.SS[i])
}
return xw.Tot(), xw.Error()
}
func (o *TestStruct) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.DecodeXDRFrom(xr)
}
func (o *TestStruct) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.DecodeXDRFrom(xr)
}
func (o *TestStruct) DecodeXDRFrom(xr *xdr.Reader) error {
o.I = int(xr.ReadUint64())
o.I8 = int8(xr.ReadUint8())
o.UI8 = xr.ReadUint8()
o.I16 = int16(xr.ReadUint16())
o.UI16 = xr.ReadUint16()
o.I32 = int32(xr.ReadUint32())
o.UI32 = xr.ReadUint32()
o.I64 = int64(xr.ReadUint64())
o.UI64 = xr.ReadUint64()
o.BS = xr.ReadBytesMax(1024)
o.S = xr.ReadStringMax(1024)
(&o.C).DecodeXDRFrom(xr)
_SSSize := int(xr.ReadUint32())
if _SSSize < 0 {
return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
}
if _SSSize > 1024 {
return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
}
o.SS = make([]string, _SSSize)
for i := range o.SS {
o.SS[i] = xr.ReadString()
}
return xr.Error()
}
-4
View File
@@ -1,4 +0,0 @@
#!/bin/sh
go run cmd/genxdr/main.go -- bench_test.go > bench_xdr_test.go
go run cmd/genxdr/main.go -- encdec_test.go > encdec_xdr_test.go
-10
View File
@@ -1,10 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build ipdr
package xdr
func pad(l int) int {
return 0
}
-14
View File
@@ -1,14 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build !ipdr
package xdr
func pad(l int) int {
d := l % 4
if d == 0 {
return 0
}
return 4 - d
}
-171
View File
@@ -1,171 +0,0 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package xdr
import (
"fmt"
"io"
"reflect"
"unsafe"
)
type Reader struct {
r io.Reader
err error
b [8]byte
}
func NewReader(r io.Reader) *Reader {
return &Reader{
r: r,
}
}
func (r *Reader) ReadRaw(bs []byte) (int, error) {
if r.err != nil {
return 0, r.err
}
var n int
n, r.err = io.ReadFull(r.r, bs)
return n, r.err
}
func (r *Reader) ReadString() string {
return r.ReadStringMax(0)
}
func (r *Reader) ReadStringMax(max int) string {
buf := r.ReadBytesMaxInto(max, nil)
bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
sh := reflect.StringHeader{
Data: bh.Data,
Len: bh.Len,
}
return *((*string)(unsafe.Pointer(&sh)))
}
func (r *Reader) ReadBytes() []byte {
return r.ReadBytesInto(nil)
}
func (r *Reader) ReadBytesMax(max int) []byte {
return r.ReadBytesMaxInto(max, nil)
}
func (r *Reader) ReadBytesInto(dst []byte) []byte {
return r.ReadBytesMaxInto(0, dst)
}
func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
if r.err != nil {
return nil
}
l := int(r.ReadUint32())
if r.err != nil {
return nil
}
if l < 0 || max > 0 && l > max {
// l may be negative on 32 bit builds
r.err = ElementSizeExceeded("bytes field", l, max)
return nil
}
if fullLen := l + pad(l); fullLen > len(dst) {
dst = make([]byte, fullLen)
} else {
dst = dst[:fullLen]
}
var n int
n, r.err = io.ReadFull(r.r, dst)
if r.err != nil {
if debug {
dl.Printf("rd bytes (%d): %v", len(dst), r.err)
}
return nil
}
if debug {
if n > maxDebugBytes {
dl.Printf("rd bytes (%d): %x...", len(dst), dst[:maxDebugBytes])
} else {
dl.Printf("rd bytes (%d): %x", len(dst), dst)
}
}
return dst[:l]
}
func (r *Reader) ReadBool() bool {
return r.ReadUint8() != 0
}
func (r *Reader) ReadUint32() uint32 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:4])
if r.err != nil {
if debug {
dl.Printf("rd uint32: %v", r.err)
}
return 0
}
v := uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
if debug {
dl.Printf("rd uint32=%d (0x%08x)", v, v)
}
return v
}
func (r *Reader) ReadUint64() uint64 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:8])
if r.err != nil {
if debug {
dl.Printf("rd uint64: %v", r.err)
}
return 0
}
v := uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
if debug {
dl.Printf("rd uint64=%d (0x%016x)", v, v)
}
return v
}
type XDRError struct {
op string
err error
}
func (e XDRError) Error() string {
return "xdr " + e.op + ": " + e.err.Error()
}
func (e XDRError) IsEOF() bool {
return e.err == io.EOF
}
func (r *Reader) Error() error {
if r.err == nil {
return nil
}
return XDRError{"read", r.err}
}
func ElementSizeExceeded(field string, size, limit int) error {
return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
}
-49
View File
@@ -1,49 +0,0 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
// +build ipdr
package xdr
import "io"
func (r *Reader) ReadUint8() uint8 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:1])
if r.err != nil {
if debug {
dl.Printf("rd uint8: %v", r.err)
}
return 0
}
if debug {
dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
}
return r.b[0]
}
func (r *Reader) ReadUint16() uint16 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:2])
if r.err != nil {
if debug {
dl.Printf("rd uint16: %v", r.err)
}
return 0
}
v := uint16(r.b[1]) | uint16(r.b[0])<<8
if debug {
dl.Printf("rd uint16=%d (0x%04x)", v, v)
}
return v
}
-15
View File
@@ -1,15 +0,0 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
// +build !ipdr
package xdr
func (r *Reader) ReadUint8() uint8 {
return uint8(r.ReadUint32())
}
func (r *Reader) ReadUint16() uint16 {
return uint16(r.ReadUint32())
}
-44
View File
@@ -1,44 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build refl
package xdr_test
import (
"bytes"
"testing"
refl "github.com/davecgh/go-xdr/xdr"
)
func TestCompareMarshals(t *testing.T) {
e0 := s.MarshalXDR()
e1, err := refl.Marshal(s)
if err != nil {
t.Fatal(err)
}
if bytes.Compare(e0, e1) != 0 {
t.Fatalf("Encoding mismatch;\n\t%x (this)\n\t%x (refl)", e0, e1)
}
}
func BenchmarkReflMarshal(b *testing.B) {
var err error
for i := 0; i < b.N; i++ {
res, err = refl.Marshal(s)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkReflUnmarshal(b *testing.B) {
var t XDRBenchStruct
for i := 0; i < b.N; i++ {
_, err := refl.Unmarshal(e, &t)
if err != nil {
b.Fatal(err)
}
}
}
-146
View File
@@ -1,146 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package xdr
import (
"io"
"reflect"
"unsafe"
)
var padBytes = []byte{0, 0, 0}
type Writer struct {
w io.Writer
tot int
err error
b [8]byte
}
type AppendWriter []byte
func (w *AppendWriter) Write(bs []byte) (int, error) {
*w = append(*w, bs...)
return len(bs), nil
}
func NewWriter(w io.Writer) *Writer {
return &Writer{
w: w,
}
}
func (w *Writer) WriteRaw(bs []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
var n int
n, w.err = w.w.Write(bs)
return n, w.err
}
func (w *Writer) WriteString(s string) (int, error) {
sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
bh := reflect.SliceHeader{
Data: sh.Data,
Len: sh.Len,
Cap: sh.Len,
}
return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
}
func (w *Writer) WriteBytes(bs []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
w.WriteUint32(uint32(len(bs)))
if w.err != nil {
return 0, w.err
}
if debug {
if len(bs) > maxDebugBytes {
dl.Printf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
} else {
dl.Printf("wr bytes (%d): %x", len(bs), bs)
}
}
var l, n int
n, w.err = w.w.Write(bs)
l += n
if p := pad(len(bs)); w.err == nil && p > 0 {
n, w.err = w.w.Write(padBytes[:p])
l += n
}
w.tot += l
return l, w.err
}
func (w *Writer) WriteBool(v bool) (int, error) {
if v {
return w.WriteUint8(1)
} else {
return w.WriteUint8(0)
}
}
func (w *Writer) WriteUint32(v uint32) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint32=%d", v)
}
w.b[0] = byte(v >> 24)
w.b[1] = byte(v >> 16)
w.b[2] = byte(v >> 8)
w.b[3] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:4])
w.tot += l
return l, w.err
}
func (w *Writer) WriteUint64(v uint64) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint64=%d", v)
}
w.b[0] = byte(v >> 56)
w.b[1] = byte(v >> 48)
w.b[2] = byte(v >> 40)
w.b[3] = byte(v >> 32)
w.b[4] = byte(v >> 24)
w.b[5] = byte(v >> 16)
w.b[6] = byte(v >> 8)
w.b[7] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:8])
w.tot += l
return l, w.err
}
func (w *Writer) Tot() int {
return w.tot
}
func (w *Writer) Error() error {
if w.err == nil {
return nil
}
return XDRError{"write", w.err}
}
-41
View File
@@ -1,41 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build ipdr
package xdr
func (w *Writer) WriteUint8(v uint8) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint8=%d", v)
}
w.b[0] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:1])
w.tot += l
return l, w.err
}
func (w *Writer) WriteUint16(v uint16) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint8=%d", v)
}
w.b[0] = byte(v >> 8)
w.b[1] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:2])
w.tot += l
return l, w.err
}
-14
View File
@@ -1,14 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build !ipdr
package xdr
func (w *Writer) WriteUint8(v uint8) (int, error) {
return w.WriteUint32(uint32(v))
}
func (w *Writer) WriteUint16(v uint16) (int, error) {
return w.WriteUint32(uint32(v))
}
-93
View File
@@ -1,93 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package xdr
import (
"bytes"
"strings"
"testing"
"testing/quick"
)
func TestBytesNil(t *testing.T) {
fn := func(bs []byte) bool {
var b = new(bytes.Buffer)
var w = NewWriter(b)
var r = NewReader(b)
w.WriteBytes(bs)
w.WriteBytes(bs)
r.ReadBytes()
res := r.ReadBytes()
return bytes.Compare(bs, res) == 0
}
if err := quick.Check(fn, nil); err != nil {
t.Error(err)
}
}
func TestBytesGiven(t *testing.T) {
fn := func(bs []byte) bool {
var b = new(bytes.Buffer)
var w = NewWriter(b)
var r = NewReader(b)
w.WriteBytes(bs)
w.WriteBytes(bs)
res := make([]byte, 12)
res = r.ReadBytesInto(res)
res = r.ReadBytesInto(res)
return bytes.Compare(bs, res) == 0
}
if err := quick.Check(fn, nil); err != nil {
t.Error(err)
}
}
func TestReadBytesMaxInto(t *testing.T) {
var max = 64
for tot := 32; tot < 128; tot++ {
for diff := -32; diff <= 32; diff++ {
var b = new(bytes.Buffer)
var r = NewReader(b)
var w = NewWriter(b)
var toWrite = make([]byte, tot)
w.WriteBytes(toWrite)
var buf = make([]byte, tot+diff)
var bs = r.ReadBytesMaxInto(max, buf)
if tot <= max {
if read := len(bs); read != tot {
t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
}
} else if !strings.Contains(r.err.Error(), "exceeds size") {
t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
}
}
}
}
func TestReadStringMax(t *testing.T) {
for tot := 42; tot < 72; tot++ {
for max := 0; max < 128; max++ {
var b = new(bytes.Buffer)
var r = NewReader(b)
var w = NewWriter(b)
var toWrite = make([]byte, tot)
w.WriteBytes(toWrite)
var str = r.ReadStringMax(max)
var read = len(str)
if max == 0 || tot <= max {
if read != tot {
t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
}
} else if !strings.Contains(r.err.Error(), "exceeds size") {
t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
}
}
}
}
-188
View File
@@ -1,188 +0,0 @@
This package contains an efficient token-bucket-based rate limiter.
Copyright (C) 2015 Canonical Ltd.
This software is licensed under the LGPLv3, included below.
As a special exception to the GNU Lesser General Public License version 3
("LGPL3"), the copyright holders of this Library give you permission to
convey to a third party a Combined Work that links statically or dynamically
to this Library without providing any Minimal Corresponding Source or
Minimal Application Code as set out in 4d or providing the installation
information set out in section 4e, provided that you comply with the other
provisions of LGPL3 and provided that you meet, for the Application the
terms and conditions of the license(s) which apply to the Application.
Except as stated in this special exception, the provisions of LGPL3 will
continue to comply in full to this Library. If you modify this Library, you
may apply this exception to your version of this Library, but you are not
obliged to do so. If you do not wish to do so, delete this exception
statement from your version. This exception does not (and cannot) modify any
license terms which apply to the Application, with which you must still
comply.
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
-117
View File
@@ -1,117 +0,0 @@
# ratelimit
--
import "github.com/juju/ratelimit"
The ratelimit package provides an efficient token bucket implementation. See
http://en.wikipedia.org/wiki/Token_bucket.
## Usage
#### func Reader
```go
func Reader(r io.Reader, bucket *Bucket) io.Reader
```
Reader returns a reader that is rate limited by the given token bucket. Each
token in the bucket represents one byte.
#### func Writer
```go
func Writer(w io.Writer, bucket *Bucket) io.Writer
```
Writer returns a reader that is rate limited by the given token bucket. Each
token in the bucket represents one byte.
#### type Bucket
```go
type Bucket struct {
}
```
Bucket represents a token bucket that fills at a predetermined rate. Methods on
Bucket may be called concurrently.
#### func NewBucket
```go
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket
```
NewBucket returns a new token bucket that fills at the rate of one token every
fillInterval, up to the given maximum capacity. Both arguments must be positive.
The bucket is initially full.
#### func NewBucketWithQuantum
```go
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket
```
NewBucketWithQuantum is similar to NewBucket, but allows the specification of
the quantum size - quantum tokens are added every fillInterval.
#### func NewBucketWithRate
```go
func NewBucketWithRate(rate float64, capacity int64) *Bucket
```
NewBucketWithRate returns a token bucket that fills the bucket at the rate of
rate tokens per second up to the given maximum capacity. Because of limited
clock resolution, at high rates, the actual rate may be up to 1% different from
the specified rate.
#### func (*Bucket) Rate
```go
func (tb *Bucket) Rate() float64
```
Rate returns the fill rate of the bucket, in tokens per second.
#### func (*Bucket) Take
```go
func (tb *Bucket) Take(count int64) time.Duration
```
Take takes count tokens from the bucket without blocking. It returns the time
that the caller should wait until the tokens are actually available.
Note that if the request is irrevocable - there is no way to return tokens to
the bucket once this method commits us to taking them.
#### func (*Bucket) TakeAvailable
```go
func (tb *Bucket) TakeAvailable(count int64) int64
```
TakeAvailable takes up to count immediately available tokens from the bucket. It
returns the number of tokens removed, or zero if there are no available tokens.
It does not block.
#### func (*Bucket) TakeMaxDuration
```go
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool)
```
TakeMaxDuration is like Take, except that it will only take tokens from the
bucket if the wait time for the tokens is no greater than maxWait.
If it would take longer than maxWait for the tokens to become available, it does
nothing and reports false, otherwise it returns the time that the caller should
wait until the tokens are actually available, and reports true.
#### func (*Bucket) Wait
```go
func (tb *Bucket) Wait(count int64)
```
Wait takes count tokens from the bucket, waiting until they are available.
#### func (*Bucket) WaitMaxDuration
```go
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool
```
WaitMaxDuration is like Wait except that it will only take tokens from the
bucket if it needs to wait for no greater than maxWait. It reports whether any
tokens have been removed from the bucket If no tokens have been removed, it
returns immediately.
-226
View File
@@ -1,226 +0,0 @@
// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3 with static-linking exception.
// See LICENCE file for details.
// The ratelimit package provides an efficient token bucket implementation.
// See http://en.wikipedia.org/wiki/Token_bucket.
package ratelimit
import (
"strconv"
"sync"
"time"
)
// Bucket represents a token bucket that fills at a predetermined rate.
// Methods on Bucket may be called concurrently.
type Bucket struct {
startTime time.Time
capacity int64
quantum int64
fillInterval time.Duration
// The mutex guards the fields following it.
mu sync.Mutex
// avail holds the number of available tokens
// in the bucket, as of availTick ticks from startTime.
// It will be negative when there are consumers
// waiting for tokens.
avail int64
availTick int64
}
// NewBucket returns a new token bucket that fills at the
// rate of one token every fillInterval, up to the given
// maximum capacity. Both arguments must be
// positive. The bucket is initially full.
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket {
return NewBucketWithQuantum(fillInterval, capacity, 1)
}
// rateMargin specifes the allowed variance of actual
// rate from specified rate. 1% seems reasonable.
const rateMargin = 0.01
// NewBucketWithRate returns a token bucket that fills the bucket
// at the rate of rate tokens per second up to the given
// maximum capacity. Because of limited clock resolution,
// at high rates, the actual rate may be up to 1% different from the
// specified rate.
func NewBucketWithRate(rate float64, capacity int64) *Bucket {
for quantum := int64(1); quantum < 1<<50; quantum = nextQuantum(quantum) {
fillInterval := time.Duration(1e9 * float64(quantum) / rate)
if fillInterval <= 0 {
continue
}
tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
if diff := abs(tb.Rate() - rate); diff/rate <= rateMargin {
return tb
}
}
panic("cannot find suitable quantum for " + strconv.FormatFloat(rate, 'g', -1, 64))
}
// nextQuantum returns the next quantum to try after q.
// We grow the quantum exponentially, but slowly, so we
// get a good fit in the lower numbers.
func nextQuantum(q int64) int64 {
q1 := q * 11 / 10
if q1 == q {
q1++
}
return q1
}
// NewBucketWithQuantum is similar to NewBucket, but allows
// the specification of the quantum size - quantum tokens
// are added every fillInterval.
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket {
if fillInterval <= 0 {
panic("token bucket fill interval is not > 0")
}
if capacity <= 0 {
panic("token bucket capacity is not > 0")
}
if quantum <= 0 {
panic("token bucket quantum is not > 0")
}
return &Bucket{
startTime: time.Now(),
capacity: capacity,
quantum: quantum,
avail: capacity,
fillInterval: fillInterval,
}
}
// Wait takes count tokens from the bucket, waiting until they are
// available.
func (tb *Bucket) Wait(count int64) {
if d := tb.Take(count); d > 0 {
time.Sleep(d)
}
}
// WaitMaxDuration is like Wait except that it will
// only take tokens from the bucket if it needs to wait
// for no greater than maxWait. It reports whether
// any tokens have been removed from the bucket
// If no tokens have been removed, it returns immediately.
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool {
d, ok := tb.TakeMaxDuration(count, maxWait)
if d > 0 {
time.Sleep(d)
}
return ok
}
const infinityDuration time.Duration = 0x7fffffffffffffff
// Take takes count tokens from the bucket without blocking. It returns
// the time that the caller should wait until the tokens are actually
// available.
//
// Note that if the request is irrevocable - there is no way to return
// tokens to the bucket once this method commits us to taking them.
func (tb *Bucket) Take(count int64) time.Duration {
d, _ := tb.take(time.Now(), count, infinityDuration)
return d
}
// TakeMaxDuration is like Take, except that
// it will only take tokens from the bucket if the wait
// time for the tokens is no greater than maxWait.
//
// If it would take longer than maxWait for the tokens
// to become available, it does nothing and reports false,
// otherwise it returns the time that the caller should
// wait until the tokens are actually available, and reports
// true.
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool) {
return tb.take(time.Now(), count, maxWait)
}
// TakeAvailable takes up to count immediately available tokens from the
// bucket. It returns the number of tokens removed, or zero if there are
// no available tokens. It does not block.
func (tb *Bucket) TakeAvailable(count int64) int64 {
return tb.takeAvailable(time.Now(), count)
}
// takeAvailable is the internal version of TakeAvailable - it takes the
// current time as an argument to enable easy testing.
func (tb *Bucket) takeAvailable(now time.Time, count int64) int64 {
if count <= 0 {
return 0
}
tb.mu.Lock()
defer tb.mu.Unlock()
tb.adjust(now)
if tb.avail <= 0 {
return 0
}
if count > tb.avail {
count = tb.avail
}
tb.avail -= count
return count
}
// Rate returns the fill rate of the bucket, in tokens per second.
func (tb *Bucket) Rate() float64 {
return 1e9 * float64(tb.quantum) / float64(tb.fillInterval)
}
// take is the internal version of Take - it takes the current time as
// an argument to enable easy testing.
func (tb *Bucket) take(now time.Time, count int64, maxWait time.Duration) (time.Duration, bool) {
if count <= 0 {
return 0, true
}
tb.mu.Lock()
defer tb.mu.Unlock()
currentTick := tb.adjust(now)
avail := tb.avail - count
if avail >= 0 {
tb.avail = avail
return 0, true
}
// Round up the missing tokens to the nearest multiple
// of quantum - the tokens won't be available until
// that tick.
endTick := currentTick + (-avail+tb.quantum-1)/tb.quantum
endTime := tb.startTime.Add(time.Duration(endTick) * tb.fillInterval)
waitTime := endTime.Sub(now)
if waitTime > maxWait {
return 0, false
}
tb.avail = avail
return waitTime, true
}
// adjust adjusts the current bucket capacity based on the current time.
// It returns the current tick.
func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
currentTick = int64(now.Sub(tb.startTime) / tb.fillInterval)
if tb.avail >= tb.capacity {
return
}
tb.avail += (currentTick - tb.availTick) * tb.quantum
if tb.avail > tb.capacity {
tb.avail = tb.capacity
}
tb.availTick = currentTick
return
}
func abs(f float64) float64 {
if f < 0 {
return -f
}
return f
}
-328
View File
@@ -1,328 +0,0 @@
// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3 with static-linking exception.
// See LICENCE file for details.
package ratelimit
import (
gc "launchpad.net/gocheck"
"testing"
"time"
)
func TestPackage(t *testing.T) {
gc.TestingT(t)
}
type rateLimitSuite struct{}
var _ = gc.Suite(rateLimitSuite{})
type takeReq struct {
time time.Duration
count int64
expectWait time.Duration
}
var takeTests = []struct {
about string
fillInterval time.Duration
capacity int64
reqs []takeReq
}{{
about: "serial requests",
fillInterval: 250 * time.Millisecond,
capacity: 10,
reqs: []takeReq{{
time: 0,
count: 0,
expectWait: 0,
}, {
time: 0,
count: 10,
expectWait: 0,
}, {
time: 0,
count: 1,
expectWait: 250 * time.Millisecond,
}, {
time: 250 * time.Millisecond,
count: 1,
expectWait: 250 * time.Millisecond,
}},
}, {
about: "concurrent requests",
fillInterval: 250 * time.Millisecond,
capacity: 10,
reqs: []takeReq{{
time: 0,
count: 10,
expectWait: 0,
}, {
time: 0,
count: 2,
expectWait: 500 * time.Millisecond,
}, {
time: 0,
count: 2,
expectWait: 1000 * time.Millisecond,
}, {
time: 0,
count: 1,
expectWait: 1250 * time.Millisecond,
}},
}, {
about: "more than capacity",
fillInterval: 1 * time.Millisecond,
capacity: 10,
reqs: []takeReq{{
time: 0,
count: 10,
expectWait: 0,
}, {
time: 20 * time.Millisecond,
count: 15,
expectWait: 5 * time.Millisecond,
}},
}, {
about: "sub-quantum time",
fillInterval: 10 * time.Millisecond,
capacity: 10,
reqs: []takeReq{{
time: 0,
count: 10,
expectWait: 0,
}, {
time: 7 * time.Millisecond,
count: 1,
expectWait: 3 * time.Millisecond,
}, {
time: 8 * time.Millisecond,
count: 1,
expectWait: 12 * time.Millisecond,
}},
}, {
about: "within capacity",
fillInterval: 10 * time.Millisecond,
capacity: 5,
reqs: []takeReq{{
time: 0,
count: 5,
expectWait: 0,
}, {
time: 60 * time.Millisecond,
count: 5,
expectWait: 0,
}, {
time: 60 * time.Millisecond,
count: 1,
expectWait: 10 * time.Millisecond,
}, {
time: 80 * time.Millisecond,
count: 2,
expectWait: 10 * time.Millisecond,
}},
}}
func (rateLimitSuite) TestTake(c *gc.C) {
for i, test := range takeTests {
tb := NewBucket(test.fillInterval, test.capacity)
for j, req := range test.reqs {
d, ok := tb.take(tb.startTime.Add(req.time), req.count, infinityDuration)
c.Assert(ok, gc.Equals, true)
if d != req.expectWait {
c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
}
}
}
}
func (rateLimitSuite) TestTakeMaxDuration(c *gc.C) {
for i, test := range takeTests {
tb := NewBucket(test.fillInterval, test.capacity)
for j, req := range test.reqs {
if req.expectWait > 0 {
d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait-1)
c.Assert(ok, gc.Equals, false)
c.Assert(d, gc.Equals, time.Duration(0))
}
d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait)
c.Assert(ok, gc.Equals, true)
if d != req.expectWait {
c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
}
}
}
}
type takeAvailableReq struct {
time time.Duration
count int64
expect int64
}
var takeAvailableTests = []struct {
about string
fillInterval time.Duration
capacity int64
reqs []takeAvailableReq
}{{
about: "serial requests",
fillInterval: 250 * time.Millisecond,
capacity: 10,
reqs: []takeAvailableReq{{
time: 0,
count: 0,
expect: 0,
}, {
time: 0,
count: 10,
expect: 10,
}, {
time: 0,
count: 1,
expect: 0,
}, {
time: 250 * time.Millisecond,
count: 1,
expect: 1,
}},
}, {
about: "concurrent requests",
fillInterval: 250 * time.Millisecond,
capacity: 10,
reqs: []takeAvailableReq{{
time: 0,
count: 5,
expect: 5,
}, {
time: 0,
count: 2,
expect: 2,
}, {
time: 0,
count: 5,
expect: 3,
}, {
time: 0,
count: 1,
expect: 0,
}},
}, {
about: "more than capacity",
fillInterval: 1 * time.Millisecond,
capacity: 10,
reqs: []takeAvailableReq{{
time: 0,
count: 10,
expect: 10,
}, {
time: 20 * time.Millisecond,
count: 15,
expect: 10,
}},
}, {
about: "within capacity",
fillInterval: 10 * time.Millisecond,
capacity: 5,
reqs: []takeAvailableReq{{
time: 0,
count: 5,
expect: 5,
}, {
time: 60 * time.Millisecond,
count: 5,
expect: 5,
}, {
time: 70 * time.Millisecond,
count: 1,
expect: 1,
}},
}}
func (rateLimitSuite) TestTakeAvailable(c *gc.C) {
for i, test := range takeAvailableTests {
tb := NewBucket(test.fillInterval, test.capacity)
for j, req := range test.reqs {
d := tb.takeAvailable(tb.startTime.Add(req.time), req.count)
if d != req.expect {
c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expect)
}
}
}
}
func (rateLimitSuite) TestPanics(c *gc.C) {
c.Assert(func() { NewBucket(0, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
c.Assert(func() { NewBucket(-2, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
c.Assert(func() { NewBucket(1, 0) }, gc.PanicMatches, "token bucket capacity is not > 0")
c.Assert(func() { NewBucket(1, -2) }, gc.PanicMatches, "token bucket capacity is not > 0")
}
func isCloseTo(x, y, tolerance float64) bool {
return abs(x-y)/y < tolerance
}
func (rateLimitSuite) TestRate(c *gc.C) {
tb := NewBucket(1, 1)
if !isCloseTo(tb.Rate(), 1e9, 0.00001) {
c.Fatalf("got %v want 1e9", tb.Rate())
}
tb = NewBucket(2*time.Second, 1)
if !isCloseTo(tb.Rate(), 0.5, 0.00001) {
c.Fatalf("got %v want 0.5", tb.Rate())
}
tb = NewBucketWithQuantum(100*time.Millisecond, 1, 5)
if !isCloseTo(tb.Rate(), 50, 0.00001) {
c.Fatalf("got %v want 50", tb.Rate())
}
}
func checkRate(c *gc.C, rate float64) {
tb := NewBucketWithRate(rate, 1<<62)
if !isCloseTo(tb.Rate(), rate, rateMargin) {
c.Fatalf("got %g want %v", tb.Rate(), rate)
}
d, ok := tb.take(tb.startTime, 1<<62, infinityDuration)
c.Assert(ok, gc.Equals, true)
c.Assert(d, gc.Equals, time.Duration(0))
// Check that the actual rate is as expected by
// asking for a not-quite multiple of the bucket's
// quantum and checking that the wait time
// correct.
d, ok = tb.take(tb.startTime, tb.quantum*2-tb.quantum/2, infinityDuration)
c.Assert(ok, gc.Equals, true)
expectTime := 1e9 * float64(tb.quantum) * 2 / rate
if !isCloseTo(float64(d), expectTime, rateMargin) {
c.Fatalf("rate %g: got %g want %v", rate, float64(d), expectTime)
}
}
func (rateLimitSuite) TestNewWithRate(c *gc.C) {
for rate := float64(1); rate < 1e6; rate += 7 {
checkRate(c, rate)
}
for _, rate := range []float64{
1024 * 1024 * 1024,
1e-5,
0.9e-5,
0.5,
0.9,
0.9e8,
3e12,
4e18,
} {
checkRate(c, rate)
checkRate(c, rate/3)
checkRate(c, rate*1.3)
}
}
func BenchmarkWait(b *testing.B) {
tb := NewBucket(1, 16*1024)
for i := b.N - 1; i >= 0; i-- {
tb.Wait(1)
}
}
-51
View File
@@ -1,51 +0,0 @@
// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3 with static-linking exception.
// See LICENCE file for details.
package ratelimit
import "io"
type reader struct {
r io.Reader
bucket *Bucket
}
// Reader returns a reader that is rate limited by
// the given token bucket. Each token in the bucket
// represents one byte.
func Reader(r io.Reader, bucket *Bucket) io.Reader {
return &reader{
r: r,
bucket: bucket,
}
}
func (r *reader) Read(buf []byte) (int, error) {
n, err := r.r.Read(buf)
if n <= 0 {
return n, err
}
r.bucket.Wait(int64(n))
return n, err
}
type writer struct {
w io.Writer
bucket *Bucket
}
// Writer returns a reader that is rate limited by
// the given token bucket. Each token in the bucket
// represents one byte.
func Writer(w io.Writer, bucket *Bucket) io.Writer {
return &writer{
w: w,
bucket: bucket,
}
}
func (w *writer) Write(buf []byte) (int, error) {
w.bucket.Wait(int64(len(buf)))
return w.w.Write(buf)
}
-27
View File
@@ -1,27 +0,0 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-14
View File
@@ -1,14 +0,0 @@
### Extensions to the "os" package.
## Find the current Executable and ExecutableFolder.
There is sometimes utility in finding the current executable file
that is running. This can be used for upgrading the current executable
or finding resources located relative to the executable file.
Multi-platform and supports:
* Linux
* OS X
* Windows
* Plan 9
* BSDs.
-27
View File
@@ -1,27 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Extensions to the standard "os" package.
package osext
import "path/filepath"
// Executable returns an absolute path that can be used to
// re-invoke the current program.
// It may not be valid after the current program exits.
func Executable() (string, error) {
p, err := executable()
return filepath.Clean(p), err
}
// Returns same path as Executable, returns just the folder
// path. Excludes the executable name.
func ExecutableFolder() (string, error) {
p, err := Executable()
if err != nil {
return "", err
}
folder, _ := filepath.Split(p)
return folder, nil
}
-20
View File
@@ -1,20 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package osext
import (
"os"
"strconv"
"syscall"
)
func executable() (string, error) {
f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
if err != nil {
return "", err
}
defer f.Close()
return syscall.Fd2path(int(f.Fd()))
}
-34
View File
@@ -1,34 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build linux netbsd openbsd solaris dragonfly
package osext
import (
"errors"
"fmt"
"os"
"runtime"
"strings"
)
func executable() (string, error) {
switch runtime.GOOS {
case "linux":
const deletedSuffix = " (deleted)"
execpath, err := os.Readlink("/proc/self/exe")
if err != nil {
return execpath, err
}
return strings.TrimSuffix(execpath, deletedSuffix), nil
case "netbsd":
return os.Readlink("/proc/curproc/exe")
case "openbsd", "dragonfly":
return os.Readlink("/proc/curproc/file")
case "solaris":
return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
}
return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
}
-79
View File
@@ -1,79 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build darwin freebsd
package osext
import (
"os"
"path/filepath"
"runtime"
"syscall"
"unsafe"
)
var initCwd, initCwdErr = os.Getwd()
func executable() (string, error) {
var mib [4]int32
switch runtime.GOOS {
case "freebsd":
mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
case "darwin":
mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
}
n := uintptr(0)
// Get length.
_, _, errNum := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
if errNum != 0 {
return "", errNum
}
if n == 0 { // This shouldn't happen.
return "", nil
}
buf := make([]byte, n)
_, _, errNum = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
if errNum != 0 {
return "", errNum
}
if n == 0 { // This shouldn't happen.
return "", nil
}
for i, v := range buf {
if v == 0 {
buf = buf[:i]
break
}
}
var err error
execPath := string(buf)
// execPath will not be empty due to above checks.
// Try to get the absolute path if the execPath is not rooted.
if execPath[0] != '/' {
execPath, err = getAbs(execPath)
if err != nil {
return execPath, err
}
}
// For darwin KERN_PROCARGS may return the path to a symlink rather than the
// actual executable.
if runtime.GOOS == "darwin" {
if execPath, err = filepath.EvalSymlinks(execPath); err != nil {
return execPath, err
}
}
return execPath, nil
}
func getAbs(execPath string) (string, error) {
if initCwdErr != nil {
return execPath, initCwdErr
}
// The execPath may begin with a "../" or a "./" so clean it first.
// Join the two paths, trailing and starting slashes undetermined, so use
// the generic Join function.
return filepath.Join(initCwd, filepath.Clean(execPath)), nil
}
-180
View File
@@ -1,180 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build darwin linux freebsd netbsd windows
package osext
import (
"bytes"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"testing"
)
const (
executableEnvVar = "OSTEST_OUTPUT_EXECUTABLE"
executableEnvValueMatch = "match"
executableEnvValueDelete = "delete"
)
func TestExecutableMatch(t *testing.T) {
ep, err := Executable()
if err != nil {
t.Fatalf("Executable failed: %v", err)
}
// fullpath to be of the form "dir/prog".
dir := filepath.Dir(filepath.Dir(ep))
fullpath, err := filepath.Rel(dir, ep)
if err != nil {
t.Fatalf("filepath.Rel: %v", err)
}
// Make child start with a relative program path.
// Alter argv[0] for child to verify getting real path without argv[0].
cmd := &exec.Cmd{
Dir: dir,
Path: fullpath,
Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueMatch)},
}
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("exec(self) failed: %v", err)
}
outs := string(out)
if !filepath.IsAbs(outs) {
t.Fatalf("Child returned %q, want an absolute path", out)
}
if !sameFile(outs, ep) {
t.Fatalf("Child returned %q, not the same file as %q", out, ep)
}
}
func TestExecutableDelete(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip()
}
fpath, err := Executable()
if err != nil {
t.Fatalf("Executable failed: %v", err)
}
r, w := io.Pipe()
stderrBuff := &bytes.Buffer{}
stdoutBuff := &bytes.Buffer{}
cmd := &exec.Cmd{
Path: fpath,
Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueDelete)},
Stdin: r,
Stderr: stderrBuff,
Stdout: stdoutBuff,
}
err = cmd.Start()
if err != nil {
t.Fatalf("exec(self) start failed: %v", err)
}
tempPath := fpath + "_copy"
_ = os.Remove(tempPath)
err = copyFile(tempPath, fpath)
if err != nil {
t.Fatalf("copy file failed: %v", err)
}
err = os.Remove(fpath)
if err != nil {
t.Fatalf("remove running test file failed: %v", err)
}
err = os.Rename(tempPath, fpath)
if err != nil {
t.Fatalf("rename copy to previous name failed: %v", err)
}
w.Write([]byte{0})
w.Close()
err = cmd.Wait()
if err != nil {
t.Fatalf("exec wait failed: %v", err)
}
childPath := stderrBuff.String()
if !filepath.IsAbs(childPath) {
t.Fatalf("Child returned %q, want an absolute path", childPath)
}
if !sameFile(childPath, fpath) {
t.Fatalf("Child returned %q, not the same file as %q", childPath, fpath)
}
}
func sameFile(fn1, fn2 string) bool {
fi1, err := os.Stat(fn1)
if err != nil {
return false
}
fi2, err := os.Stat(fn2)
if err != nil {
return false
}
return os.SameFile(fi1, fi2)
}
func copyFile(dest, src string) error {
df, err := os.Create(dest)
if err != nil {
return err
}
defer df.Close()
sf, err := os.Open(src)
if err != nil {
return err
}
defer sf.Close()
_, err = io.Copy(df, sf)
return err
}
func TestMain(m *testing.M) {
env := os.Getenv(executableEnvVar)
switch env {
case "":
os.Exit(m.Run())
case executableEnvValueMatch:
// First chdir to another path.
dir := "/"
if runtime.GOOS == "windows" {
dir = filepath.VolumeName(".")
}
os.Chdir(dir)
if ep, err := Executable(); err != nil {
fmt.Fprint(os.Stderr, "ERROR: ", err)
} else {
fmt.Fprint(os.Stderr, ep)
}
case executableEnvValueDelete:
bb := make([]byte, 1)
var err error
n, err := os.Stdin.Read(bb)
if err != nil {
fmt.Fprint(os.Stderr, "ERROR: ", err)
os.Exit(2)
}
if n != 1 {
fmt.Fprint(os.Stderr, "ERROR: n != 1, n == ", n)
os.Exit(2)
}
if ep, err := Executable(); err != nil {
fmt.Fprint(os.Stderr, "ERROR: ", err)
} else {
fmt.Fprint(os.Stderr, ep)
}
}
os.Exit(0)
}
-34
View File
@@ -1,34 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package osext
import (
"syscall"
"unicode/utf16"
"unsafe"
)
var (
kernel = syscall.MustLoadDLL("kernel32.dll")
getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW")
)
// GetModuleFileName() with hModule = NULL
func executable() (exePath string, err error) {
return getModuleFileName()
}
func getModuleFileName() (string, error) {
var n uint32
b := make([]uint16, syscall.MAX_PATH)
size := uint32(len(b))
r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size))
n = uint32(r0)
if n == 0 {
return "", e1
}
return string(utf16.Decode(b[0:n])), nil
}
-4
View File
@@ -1,4 +0,0 @@
# This is the official list of Protocol Authors for copyright purposes.
Audrius Butkevicius <audrius.butkevicius@gmail.com>
Jakob Borg <jakob@nym.se>
-76
View File
@@ -1,76 +0,0 @@
## Reporting Bugs
Please file bugs in the [Github Issue
Tracker](https://github.com/syncthing/protocol/issues).
## Contributing Code
Every contribution is welcome. Following the points below will make this
a smoother process.
Individuals making significant and valuable contributions are given
commit-access to the project. If you make a significant contribution and
are not considered for commit-access, please contact any of the
Syncthing core team members.
All nontrivial contributions should go through the pull request
mechanism for internal review. Determining what is "nontrivial" is left
at the discretion of the contributor.
### Authorship
All code authors are listed in the AUTHORS file. Commits must be made
with the same name and email as listed in the AUTHORS file. To
accomplish this, ensure that your git configuration is set correctly
prior to making your first commit;
$ git config --global user.name "Jane Doe"
$ git config --global user.email janedoe@example.com
You must be reachable on the given email address. If you do not wish to
use your real name for whatever reason, using a nickname or pseudonym is
perfectly acceptable.
## Coding Style
- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
as much as makes sense.
- All text files use Unix line endings.
- Each commit should be `go fmt` clean.
- The commit message subject should be a single short sentence
describing the change, starting with a capital letter.
- Commits that resolve an existing issue must include the issue number
as `(fixes #123)` at the end of the commit message subject.
- Imports are grouped per `goimports` standard; that is, standard
library first, then third party libraries after a blank line.
- A contribution solving a single issue or introducing a single new
feature should probably be a single commit based on the current
`master` branch. You may be asked to "rebase" or "squash" your pull
request to make sure this is the case, especially if there have been
amendments during review.
## Licensing
All contributions are made under the same MIT license as the rest of the
project, except documentation, user interface text and translation
strings which are licensed under the Creative Commons Attribution 4.0
International License. You retain the copyright to code you have
written.
When accepting your first contribution, the maintainer of the project
will ensure that you are added to the AUTHORS file. You are welcome to
add yourself as a separate commit in your first pull request.
## Tests
Yes please!
## License
MIT
-19
View File
@@ -1,19 +0,0 @@
Copyright (C) 2014-2015 The Protocol Authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-13
View File
@@ -1,13 +0,0 @@
The BEPv1 Protocol
==================
[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/)
[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol)
[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT)
This is the protocol implementation used by Syncthing.
License
=======
MIT
-80
View File
@@ -1,80 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"io"
"time"
)
type TestModel struct {
data []byte
folder string
name string
offset int64
size int
hash []byte
flags uint32
options []Option
closedCh chan bool
}
func newTestModel() *TestModel {
return &TestModel{
closedCh: make(chan bool),
}
}
func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
}
func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
}
func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
t.folder = folder
t.name = name
t.offset = offset
t.size = size
t.hash = hash
t.flags = flags
t.options = options
return t.data, nil
}
func (t *TestModel) Close(deviceID DeviceID, err error) {
close(t.closedCh)
}
func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
}
func (t *TestModel) isClosed() bool {
select {
case <-t.closedCh:
return true
case <-time.After(1 * time.Second):
return false // Timeout
}
}
type ErrPipe struct {
io.PipeWriter
written int
max int
err error
closed bool
}
func (e *ErrPipe) Write(data []byte) (int, error) {
if e.closed {
return 0, e.err
}
if e.written+len(data) > e.max {
n, _ := e.PipeWriter.Write(data[:e.max-e.written])
e.PipeWriter.CloseWithError(e.err)
e.closed = true
return n, e.err
}
return e.PipeWriter.Write(data)
}
-53
View File
@@ -1,53 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
import "fmt"
type Compression int
const (
CompressMetadata Compression = iota // zero value is the default, default should be "metadata"
CompressNever
CompressAlways
compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes
)
var compressionMarshal = map[Compression]string{
CompressNever: "never",
CompressMetadata: "metadata",
CompressAlways: "always",
}
var compressionUnmarshal = map[string]Compression{
// Legacy
"false": CompressNever,
"true": CompressMetadata,
// Current
"never": CompressNever,
"metadata": CompressMetadata,
"always": CompressAlways,
}
func (c Compression) String() string {
s, ok := compressionMarshal[c]
if !ok {
return fmt.Sprintf("unknown:%d", c)
}
return s
}
func (c Compression) GoString() string {
return fmt.Sprintf("%q", c.String())
}
func (c Compression) MarshalText() ([]byte, error) {
return []byte(compressionMarshal[c]), nil
}
func (c *Compression) UnmarshalText(bs []byte) error {
*c = compressionUnmarshal[string(bs)]
return nil
}
@@ -1,49 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
import "testing"
func TestCompressionMarshal(t *testing.T) {
uTestcases := []struct {
s string
c Compression
}{
{"true", CompressMetadata},
{"false", CompressNever},
{"never", CompressNever},
{"metadata", CompressMetadata},
{"always", CompressAlways},
{"whatever", CompressMetadata},
}
mTestcases := []struct {
s string
c Compression
}{
{"never", CompressNever},
{"metadata", CompressMetadata},
{"always", CompressAlways},
}
var c Compression
for _, tc := range uTestcases {
err := c.UnmarshalText([]byte(tc.s))
if err != nil {
t.Error(err)
}
if c != tc.c {
t.Errorf("%s unmarshalled to %d, not %d", tc.s, c, tc.c)
}
}
for _, tc := range mTestcases {
bs, err := tc.c.MarshalText()
if err != nil {
t.Error(err)
}
if s := string(bs); s != tc.s {
t.Errorf("%d marshalled to %q, not %q", tc.c, s, tc.s)
}
}
}
-62
View File
@@ -1,62 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"io"
"sync/atomic"
"time"
)
type countingReader struct {
io.Reader
tot int64 // bytes
last int64 // unix nanos
}
var (
totalIncoming int64
totalOutgoing int64
)
func (c *countingReader) Read(bs []byte) (int, error) {
n, err := c.Reader.Read(bs)
atomic.AddInt64(&c.tot, int64(n))
atomic.AddInt64(&totalIncoming, int64(n))
atomic.StoreInt64(&c.last, time.Now().UnixNano())
return n, err
}
func (c *countingReader) Tot() int64 {
return atomic.LoadInt64(&c.tot)
}
func (c *countingReader) Last() time.Time {
return time.Unix(0, atomic.LoadInt64(&c.last))
}
type countingWriter struct {
io.Writer
tot int64 // bytes
last int64 // unix nanos
}
func (c *countingWriter) Write(bs []byte) (int, error) {
n, err := c.Writer.Write(bs)
atomic.AddInt64(&c.tot, int64(n))
atomic.AddInt64(&totalOutgoing, int64(n))
atomic.StoreInt64(&c.last, time.Now().UnixNano())
return n, err
}
func (c *countingWriter) Tot() int64 {
return atomic.LoadInt64(&c.tot)
}
func (c *countingWriter) Last() time.Time {
return time.Unix(0, atomic.LoadInt64(&c.last))
}
func TotalInOut() (int64, int64) {
return atomic.LoadInt64(&totalIncoming), atomic.LoadInt64(&totalOutgoing)
}
-15
View File
@@ -1,15 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"os"
"strings"
"github.com/calmh/logger"
)
var (
debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all"
l = logger.DefaultLogger
)
-163
View File
@@ -1,163 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"bytes"
"crypto/sha256"
"encoding/base32"
"encoding/binary"
"errors"
"fmt"
"regexp"
"strings"
"github.com/calmh/luhn"
)
type DeviceID [32]byte
var LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
// NewDeviceID generates a new device ID from the raw bytes of a certificate
func NewDeviceID(rawCert []byte) DeviceID {
var n DeviceID
hf := sha256.New()
hf.Write(rawCert)
hf.Sum(n[:0])
return n
}
func DeviceIDFromString(s string) (DeviceID, error) {
var n DeviceID
err := n.UnmarshalText([]byte(s))
return n, err
}
func DeviceIDFromBytes(bs []byte) DeviceID {
var n DeviceID
if len(bs) != len(n) {
panic("incorrect length of byte slice representing device ID")
}
copy(n[:], bs)
return n
}
// String returns the canonical string representation of the device ID
func (n DeviceID) String() string {
id := base32.StdEncoding.EncodeToString(n[:])
id = strings.Trim(id, "=")
id, err := luhnify(id)
if err != nil {
// Should never happen
panic(err)
}
id = chunkify(id)
return id
}
func (n DeviceID) GoString() string {
return n.String()
}
func (n DeviceID) Compare(other DeviceID) int {
return bytes.Compare(n[:], other[:])
}
func (n DeviceID) Equals(other DeviceID) bool {
return bytes.Compare(n[:], other[:]) == 0
}
// Short returns an integer representing bits 0-63 of the device ID.
func (n DeviceID) Short() uint64 {
return binary.BigEndian.Uint64(n[:])
}
func (n *DeviceID) MarshalText() ([]byte, error) {
return []byte(n.String()), nil
}
func (n *DeviceID) UnmarshalText(bs []byte) error {
id := string(bs)
id = strings.Trim(id, "=")
id = strings.ToUpper(id)
id = untypeoify(id)
id = unchunkify(id)
var err error
switch len(id) {
case 56:
// New style, with check digits
id, err = unluhnify(id)
if err != nil {
return err
}
fallthrough
case 52:
// Old style, no check digits
dec, err := base32.StdEncoding.DecodeString(id + "====")
if err != nil {
return err
}
copy(n[:], dec)
return nil
default:
return errors.New("device ID invalid: incorrect length")
}
}
func luhnify(s string) (string, error) {
if len(s) != 52 {
panic("unsupported string length")
}
res := make([]string, 0, 4)
for i := 0; i < 4; i++ {
p := s[i*13 : (i+1)*13]
l, err := luhn.Base32.Generate(p)
if err != nil {
return "", err
}
res = append(res, fmt.Sprintf("%s%c", p, l))
}
return res[0] + res[1] + res[2] + res[3], nil
}
func unluhnify(s string) (string, error) {
if len(s) != 56 {
return "", fmt.Errorf("unsupported string length %d", len(s))
}
res := make([]string, 0, 4)
for i := 0; i < 4; i++ {
p := s[i*14 : (i+1)*14-1]
l, err := luhn.Base32.Generate(p)
if err != nil {
return "", err
}
if g := fmt.Sprintf("%s%c", p, l); g != s[i*14:(i+1)*14] {
return "", errors.New("check digit incorrect")
}
res = append(res, p)
}
return res[0] + res[1] + res[2] + res[3], nil
}
func chunkify(s string) string {
s = regexp.MustCompile("(.{7})").ReplaceAllString(s, "$1-")
s = strings.Trim(s, "-")
return s
}
func unchunkify(s string) string {
s = strings.Replace(s, "-", "", -1)
s = strings.Replace(s, " ", "", -1)
return s
}
func untypeoify(s string) string {
s = strings.Replace(s, "0", "O", -1)
s = strings.Replace(s, "1", "I", -1)
s = strings.Replace(s, "8", "B", -1)
return s
}
-76
View File
@@ -1,76 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import "testing"
var formatted = "P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"
var formatCases = []string{
"P56IOI-7MZJNU-2IQGDR-EYDM2M-GTMGL3-BXNPQ6-W5BTBB-Z4TJXZ-WICQ",
"P56IOI-7MZJNU2Y-IQGDR-EYDM2M-GTI-MGL3-BXNPQ6-W5BM-TBB-Z4TJXZ-WICQ2",
"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ",
"P56IOI7 MZJNU2Y IQGDREY DM2MGTI MGL3BXN PQ6W5BM TBBZ4TJ XZWICQ2",
"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ",
"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq",
"P56IOI7MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMTBBZ4TJXZWICQ2",
"P561017MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMT88Z4TJXZWICQ2",
"p56ioi7mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmtbbz4tjxzwicq2",
"p561017mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmt88z4tjxzwicq2",
}
func TestFormatDeviceID(t *testing.T) {
for i, tc := range formatCases {
var id DeviceID
err := id.UnmarshalText([]byte(tc))
if err != nil {
t.Errorf("#%d UnmarshalText(%q); %v", i, tc, err)
} else if f := id.String(); f != formatted {
t.Errorf("#%d FormatDeviceID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted)
}
}
}
var validateCases = []struct {
s string
ok bool
}{
{"", false},
{"P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2", true},
{"P56IOI7-MZJNU2-IQGDREY-DM2MGT-MGL3BXN-PQ6W5B-TBBZ4TJ-XZWICQ", true},
{"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", true},
{"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", true},
{"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQCCCC", false},
{"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", true},
{"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicqCCCC", false},
}
func TestValidateDeviceID(t *testing.T) {
for _, tc := range validateCases {
var id DeviceID
err := id.UnmarshalText([]byte(tc.s))
if (err == nil && !tc.ok) || (err != nil && tc.ok) {
t.Errorf("ValidateDeviceID(%q); %v != %v", tc.s, err, tc.ok)
}
}
}
func TestMarshallingDeviceID(t *testing.T) {
n0 := DeviceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}
n1 := DeviceID{}
n2 := DeviceID{}
bs, _ := n0.MarshalText()
n1.UnmarshalText(bs)
bs, _ = n1.MarshalText()
n2.UnmarshalText(bs)
if n2.String() != n0.String() {
t.Errorf("String marshalling error; %q != %q", n2.String(), n0.String())
}
if !n2.Equals(n0) {
t.Error("Equals error")
}
if n2.Compare(n0) != 0 {
t.Error("Compare error")
}
}
-4
View File
@@ -1,4 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
// Package protocol implements the Block Exchange Protocol.
package protocol
-51
View File
@@ -1,51 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"errors"
)
const (
ecNoError int32 = iota
ecGeneric
ecNoSuchFile
ecInvalid
)
var (
ErrNoError error = nil
ErrGeneric = errors.New("generic error")
ErrNoSuchFile = errors.New("no such file")
ErrInvalid = errors.New("file is invalid")
)
var lookupError = map[int32]error{
ecNoError: ErrNoError,
ecGeneric: ErrGeneric,
ecNoSuchFile: ErrNoSuchFile,
ecInvalid: ErrInvalid,
}
var lookupCode = map[error]int32{
ErrNoError: ecNoError,
ErrGeneric: ecGeneric,
ErrNoSuchFile: ecNoSuchFile,
ErrInvalid: ecInvalid,
}
func codeToError(errcode int32) error {
err, ok := lookupError[errcode]
if !ok {
return ErrGeneric
}
return err
}
func errorToCode(err error) int32 {
code, ok := lookupCode[err]
if !ok {
return ecGeneric
}
return code
}
-43
View File
@@ -1,43 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import "github.com/calmh/xdr"
type header struct {
version int
msgID int
msgType int
compression bool
}
func (h header) encodeXDR(xw *xdr.Writer) (int, error) {
u := encodeHeader(h)
return xw.WriteUint32(u)
}
func (h *header) decodeXDR(xr *xdr.Reader) error {
u := xr.ReadUint32()
*h = decodeHeader(u)
return xr.Error()
}
func encodeHeader(h header) uint32 {
var isComp uint32
if h.compression {
isComp = 1 << 0 // the zeroth bit is the compression bit
}
return uint32(h.version&0xf)<<28 +
uint32(h.msgID&0xfff)<<16 +
uint32(h.msgType&0xff)<<8 +
isComp
}
func decodeHeader(u uint32) header {
return header{
version: int(u>>28) & 0xf,
msgID: int(u>>16) & 0xfff,
msgType: int(u>>8) & 0xff,
compression: u&1 == 1,
}
}
-126
View File
@@ -1,126 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o message_xdr.go message.go
package protocol
import "fmt"
type IndexMessage struct {
Folder string
Files []FileInfo
Flags uint32
Options []Option // max:64
}
type FileInfo struct {
Name string // max:8192
Flags uint32
Modified int64
Version Vector
LocalVersion int64
Blocks []BlockInfo
}
func (f FileInfo) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
}
func (f FileInfo) Size() (bytes int64) {
if f.IsDeleted() || f.IsDirectory() {
return 128
}
for _, b := range f.Blocks {
bytes += int64(b.Size)
}
return
}
func (f FileInfo) IsDeleted() bool {
return f.Flags&FlagDeleted != 0
}
func (f FileInfo) IsInvalid() bool {
return f.Flags&FlagInvalid != 0
}
func (f FileInfo) IsDirectory() bool {
return f.Flags&FlagDirectory != 0
}
func (f FileInfo) IsSymlink() bool {
return f.Flags&FlagSymlink != 0
}
func (f FileInfo) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
type BlockInfo struct {
Offset int64 // noencode (cache only)
Size int32
Hash []byte // max:64
}
func (b BlockInfo) String() string {
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
}
type RequestMessage struct {
Folder string // max:64
Name string // max:8192
Offset int64
Size int32
Hash []byte // max:64
Flags uint32
Options []Option // max:64
}
type ResponseMessage struct {
Data []byte
Code int32
}
type ClusterConfigMessage struct {
ClientName string // max:64
ClientVersion string // max:64
Folders []Folder
Options []Option // max:64
}
func (o *ClusterConfigMessage) GetOption(key string) string {
for _, option := range o.Options {
if option.Key == key {
return option.Value
}
}
return ""
}
type Folder struct {
ID string // max:64
Devices []Device
Flags uint32
Options []Option // max:64
}
type Device struct {
ID []byte // max:32
MaxLocalVersion int64
Flags uint32
Options []Option // max:64
}
type Option struct {
Key string // max:64
Value string // max:1024
}
type CloseMessage struct {
Reason string // max:1024
Code int32
}
type EmptyMessage struct{}
File diff suppressed because it is too large Load Diff
@@ -1,40 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
// +build darwin
package protocol
// Darwin uses NFD normalization
import "golang.org/x/text/unicode/norm"
type nativeModel struct {
next Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
for i := range files {
files[i].Name = norm.NFD.String(files[i].Name)
}
m.next.Index(deviceID, folder, files, flags, options)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
for i := range files {
files[i].Name = norm.NFD.String(files[i].Name)
}
m.next.IndexUpdate(deviceID, folder, files, flags, options)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
name = norm.NFD.String(name)
return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
}
func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
m.next.ClusterConfig(deviceID, config)
}
func (m nativeModel) Close(deviceID DeviceID, err error) {
m.next.Close(deviceID, err)
}
@@ -1,31 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
// +build !windows,!darwin
package protocol
// Normal Unixes uses NFC and slashes, which is the wire format.
type nativeModel struct {
next Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
m.next.Index(deviceID, folder, files, flags, options)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
m.next.IndexUpdate(deviceID, folder, files, flags, options)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
}
func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
m.next.ClusterConfig(deviceID, config)
}
func (m nativeModel) Close(deviceID DeviceID, err error) {
m.next.Close(deviceID, err)
}
@@ -1,63 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
// +build windows
package protocol
// Windows uses backslashes as file separator and disallows a bunch of
// characters in the filename
import (
"path/filepath"
"strings"
)
var disallowedCharacters = string([]rune{
'<', '>', ':', '"', '|', '?', '*',
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
31,
})
type nativeModel struct {
next Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
fixupFiles(files)
m.next.Index(deviceID, folder, files, flags, options)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
fixupFiles(files)
m.next.IndexUpdate(deviceID, folder, files, flags, options)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
name = filepath.FromSlash(name)
return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
}
func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
m.next.ClusterConfig(deviceID, config)
}
func (m nativeModel) Close(deviceID DeviceID, err error) {
m.next.Close(deviceID, err)
}
func fixupFiles(files []FileInfo) {
for i, f := range files {
if strings.ContainsAny(f.Name, disallowedCharacters) {
if f.IsDeleted() {
// Don't complain if the file is marked as deleted, since it
// can't possibly exist here anyway.
continue
}
files[i].Flags |= FlagInvalid
l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name)
}
files[i].Name = filepath.FromSlash(files[i].Name)
}
}
-736
View File
@@ -1,736 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"sync"
"time"
lz4 "github.com/bkaradzic/go-lz4"
)
const (
BlockSize = 128 * 1024
)
const (
messageTypeClusterConfig = 0
messageTypeIndex = 1
messageTypeRequest = 2
messageTypeResponse = 3
messageTypePing = 4
messageTypePong = 5
messageTypeIndexUpdate = 6
messageTypeClose = 7
)
const (
stateInitial = iota
stateCCRcvd
stateIdxRcvd
)
// FileInfo flags
const (
FlagDeleted uint32 = 1 << 12
FlagInvalid = 1 << 13
FlagDirectory = 1 << 14
FlagNoPermBits = 1 << 15
FlagSymlink = 1 << 16
FlagSymlinkMissingTarget = 1 << 17
FlagsAll = (1 << 18) - 1
SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget
)
// IndexMessage message flags (for IndexUpdate)
const (
FlagIndexTemporary uint32 = 1 << iota
)
// Request message flags
const (
FlagRequestTemporary uint32 = 1 << iota
)
// ClusterConfigMessage.Folders.Devices flags
const (
FlagShareTrusted uint32 = 1 << 0
FlagShareReadOnly = 1 << 1
FlagIntroducer = 1 << 2
FlagShareBits = 0x000000ff
)
var (
ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
ErrClosed = errors.New("connection closed")
)
// Specific variants of empty messages...
type pingMessage struct{ EmptyMessage }
type pongMessage struct{ EmptyMessage }
type Model interface {
// An index was received from the peer device
Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
// An index update was received from the peer device
IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
// A request was made by the peer device
Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
// A cluster configuration message was received
ClusterConfig(deviceID DeviceID, config ClusterConfigMessage)
// The peer device closed the connection
Close(deviceID DeviceID, err error)
}
type Connection interface {
ID() DeviceID
Name() string
Index(folder string, files []FileInfo, flags uint32, options []Option) error
IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error
Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
ClusterConfig(config ClusterConfigMessage)
Statistics() Statistics
}
type rawConnection struct {
id DeviceID
name string
receiver Model
state int
cr *countingReader
cw *countingWriter
awaiting [4096]chan asyncResult
awaitingMut sync.Mutex
idxMut sync.Mutex // ensures serialization of Index calls
nextID chan int
outbox chan hdrMsg
closed chan struct{}
once sync.Once
compression Compression
rdbuf0 []byte // used & reused by readMessage
rdbuf1 []byte // used & reused by readMessage
}
type asyncResult struct {
val []byte
err error
}
type hdrMsg struct {
hdr header
msg encodable
}
type encodable interface {
AppendXDR([]byte) ([]byte, error)
}
type isEofer interface {
IsEOF() bool
}
const (
pingTimeout = 30 * time.Second
pingIdleTime = 60 * time.Second
)
func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection {
cr := &countingReader{Reader: reader}
cw := &countingWriter{Writer: writer}
c := rawConnection{
id: deviceID,
name: name,
receiver: nativeModel{receiver},
state: stateInitial,
cr: cr,
cw: cw,
outbox: make(chan hdrMsg),
nextID: make(chan int),
closed: make(chan struct{}),
compression: compress,
}
go c.readerLoop()
go c.writerLoop()
go c.pingerLoop()
go c.idGenerator()
return wireFormatConnection{&c}
}
func (c *rawConnection) ID() DeviceID {
return c.id
}
func (c *rawConnection) Name() string {
return c.name
}
// Index writes the list of file information to the connected peer device
func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error {
select {
case <-c.closed:
return ErrClosed
default:
}
c.idxMut.Lock()
c.send(-1, messageTypeIndex, IndexMessage{
Folder: folder,
Files: idx,
Flags: flags,
Options: options,
})
c.idxMut.Unlock()
return nil
}
// IndexUpdate writes the list of file information to the connected peer device as an update
func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error {
select {
case <-c.closed:
return ErrClosed
default:
}
c.idxMut.Lock()
c.send(-1, messageTypeIndexUpdate, IndexMessage{
Folder: folder,
Files: idx,
Flags: flags,
Options: options,
})
c.idxMut.Unlock()
return nil
}
// Request returns the bytes for the specified block after fetching them from the connected peer.
func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
var id int
select {
case id = <-c.nextID:
case <-c.closed:
return nil, ErrClosed
}
c.awaitingMut.Lock()
if ch := c.awaiting[id]; ch != nil {
panic("id taken")
}
rc := make(chan asyncResult, 1)
c.awaiting[id] = rc
c.awaitingMut.Unlock()
ok := c.send(id, messageTypeRequest, RequestMessage{
Folder: folder,
Name: name,
Offset: offset,
Size: int32(size),
Hash: hash,
Flags: flags,
Options: options,
})
if !ok {
return nil, ErrClosed
}
res, ok := <-rc
if !ok {
return nil, ErrClosed
}
return res.val, res.err
}
// ClusterConfig send the cluster configuration message to the peer and returns any error
func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
c.send(-1, messageTypeClusterConfig, config)
}
func (c *rawConnection) ping() bool {
var id int
select {
case id = <-c.nextID:
case <-c.closed:
return false
}
rc := make(chan asyncResult, 1)
c.awaitingMut.Lock()
c.awaiting[id] = rc
c.awaitingMut.Unlock()
ok := c.send(id, messageTypePing, nil)
if !ok {
return false
}
res, ok := <-rc
return ok && res.err == nil
}
func (c *rawConnection) readerLoop() (err error) {
defer func() {
c.close(err)
}()
for {
select {
case <-c.closed:
return ErrClosed
default:
}
hdr, msg, err := c.readMessage()
if err != nil {
return err
}
switch msg := msg.(type) {
case IndexMessage:
switch hdr.msgType {
case messageTypeIndex:
if c.state < stateCCRcvd {
return fmt.Errorf("protocol error: index message in state %d", c.state)
}
c.handleIndex(msg)
c.state = stateIdxRcvd
case messageTypeIndexUpdate:
if c.state < stateIdxRcvd {
return fmt.Errorf("protocol error: index update message in state %d", c.state)
}
c.handleIndexUpdate(msg)
}
case RequestMessage:
if c.state < stateIdxRcvd {
return fmt.Errorf("protocol error: request message in state %d", c.state)
}
// Requests are handled asynchronously
go c.handleRequest(hdr.msgID, msg)
case ResponseMessage:
if c.state < stateIdxRcvd {
return fmt.Errorf("protocol error: response message in state %d", c.state)
}
c.handleResponse(hdr.msgID, msg)
case pingMessage:
c.send(hdr.msgID, messageTypePong, pongMessage{})
case pongMessage:
c.handlePong(hdr.msgID)
case ClusterConfigMessage:
if c.state != stateInitial {
return fmt.Errorf("protocol error: cluster config message in state %d", c.state)
}
go c.receiver.ClusterConfig(c.id, msg)
c.state = stateCCRcvd
case CloseMessage:
return errors.New(msg.Reason)
default:
return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
}
}
}
func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
if cap(c.rdbuf0) < 8 {
c.rdbuf0 = make([]byte, 8)
} else {
c.rdbuf0 = c.rdbuf0[:8]
}
_, err = io.ReadFull(c.cr, c.rdbuf0)
if err != nil {
return
}
hdr = decodeHeader(binary.BigEndian.Uint32(c.rdbuf0[0:4]))
msglen := int(binary.BigEndian.Uint32(c.rdbuf0[4:8]))
if debug {
l.Debugf("read header %v (msglen=%d)", hdr, msglen)
}
if hdr.version != 0 {
err = fmt.Errorf("unknown protocol version 0x%x", hdr.version)
return
}
if cap(c.rdbuf0) < msglen {
c.rdbuf0 = make([]byte, msglen)
} else {
c.rdbuf0 = c.rdbuf0[:msglen]
}
_, err = io.ReadFull(c.cr, c.rdbuf0)
if err != nil {
return
}
if debug {
l.Debugf("read %d bytes", len(c.rdbuf0))
}
msgBuf := c.rdbuf0
if hdr.compression {
c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)]
c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0)
if err != nil {
return
}
msgBuf = c.rdbuf1
if debug {
l.Debugf("decompressed to %d bytes", len(msgBuf))
}
}
if debug {
if len(msgBuf) > 1024 {
l.Debugf("message data:\n%s", hex.Dump(msgBuf[:1024]))
} else {
l.Debugf("message data:\n%s", hex.Dump(msgBuf))
}
}
// We check each returned error for the XDRError.IsEOF() method.
// IsEOF()==true here means that the message contained fewer fields than
// expected. It does not signify an EOF on the socket, because we've
// successfully read a size value and that many bytes already. New fields
// we expected but the other peer didn't send should be interpreted as
// zero/nil, and if that's not valid we'll verify it somewhere else.
switch hdr.msgType {
case messageTypeIndex, messageTypeIndexUpdate:
var idx IndexMessage
err = idx.UnmarshalXDR(msgBuf)
if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
err = nil
}
msg = idx
case messageTypeRequest:
var req RequestMessage
err = req.UnmarshalXDR(msgBuf)
if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
err = nil
}
msg = req
case messageTypeResponse:
var resp ResponseMessage
err = resp.UnmarshalXDR(msgBuf)
if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
err = nil
}
msg = resp
case messageTypePing:
msg = pingMessage{}
case messageTypePong:
msg = pongMessage{}
case messageTypeClusterConfig:
var cc ClusterConfigMessage
err = cc.UnmarshalXDR(msgBuf)
if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
err = nil
}
msg = cc
case messageTypeClose:
var cm CloseMessage
err = cm.UnmarshalXDR(msgBuf)
if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
err = nil
}
msg = cm
default:
err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
}
return
}
func (c *rawConnection) handleIndex(im IndexMessage) {
if debug {
l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
}
c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
}
func (c *rawConnection) handleIndexUpdate(im IndexMessage) {
if debug {
l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
}
c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
}
func filterIndexMessageFiles(fs []FileInfo) []FileInfo {
var out []FileInfo
for i, f := range fs {
switch f.Name {
case "", ".", "..", "/": // A few obviously invalid filenames
l.Infof("Dropping invalid filename %q from incoming index", f.Name)
if out == nil {
// Most incoming updates won't contain anything invalid, so we
// delay the allocation and copy to output slice until we
// really need to do it, then copy all the so var valid files
// to it.
out = make([]FileInfo, i, len(fs)-1)
copy(out, fs)
}
default:
if out != nil {
out = append(out, f)
}
}
}
if out != nil {
return out
}
return fs
}
func (c *rawConnection) handleRequest(msgID int, req RequestMessage) {
data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options)
c.send(msgID, messageTypeResponse, ResponseMessage{
Data: data,
Code: errorToCode(err),
})
}
func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) {
c.awaitingMut.Lock()
if rc := c.awaiting[msgID]; rc != nil {
c.awaiting[msgID] = nil
rc <- asyncResult{resp.Data, codeToError(resp.Code)}
close(rc)
}
c.awaitingMut.Unlock()
}
func (c *rawConnection) handlePong(msgID int) {
c.awaitingMut.Lock()
if rc := c.awaiting[msgID]; rc != nil {
c.awaiting[msgID] = nil
rc <- asyncResult{}
close(rc)
}
c.awaitingMut.Unlock()
}
func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool {
if msgID < 0 {
select {
case id := <-c.nextID:
msgID = id
case <-c.closed:
return false
}
}
hdr := header{
version: 0,
msgID: msgID,
msgType: msgType,
}
select {
case c.outbox <- hdrMsg{hdr, msg}:
return true
case <-c.closed:
return false
}
}
func (c *rawConnection) writerLoop() {
var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused
var uncBuf []byte // buffer for uncompressed message, kept and reused
for {
var tempBuf []byte
var err error
select {
case hm := <-c.outbox:
if hm.msg != nil {
// Uncompressed message in uncBuf
uncBuf, err = hm.msg.AppendXDR(uncBuf[:0])
if err != nil {
c.close(err)
return
}
compress := false
switch c.compression {
case CompressAlways:
compress = true
case CompressMetadata:
compress = hm.hdr.msgType != messageTypeResponse
}
if compress && len(uncBuf) >= compressionThreshold {
// Use compression for large messages
hm.hdr.compression = true
// Make sure we have enough space for the compressed message plus header in msgBug
msgBuf = msgBuf[:cap(msgBuf)]
if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) {
msgBuf = make([]byte, maxLen)
}
// Compressed is written to msgBuf, we keep tb for the length only
tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf)
binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf)))
msgBuf = msgBuf[0 : len(tempBuf)+8]
if debug {
l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf))
}
} else {
// No point in compressing very short messages
hm.hdr.compression = false
msgBuf = msgBuf[:cap(msgBuf)]
if l := len(uncBuf) + 8; l > len(msgBuf) {
msgBuf = make([]byte, l)
}
binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf)))
msgBuf = msgBuf[0 : len(uncBuf)+8]
copy(msgBuf[8:], uncBuf)
if debug {
l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf))
}
}
} else {
if debug {
l.Debugf("write empty message; %v", hm.hdr)
}
binary.BigEndian.PutUint32(msgBuf[4:8], 0)
msgBuf = msgBuf[:8]
}
binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr))
if err == nil {
var n int
n, err = c.cw.Write(msgBuf)
if debug {
l.Debugf("wrote %d bytes on the wire", n)
}
}
if err != nil {
c.close(err)
return
}
case <-c.closed:
return
}
}
}
func (c *rawConnection) close(err error) {
c.once.Do(func() {
close(c.closed)
c.awaitingMut.Lock()
for i, ch := range c.awaiting {
if ch != nil {
close(ch)
c.awaiting[i] = nil
}
}
c.awaitingMut.Unlock()
go c.receiver.Close(c.id, err)
})
}
func (c *rawConnection) idGenerator() {
nextID := 0
for {
nextID = (nextID + 1) & 0xfff
select {
case c.nextID <- nextID:
case <-c.closed:
return
}
}
}
func (c *rawConnection) pingerLoop() {
var rc = make(chan bool, 1)
ticker := time.Tick(pingIdleTime / 2)
for {
select {
case <-ticker:
if d := time.Since(c.cr.Last()); d < pingIdleTime {
if debug {
l.Debugln(c.id, "ping skipped after rd", d)
}
continue
}
if d := time.Since(c.cw.Last()); d < pingIdleTime {
if debug {
l.Debugln(c.id, "ping skipped after wr", d)
}
continue
}
go func() {
if debug {
l.Debugln(c.id, "ping ->")
}
rc <- c.ping()
}()
select {
case ok := <-rc:
if debug {
l.Debugln(c.id, "<- pong")
}
if !ok {
c.close(fmt.Errorf("ping failure"))
}
case <-time.After(pingTimeout):
c.close(fmt.Errorf("ping timeout"))
case <-c.closed:
return
}
case <-c.closed:
return
}
}
}
type Statistics struct {
At time.Time
InBytesTotal int64
OutBytesTotal int64
}
func (c *rawConnection) Statistics() Statistics {
return Statistics{
At: time.Now(),
InBytesTotal: c.cr.Tot(),
OutBytesTotal: c.cw.Tot(),
}
}
-382
View File
@@ -1,382 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"bytes"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"reflect"
"strings"
"testing"
"testing/quick"
"github.com/calmh/xdr"
)
var (
c0ID = NewDeviceID([]byte{1})
c1ID = NewDeviceID([]byte{2})
)
func TestHeaderFunctions(t *testing.T) {
f := func(ver, id, typ int) bool {
ver = int(uint(ver) % 16)
id = int(uint(id) % 4096)
typ = int(uint(typ) % 256)
h0 := header{version: ver, msgID: id, msgType: typ}
h1 := decodeHeader(encodeHeader(h0))
return h0 == h1
}
if err := quick.Check(f, nil); err != nil {
t.Error(err)
}
}
func TestHeaderLayout(t *testing.T) {
var e, a uint32
// Version are the first four bits
e = 0xf0000000
a = encodeHeader(header{version: 0xf})
if a != e {
t.Errorf("Header layout incorrect; %08x != %08x", a, e)
}
// Message ID are the following 12 bits
e = 0x0fff0000
a = encodeHeader(header{msgID: 0xfff})
if a != e {
t.Errorf("Header layout incorrect; %08x != %08x", a, e)
}
// Type are the last 8 bits before reserved
e = 0x0000ff00
a = encodeHeader(header{msgType: 0xff})
if a != e {
t.Errorf("Header layout incorrect; %08x != %08x", a, e)
}
}
func TestPing(t *testing.T) {
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
if ok := c0.ping(); !ok {
t.Error("c0 ping failed")
}
if ok := c1.ping(); !ok {
t.Error("c1 ping failed")
}
}
func TestPingErr(t *testing.T) {
e := errors.New("something broke")
for i := 0; i < 16; i++ {
for j := 0; j < 16; j++ {
m0 := newTestModel()
m1 := newTestModel()
ar, aw := io.Pipe()
br, bw := io.Pipe()
eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
NewConnection(c1ID, br, eaw, m1, "name", CompressAlways)
res := c0.ping()
if (i < 8 || j < 8) && res {
t.Errorf("Unexpected ping success; i=%d, j=%d", i, j)
} else if (i >= 12 && j >= 12) && !res {
t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j)
}
}
}
}
// func TestRequestResponseErr(t *testing.T) {
// e := errors.New("something broke")
// var pass bool
// for i := 0; i < 48; i++ {
// for j := 0; j < 38; j++ {
// m0 := newTestModel()
// m0.data = []byte("response data")
// m1 := newTestModel()
// ar, aw := io.Pipe()
// br, bw := io.Pipe()
// eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
// ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
// NewConnection(c0ID, ar, ebw, m0, nil)
// c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection)
// d, err := c1.Request("default", "tn", 1234, 5678)
// if err == e || err == ErrClosed {
// t.Logf("Error at %d+%d bytes", i, j)
// if !m1.isClosed() {
// t.Fatal("c1 not closed")
// }
// if !m0.isClosed() {
// t.Fatal("c0 not closed")
// }
// continue
// }
// if err != nil {
// t.Fatal(err)
// }
// if string(d) != "response data" {
// t.Fatalf("Incorrect response data %q", string(d))
// }
// if m0.folder != "default" {
// t.Fatalf("Incorrect folder %q", m0.folder)
// }
// if m0.name != "tn" {
// t.Fatalf("Incorrect name %q", m0.name)
// }
// if m0.offset != 1234 {
// t.Fatalf("Incorrect offset %d", m0.offset)
// }
// if m0.size != 5678 {
// t.Fatalf("Incorrect size %d", m0.size)
// }
// t.Logf("Pass at %d+%d bytes", i, j)
// pass = true
// }
// }
// if !pass {
// t.Fatal("Never passed")
// }
// }
func TestVersionErr(t *testing.T) {
m0 := newTestModel()
m1 := newTestModel()
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
w := xdr.NewWriter(c0.cw)
w.WriteUint32(encodeHeader(header{
version: 2,
msgID: 0,
msgType: 0,
}))
w.WriteUint32(0) // Avoids reader closing due to EOF
if !m1.isClosed() {
t.Error("Connection should close due to unknown version")
}
}
func TestTypeErr(t *testing.T) {
m0 := newTestModel()
m1 := newTestModel()
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
w := xdr.NewWriter(c0.cw)
w.WriteUint32(encodeHeader(header{
version: 0,
msgID: 0,
msgType: 42,
}))
w.WriteUint32(0) // Avoids reader closing due to EOF
if !m1.isClosed() {
t.Error("Connection should close due to unknown message type")
}
}
func TestClose(t *testing.T) {
m0 := newTestModel()
m1 := newTestModel()
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
c0.close(nil)
<-c0.closed
if !m0.isClosed() {
t.Fatal("Connection should be closed")
}
// None of these should panic, some should return an error
if c0.ping() {
t.Error("Ping should not return true")
}
c0.Index("default", nil, 0, nil)
c0.Index("default", nil, 0, nil)
if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil {
t.Error("Request should return an error")
}
}
func TestElementSizeExceededNested(t *testing.T) {
m := ClusterConfigMessage{
Folders: []Folder{
{ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"},
},
}
_, err := m.EncodeXDR(ioutil.Discard)
if err == nil {
t.Errorf("ID length %d > max 64, but no error", len(m.Folders[0].ID))
}
}
func TestMarshalIndexMessage(t *testing.T) {
var quickCfg = &quick.Config{MaxCountScale: 10}
if testing.Short() {
quickCfg = nil
}
f := func(m1 IndexMessage) bool {
for _, f := range m1.Files {
for i := range f.Blocks {
f.Blocks[i].Offset = 0
if len(f.Blocks[i].Hash) == 0 {
f.Blocks[i].Hash = nil
}
}
}
return testMarshal(t, "index", &m1, &IndexMessage{})
}
if err := quick.Check(f, quickCfg); err != nil {
t.Error(err)
}
}
func TestMarshalRequestMessage(t *testing.T) {
var quickCfg = &quick.Config{MaxCountScale: 10}
if testing.Short() {
quickCfg = nil
}
f := func(m1 RequestMessage) bool {
return testMarshal(t, "request", &m1, &RequestMessage{})
}
if err := quick.Check(f, quickCfg); err != nil {
t.Error(err)
}
}
func TestMarshalResponseMessage(t *testing.T) {
var quickCfg = &quick.Config{MaxCountScale: 10}
if testing.Short() {
quickCfg = nil
}
f := func(m1 ResponseMessage) bool {
if len(m1.Data) == 0 {
m1.Data = nil
}
return testMarshal(t, "response", &m1, &ResponseMessage{})
}
if err := quick.Check(f, quickCfg); err != nil {
t.Error(err)
}
}
func TestMarshalClusterConfigMessage(t *testing.T) {
var quickCfg = &quick.Config{MaxCountScale: 10}
if testing.Short() {
quickCfg = nil
}
f := func(m1 ClusterConfigMessage) bool {
return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{})
}
if err := quick.Check(f, quickCfg); err != nil {
t.Error(err)
}
}
func TestMarshalCloseMessage(t *testing.T) {
var quickCfg = &quick.Config{MaxCountScale: 10}
if testing.Short() {
quickCfg = nil
}
f := func(m1 CloseMessage) bool {
return testMarshal(t, "close", &m1, &CloseMessage{})
}
if err := quick.Check(f, quickCfg); err != nil {
t.Error(err)
}
}
type message interface {
EncodeXDR(io.Writer) (int, error)
DecodeXDR(io.Reader) error
}
func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
var buf bytes.Buffer
failed := func(bc []byte) {
bs, _ := json.MarshalIndent(m1, "", " ")
ioutil.WriteFile(prefix+"-1.txt", bs, 0644)
bs, _ = json.MarshalIndent(m2, "", " ")
ioutil.WriteFile(prefix+"-2.txt", bs, 0644)
if len(bc) > 0 {
f, _ := os.Create(prefix + "-data.txt")
fmt.Fprint(f, hex.Dump(bc))
f.Close()
}
}
_, err := m1.EncodeXDR(&buf)
if err != nil && strings.Contains(err.Error(), "exceeds size") {
return true
}
if err != nil {
failed(nil)
t.Fatal(err)
}
bc := make([]byte, len(buf.Bytes()))
copy(bc, buf.Bytes())
err = m2.DecodeXDR(&buf)
if err != nil {
failed(bc)
t.Fatal(err)
}
ok := reflect.DeepEqual(m1, m2)
if !ok {
failed(bc)
}
return ok
}
-115
View File
@@ -1,115 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
// The Vector type represents a version vector. The zero value is a usable
// version vector. The vector has slice semantics and some operations on it
// are "append-like" in that they may return the same vector modified, or a
// new allocated Vector with the modified contents.
type Vector []Counter
// Counter represents a single counter in the version vector.
type Counter struct {
ID uint64
Value uint64
}
// Update returns a Vector with the index for the specific ID incremented by
// one. If it is possible, the vector v is updated and returned. If it is not,
// a copy will be created, updated and returned.
func (v Vector) Update(ID uint64) Vector {
for i := range v {
if v[i].ID == ID {
// Update an existing index
v[i].Value++
return v
} else if v[i].ID > ID {
// Insert a new index
nv := make(Vector, len(v)+1)
copy(nv, v[:i])
nv[i].ID = ID
nv[i].Value = 1
copy(nv[i+1:], v[i:])
return nv
}
}
// Append a new new index
return append(v, Counter{ID, 1})
}
// Merge returns the vector containing the maximum indexes from a and b. If it
// is possible, the vector a is updated and returned. If it is not, a copy
// will be created, updated and returned.
func (a Vector) Merge(b Vector) Vector {
var ai, bi int
for bi < len(b) {
if ai == len(a) {
// We've reach the end of a, all that remains are appends
return append(a, b[bi:]...)
}
if a[ai].ID > b[bi].ID {
// The index from b should be inserted here
n := make(Vector, len(a)+1)
copy(n, a[:ai])
n[ai] = b[bi]
copy(n[ai+1:], a[ai:])
a = n
}
if a[ai].ID == b[bi].ID {
if v := b[bi].Value; v > a[ai].Value {
a[ai].Value = v
}
}
if bi < len(b) && a[ai].ID == b[bi].ID {
bi++
}
ai++
}
return a
}
// Copy returns an identical vector that is not shared with v.
func (v Vector) Copy() Vector {
nv := make(Vector, len(v))
copy(nv, v)
return nv
}
// Equal returns true when the two vectors are equivalent.
func (a Vector) Equal(b Vector) bool {
return a.Compare(b) == Equal
}
// LesserEqual returns true when the two vectors are equivalent or a is Lesser
// than b.
func (a Vector) LesserEqual(b Vector) bool {
comp := a.Compare(b)
return comp == Lesser || comp == Equal
}
// LesserEqual returns true when the two vectors are equivalent or a is Greater
// than b.
func (a Vector) GreaterEqual(b Vector) bool {
comp := a.Compare(b)
return comp == Greater || comp == Equal
}
// Concurrent returns true when the two vectors are concrurrent.
func (a Vector) Concurrent(b Vector) bool {
comp := a.Compare(b)
return comp == ConcurrentGreater || comp == ConcurrentLesser
}
// Counter returns the current value of the given counter ID.
func (v Vector) Counter(id uint64) uint64 {
for _, c := range v {
if c.ID == id {
return c.Value
}
}
return 0
}
-89
View File
@@ -1,89 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
// Ordering represents the relationship between two Vectors.
type Ordering int
const (
Equal Ordering = iota
Greater
Lesser
ConcurrentLesser
ConcurrentGreater
)
// There's really no such thing as "concurrent lesser" and "concurrent
// greater" in version vectors, just "concurrent". But it's useful to be able
// to get a strict ordering between versions for stable sorts and so on, so we
// return both variants. The convenience method Concurrent() can be used to
// check for either case.
// Compare returns the Ordering that describes a's relation to b.
func (a Vector) Compare(b Vector) Ordering {
var ai, bi int // index into a and b
var av, bv Counter // value at current index
result := Equal
for ai < len(a) || bi < len(b) {
var aMissing, bMissing bool
if ai < len(a) {
av = a[ai]
} else {
av = Counter{}
aMissing = true
}
if bi < len(b) {
bv = b[bi]
} else {
bv = Counter{}
bMissing = true
}
switch {
case av.ID == bv.ID:
// We have a counter value for each side
if av.Value > bv.Value {
if result == Lesser {
return ConcurrentLesser
}
result = Greater
} else if av.Value < bv.Value {
if result == Greater {
return ConcurrentGreater
}
result = Lesser
}
case !aMissing && av.ID < bv.ID || bMissing:
// Value is missing on the b side
if av.Value > 0 {
if result == Lesser {
return ConcurrentLesser
}
result = Greater
}
case !bMissing && bv.ID < av.ID || aMissing:
// Value is missing on the a side
if bv.Value > 0 {
if result == Greater {
return ConcurrentGreater
}
result = Lesser
}
}
if ai < len(a) && (av.ID <= bv.ID || bMissing) {
ai++
}
if bi < len(b) && (bv.ID <= av.ID || aMissing) {
bi++
}
}
return result
}
@@ -1,249 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
import (
"math"
"testing"
)
func TestCompare(t *testing.T) {
testcases := []struct {
a, b Vector
r Ordering
}{
// Empty vectors are identical
{Vector{}, Vector{}, Equal},
{Vector{}, nil, Equal},
{nil, Vector{}, Equal},
{nil, Vector{Counter{42, 0}}, Equal},
{Vector{}, Vector{Counter{42, 0}}, Equal},
{Vector{Counter{42, 0}}, nil, Equal},
{Vector{Counter{42, 0}}, Vector{}, Equal},
// Zero is the implied value for a missing Counter
{
Vector{Counter{42, 0}},
Vector{Counter{77, 0}},
Equal,
},
// Equal vectors are equal
{
Vector{Counter{42, 33}},
Vector{Counter{42, 33}},
Equal,
},
{
Vector{Counter{42, 33}, Counter{77, 24}},
Vector{Counter{42, 33}, Counter{77, 24}},
Equal,
},
// These a-vectors are all greater than the b-vector
{
Vector{Counter{42, 1}},
nil,
Greater,
},
{
Vector{Counter{42, 1}},
Vector{},
Greater,
},
{
Vector{Counter{0, 1}},
Vector{Counter{0, 0}},
Greater,
},
{
Vector{Counter{42, 1}},
Vector{Counter{42, 0}},
Greater,
},
{
Vector{Counter{math.MaxUint64, 1}},
Vector{Counter{math.MaxUint64, 0}},
Greater,
},
{
Vector{Counter{0, math.MaxUint64}},
Vector{Counter{0, 0}},
Greater,
},
{
Vector{Counter{42, math.MaxUint64}},
Vector{Counter{42, 0}},
Greater,
},
{
Vector{Counter{math.MaxUint64, math.MaxUint64}},
Vector{Counter{math.MaxUint64, 0}},
Greater,
},
{
Vector{Counter{0, math.MaxUint64}},
Vector{Counter{0, math.MaxUint64 - 1}},
Greater,
},
{
Vector{Counter{42, math.MaxUint64}},
Vector{Counter{42, math.MaxUint64 - 1}},
Greater,
},
{
Vector{Counter{math.MaxUint64, math.MaxUint64}},
Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}},
Greater,
},
{
Vector{Counter{42, 2}},
Vector{Counter{42, 1}},
Greater,
},
{
Vector{Counter{22, 22}, Counter{42, 2}},
Vector{Counter{22, 22}, Counter{42, 1}},
Greater,
},
{
Vector{Counter{42, 2}, Counter{77, 3}},
Vector{Counter{42, 1}, Counter{77, 3}},
Greater,
},
{
Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Greater,
},
{
Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Greater,
},
// These a-vectors are all lesser than the b-vector
{nil, Vector{Counter{42, 1}}, Lesser},
{Vector{}, Vector{Counter{42, 1}}, Lesser},
{
Vector{Counter{42, 0}},
Vector{Counter{42, 1}},
Lesser,
},
{
Vector{Counter{42, 1}},
Vector{Counter{42, 2}},
Lesser,
},
{
Vector{Counter{22, 22}, Counter{42, 1}},
Vector{Counter{22, 22}, Counter{42, 2}},
Lesser,
},
{
Vector{Counter{42, 1}, Counter{77, 3}},
Vector{Counter{42, 2}, Counter{77, 3}},
Lesser,
},
{
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
Lesser,
},
{
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
Lesser,
},
// These are all in conflict
{
Vector{Counter{42, 2}},
Vector{Counter{43, 1}},
ConcurrentGreater,
},
{
Vector{Counter{43, 1}},
Vector{Counter{42, 2}},
ConcurrentLesser,
},
{
Vector{Counter{22, 23}, Counter{42, 1}},
Vector{Counter{22, 22}, Counter{42, 2}},
ConcurrentGreater,
},
{
Vector{Counter{22, 21}, Counter{42, 2}},
Vector{Counter{22, 22}, Counter{42, 1}},
ConcurrentLesser,
},
{
Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}},
Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}},
ConcurrentLesser,
},
}
for i, tc := range testcases {
// Test real Compare
if r := tc.a.Compare(tc.b); r != tc.r {
t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r)
}
// Test convenience functions
switch tc.r {
case Greater:
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if !tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v not >= %+v", tc.a, tc.b)
}
if tc.a.LesserEqual(tc.b) {
t.Errorf("%+v <= %+v", tc.a, tc.b)
}
case Lesser:
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v >= %+v", tc.a, tc.b)
}
if !tc.a.LesserEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
case Equal:
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if !tc.a.Equal(tc.b) {
t.Errorf("%+v not == %+v", tc.a, tc.b)
}
if !tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
if !tc.a.LesserEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
case ConcurrentLesser, ConcurrentGreater:
if !tc.a.Concurrent(tc.b) {
t.Errorf("%+v not concurrent %+v", tc.a, tc.b)
}
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v >= %+v", tc.a, tc.b)
}
if tc.a.LesserEqual(tc.b) {
t.Errorf("%+v <= %+v", tc.a, tc.b)
}
}
}
}
-134
View File
@@ -1,134 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
import "testing"
func TestUpdate(t *testing.T) {
var v Vector
// Append
v = v.Update(42)
expected := Vector{Counter{42, 1}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
}
// Insert at front
v = v.Update(36)
expected = Vector{Counter{36, 1}, Counter{42, 1}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
}
// Insert in moddle
v = v.Update(37)
expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
}
// Update existing
v = v.Update(37)
expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
}
}
func TestCopy(t *testing.T) {
v0 := Vector{Counter{42, 1}}
v1 := v0.Copy()
v1.Update(42)
if v0.Compare(v1) != Lesser {
t.Errorf("Copy error, %+v should be ancestor of %+v", v0, v1)
}
}
func TestMerge(t *testing.T) {
testcases := []struct {
a, b, m Vector
}{
// No-ops
{
Vector{},
Vector{},
Vector{},
},
{
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
},
// Appends
{
Vector{},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
},
{
Vector{Counter{22, 1}},
Vector{Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
},
{
Vector{Counter{22, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
},
// Insert
{
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
},
{
Vector{Counter{42, 1}},
Vector{Counter{22, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
},
// Update
{
Vector{Counter{22, 1}, Counter{42, 2}},
Vector{Counter{22, 2}, Counter{42, 1}},
Vector{Counter{22, 2}, Counter{42, 2}},
},
// All of the above
{
Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}},
Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}},
Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}},
},
}
for i, tc := range testcases {
if m := tc.a.Merge(tc.b); m.Compare(tc.m) != Equal {
t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m)
}
}
}
func TestCounterValue(t *testing.T) {
v0 := Vector{Counter{42, 1}, Counter{64, 5}}
if v0.Counter(42) != 1 {
t.Error("Counter error, %d != %d", v0.Counter(42), 1)
}
if v0.Counter(64) != 5 {
t.Error("Counter error, %d != %d", v0.Counter(64), 5)
}
if v0.Counter(72) != 0 {
t.Error("Counter error, %d != %d", v0.Counter(72), 0)
}
}
-38
View File
@@ -1,38 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
// This stuff is hacked up manually because genxdr doesn't support 'type
// Vector []Counter' declarations and it was tricky when I tried to add it...
type xdrWriter interface {
WriteUint32(uint32) (int, error)
WriteUint64(uint64) (int, error)
}
type xdrReader interface {
ReadUint32() uint32
ReadUint64() uint64
}
// EncodeXDRInto encodes the vector as an XDR object into the given XDR
// encoder.
func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) {
w.WriteUint32(uint32(len(v)))
for i := range v {
w.WriteUint64(v[i].ID)
w.WriteUint64(v[i].Value)
}
return 4 + 16*len(v), nil
}
// DecodeXDRFrom decodes the XDR objects from the given reader into itself.
func (v *Vector) DecodeXDRFrom(r xdrReader) error {
l := int(r.ReadUint32())
n := make(Vector, l)
for i := range n {
n[i].ID = r.ReadUint64()
n[i].Value = r.ReadUint64()
}
*v = n
return nil
}
-56
View File
@@ -1,56 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"path/filepath"
"golang.org/x/text/unicode/norm"
)
type wireFormatConnection struct {
next Connection
}
func (c wireFormatConnection) ID() DeviceID {
return c.next.ID()
}
func (c wireFormatConnection) Name() string {
return c.next.Name()
}
func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error {
var myFs = make([]FileInfo, len(fs))
copy(myFs, fs)
for i := range fs {
myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
}
return c.next.Index(folder, myFs, flags, options)
}
func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error {
var myFs = make([]FileInfo, len(fs))
copy(myFs, fs)
for i := range fs {
myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
}
return c.next.IndexUpdate(folder, myFs, flags, options)
}
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
name = norm.NFC.String(filepath.ToSlash(name))
return c.next.Request(folder, name, offset, size, hash, flags, options)
}
func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) {
c.next.ClusterConfig(config)
}
func (c wireFormatConnection) Statistics() Statistics {
return c.next.Statistics()
}
-252
View File
@@ -1,252 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package leveldb
import (
"encoding/binary"
"fmt"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/memdb"
)
type ErrBatchCorrupted struct {
Reason string
}
func (e *ErrBatchCorrupted) Error() string {
return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason)
}
func newErrBatchCorrupted(reason string) error {
return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason})
}
const (
batchHdrLen = 8 + 4
batchGrowRec = 3000
)
type BatchReplay interface {
Put(key, value []byte)
Delete(key []byte)
}
// Batch is a write batch.
type Batch struct {
data []byte
rLen, bLen int
seq uint64
sync bool
}
func (b *Batch) grow(n int) {
off := len(b.data)
if off == 0 {
off = batchHdrLen
if b.data != nil {
b.data = b.data[:off]
}
}
if cap(b.data)-off < n {
if b.data == nil {
b.data = make([]byte, off, off+n)
} else {
odata := b.data
div := 1
if b.rLen > batchGrowRec {
div = b.rLen / batchGrowRec
}
b.data = make([]byte, off, off+n+(off-batchHdrLen)/div)
copy(b.data, odata)
}
}
}
func (b *Batch) appendRec(kt kType, key, value []byte) {
n := 1 + binary.MaxVarintLen32 + len(key)
if kt == ktVal {
n += binary.MaxVarintLen32 + len(value)
}
b.grow(n)
off := len(b.data)
data := b.data[:off+n]
data[off] = byte(kt)
off += 1
off += binary.PutUvarint(data[off:], uint64(len(key)))
copy(data[off:], key)
off += len(key)
if kt == ktVal {
off += binary.PutUvarint(data[off:], uint64(len(value)))
copy(data[off:], value)
off += len(value)
}
b.data = data[:off]
b.rLen++
// Include 8-byte ikey header
b.bLen += len(key) + len(value) + 8
}
// Put appends 'put operation' of the given key/value pair to the batch.
// It is safe to modify the contents of the argument after Put returns.
func (b *Batch) Put(key, value []byte) {
b.appendRec(ktVal, key, value)
}
// Delete appends 'delete operation' of the given key to the batch.
// It is safe to modify the contents of the argument after Delete returns.
func (b *Batch) Delete(key []byte) {
b.appendRec(ktDel, key, nil)
}
// Dump dumps batch contents. The returned slice can be loaded into the
// batch using Load method.
// The returned slice is not its own copy, so the contents should not be
// modified.
func (b *Batch) Dump() []byte {
return b.encode()
}
// Load loads given slice into the batch. Previous contents of the batch
// will be discarded.
// The given slice will not be copied and will be used as batch buffer, so
// it is not safe to modify the contents of the slice.
func (b *Batch) Load(data []byte) error {
return b.decode(0, data)
}
// Replay replays batch contents.
func (b *Batch) Replay(r BatchReplay) error {
return b.decodeRec(func(i int, kt kType, key, value []byte) {
switch kt {
case ktVal:
r.Put(key, value)
case ktDel:
r.Delete(key)
}
})
}
// Len returns number of records in the batch.
func (b *Batch) Len() int {
return b.rLen
}
// Reset resets the batch.
func (b *Batch) Reset() {
b.data = b.data[:0]
b.seq = 0
b.rLen = 0
b.bLen = 0
b.sync = false
}
func (b *Batch) init(sync bool) {
b.sync = sync
}
func (b *Batch) append(p *Batch) {
if p.rLen > 0 {
b.grow(len(p.data) - batchHdrLen)
b.data = append(b.data, p.data[batchHdrLen:]...)
b.rLen += p.rLen
}
if p.sync {
b.sync = true
}
}
// size returns sums of key/value pair length plus 8-bytes ikey.
func (b *Batch) size() int {
return b.bLen
}
func (b *Batch) encode() []byte {
b.grow(0)
binary.LittleEndian.PutUint64(b.data, b.seq)
binary.LittleEndian.PutUint32(b.data[8:], uint32(b.rLen))
return b.data
}
func (b *Batch) decode(prevSeq uint64, data []byte) error {
if len(data) < batchHdrLen {
return newErrBatchCorrupted("too short")
}
b.seq = binary.LittleEndian.Uint64(data)
if b.seq < prevSeq {
return newErrBatchCorrupted("invalid sequence number")
}
b.rLen = int(binary.LittleEndian.Uint32(data[8:]))
if b.rLen < 0 {
return newErrBatchCorrupted("invalid records length")
}
// No need to be precise at this point, it won't be used anyway
b.bLen = len(data) - batchHdrLen
b.data = data
return nil
}
func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) {
off := batchHdrLen
for i := 0; i < b.rLen; i++ {
if off >= len(b.data) {
return newErrBatchCorrupted("invalid records length")
}
kt := kType(b.data[off])
if kt > ktVal {
return newErrBatchCorrupted("bad record: invalid type")
}
off += 1
x, n := binary.Uvarint(b.data[off:])
off += n
if n <= 0 || off+int(x) > len(b.data) {
return newErrBatchCorrupted("bad record: invalid key length")
}
key := b.data[off : off+int(x)]
off += int(x)
var value []byte
if kt == ktVal {
x, n := binary.Uvarint(b.data[off:])
off += n
if n <= 0 || off+int(x) > len(b.data) {
return newErrBatchCorrupted("bad record: invalid value length")
}
value = b.data[off : off+int(x)]
off += int(x)
}
f(i, kt, key, value)
}
return nil
}
func (b *Batch) memReplay(to *memdb.DB) error {
return b.decodeRec(func(i int, kt kType, key, value []byte) {
ikey := newIkey(key, b.seq+uint64(i), kt)
to.Put(ikey, value)
})
}
func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) error {
if err := b.decode(prevSeq, data); err != nil {
return err
}
return b.memReplay(to)
}
func (b *Batch) revertMemReplay(to *memdb.DB) error {
return b.decodeRec(func(i int, kt kType, key, value []byte) {
ikey := newIkey(key, b.seq+uint64(i), kt)
to.Delete(ikey)
})
}
-120
View File
@@ -1,120 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package leveldb
import (
"bytes"
"testing"
"github.com/syndtr/goleveldb/leveldb/comparer"
"github.com/syndtr/goleveldb/leveldb/memdb"
)
type tbRec struct {
kt kType
key, value []byte
}
type testBatch struct {
rec []*tbRec
}
func (p *testBatch) Put(key, value []byte) {
p.rec = append(p.rec, &tbRec{ktVal, key, value})
}
func (p *testBatch) Delete(key []byte) {
p.rec = append(p.rec, &tbRec{ktDel, key, nil})
}
func compareBatch(t *testing.T, b1, b2 *Batch) {
if b1.seq != b2.seq {
t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
}
if b1.Len() != b2.Len() {
t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len())
}
p1, p2 := new(testBatch), new(testBatch)
err := b1.Replay(p1)
if err != nil {
t.Fatal("error when replaying batch 1: ", err)
}
err = b2.Replay(p2)
if err != nil {
t.Fatal("error when replaying batch 2: ", err)
}
for i := range p1.rec {
r1, r2 := p1.rec[i], p2.rec[i]
if r1.kt != r2.kt {
t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt)
}
if !bytes.Equal(r1.key, r2.key) {
t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
}
if r1.kt == ktVal {
if !bytes.Equal(r1.value, r2.value) {
t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
}
}
}
}
func TestBatch_EncodeDecode(t *testing.T) {
b1 := new(Batch)
b1.seq = 10009
b1.Put([]byte("key1"), []byte("value1"))
b1.Put([]byte("key2"), []byte("value2"))
b1.Delete([]byte("key1"))
b1.Put([]byte("k"), []byte(""))
b1.Put([]byte("zzzzzzzzzzz"), []byte("zzzzzzzzzzzzzzzzzzzzzzzz"))
b1.Delete([]byte("key10000"))
b1.Delete([]byte("k"))
buf := b1.encode()
b2 := new(Batch)
err := b2.decode(0, buf)
if err != nil {
t.Error("error when decoding batch: ", err)
}
compareBatch(t, b1, b2)
}
func TestBatch_Append(t *testing.T) {
b1 := new(Batch)
b1.seq = 10009
b1.Put([]byte("key1"), []byte("value1"))
b1.Put([]byte("key2"), []byte("value2"))
b1.Delete([]byte("key1"))
b1.Put([]byte("foo"), []byte("foovalue"))
b1.Put([]byte("bar"), []byte("barvalue"))
b2a := new(Batch)
b2a.seq = 10009
b2a.Put([]byte("key1"), []byte("value1"))
b2a.Put([]byte("key2"), []byte("value2"))
b2a.Delete([]byte("key1"))
b2b := new(Batch)
b2b.Put([]byte("foo"), []byte("foovalue"))
b2b.Put([]byte("bar"), []byte("barvalue"))
b2a.append(b2b)
compareBatch(t, b1, b2a)
}
func TestBatch_Size(t *testing.T) {
b := new(Batch)
for i := 0; i < 2; i++ {
b.Put([]byte("key1"), []byte("value1"))
b.Put([]byte("key2"), []byte("value2"))
b.Delete([]byte("key1"))
b.Put([]byte("foo"), []byte("foovalue"))
b.Put([]byte("bar"), []byte("barvalue"))
mem := memdb.New(&iComparer{comparer.DefaultComparer}, 0)
b.memReplay(mem)
if b.size() != mem.Size() {
t.Errorf("invalid batch size calculation, want=%d got=%d", mem.Size(), b.size())
}
b.Reset()
}
}
@@ -1,58 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// +build !go1.2
package leveldb
import (
"sync/atomic"
"testing"
)
func BenchmarkDBReadConcurrent(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.gc()
defer p.close()
b.ResetTimer()
b.SetBytes(116)
b.RunParallel(func(pb *testing.PB) {
iter := p.newIter()
defer iter.Release()
for pb.Next() && iter.Next() {
}
})
}
func BenchmarkDBReadConcurrent2(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.gc()
defer p.close()
b.ResetTimer()
b.SetBytes(116)
var dir uint32
b.RunParallel(func(pb *testing.PB) {
iter := p.newIter()
defer iter.Release()
if atomic.AddUint32(&dir, 1)%2 == 0 {
for pb.Next() && iter.Next() {
}
} else {
if pb.Next() && iter.Last() {
for pb.Next() && iter.Prev() {
}
}
}
})
}
-464
View File
@@ -1,464 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package leveldb
import (
"bytes"
"fmt"
"math/rand"
"os"
"path/filepath"
"runtime"
"testing"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
)
func randomString(r *rand.Rand, n int) []byte {
b := new(bytes.Buffer)
for i := 0; i < n; i++ {
b.WriteByte(' ' + byte(r.Intn(95)))
}
return b.Bytes()
}
func compressibleStr(r *rand.Rand, frac float32, n int) []byte {
nn := int(float32(n) * frac)
rb := randomString(r, nn)
b := make([]byte, 0, n+nn)
for len(b) < n {
b = append(b, rb...)
}
return b[:n]
}
type valueGen struct {
src []byte
pos int
}
func newValueGen(frac float32) *valueGen {
v := new(valueGen)
r := rand.New(rand.NewSource(301))
v.src = make([]byte, 0, 1048576+100)
for len(v.src) < 1048576 {
v.src = append(v.src, compressibleStr(r, frac, 100)...)
}
return v
}
func (v *valueGen) get(n int) []byte {
if v.pos+n > len(v.src) {
v.pos = 0
}
v.pos += n
return v.src[v.pos-n : v.pos]
}
var benchDB = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbbench-%d", os.Getuid()))
type dbBench struct {
b *testing.B
stor storage.Storage
db *DB
o *opt.Options
ro *opt.ReadOptions
wo *opt.WriteOptions
keys, values [][]byte
}
func openDBBench(b *testing.B, noCompress bool) *dbBench {
_, err := os.Stat(benchDB)
if err == nil {
err = os.RemoveAll(benchDB)
if err != nil {
b.Fatal("cannot remove old db: ", err)
}
}
p := &dbBench{
b: b,
o: &opt.Options{},
ro: &opt.ReadOptions{},
wo: &opt.WriteOptions{},
}
p.stor, err = storage.OpenFile(benchDB)
if err != nil {
b.Fatal("cannot open stor: ", err)
}
if noCompress {
p.o.Compression = opt.NoCompression
}
p.db, err = Open(p.stor, p.o)
if err != nil {
b.Fatal("cannot open db: ", err)
}
runtime.GOMAXPROCS(runtime.NumCPU())
return p
}
func (p *dbBench) reopen() {
p.db.Close()
var err error
p.db, err = Open(p.stor, p.o)
if err != nil {
p.b.Fatal("Reopen: got error: ", err)
}
}
func (p *dbBench) populate(n int) {
p.keys, p.values = make([][]byte, n), make([][]byte, n)
v := newValueGen(0.5)
for i := range p.keys {
p.keys[i], p.values[i] = []byte(fmt.Sprintf("%016d", i)), v.get(100)
}
}
func (p *dbBench) randomize() {
m := len(p.keys)
times := m * 2
r1, r2 := rand.New(rand.NewSource(0xdeadbeef)), rand.New(rand.NewSource(0xbeefface))
for n := 0; n < times; n++ {
i, j := r1.Int()%m, r2.Int()%m
if i == j {
continue
}
p.keys[i], p.keys[j] = p.keys[j], p.keys[i]
p.values[i], p.values[j] = p.values[j], p.values[i]
}
}
func (p *dbBench) writes(perBatch int) {
b := p.b
db := p.db
n := len(p.keys)
m := n / perBatch
if n%perBatch > 0 {
m++
}
batches := make([]Batch, m)
j := 0
for i := range batches {
first := true
for ; j < n && ((j+1)%perBatch != 0 || first); j++ {
first = false
batches[i].Put(p.keys[j], p.values[j])
}
}
runtime.GC()
b.ResetTimer()
b.StartTimer()
for i := range batches {
err := db.Write(&(batches[i]), p.wo)
if err != nil {
b.Fatal("write failed: ", err)
}
}
b.StopTimer()
b.SetBytes(116)
}
func (p *dbBench) gc() {
p.keys, p.values = nil, nil
runtime.GC()
}
func (p *dbBench) puts() {
b := p.b
db := p.db
b.ResetTimer()
b.StartTimer()
for i := range p.keys {
err := db.Put(p.keys[i], p.values[i], p.wo)
if err != nil {
b.Fatal("put failed: ", err)
}
}
b.StopTimer()
b.SetBytes(116)
}
func (p *dbBench) fill() {
b := p.b
db := p.db
perBatch := 10000
batch := new(Batch)
for i, n := 0, len(p.keys); i < n; {
first := true
for ; i < n && ((i+1)%perBatch != 0 || first); i++ {
first = false
batch.Put(p.keys[i], p.values[i])
}
err := db.Write(batch, p.wo)
if err != nil {
b.Fatal("write failed: ", err)
}
batch.Reset()
}
}
func (p *dbBench) gets() {
b := p.b
db := p.db
b.ResetTimer()
for i := range p.keys {
_, err := db.Get(p.keys[i], p.ro)
if err != nil {
b.Error("got error: ", err)
}
}
b.StopTimer()
}
func (p *dbBench) seeks() {
b := p.b
iter := p.newIter()
defer iter.Release()
b.ResetTimer()
for i := range p.keys {
if !iter.Seek(p.keys[i]) {
b.Error("value not found for: ", string(p.keys[i]))
}
}
b.StopTimer()
}
func (p *dbBench) newIter() iterator.Iterator {
iter := p.db.NewIterator(nil, p.ro)
err := iter.Error()
if err != nil {
p.b.Fatal("cannot create iterator: ", err)
}
return iter
}
func (p *dbBench) close() {
if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
p.b.Log("Block pool stats: ", bp)
}
p.db.Close()
p.stor.Close()
os.RemoveAll(benchDB)
p.db = nil
p.keys = nil
p.values = nil
runtime.GC()
runtime.GOMAXPROCS(1)
}
func BenchmarkDBWrite(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.writes(1)
p.close()
}
func BenchmarkDBWriteBatch(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.writes(1000)
p.close()
}
func BenchmarkDBWriteUncompressed(b *testing.B) {
p := openDBBench(b, true)
p.populate(b.N)
p.writes(1)
p.close()
}
func BenchmarkDBWriteBatchUncompressed(b *testing.B) {
p := openDBBench(b, true)
p.populate(b.N)
p.writes(1000)
p.close()
}
func BenchmarkDBWriteRandom(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.randomize()
p.writes(1)
p.close()
}
func BenchmarkDBWriteRandomSync(b *testing.B) {
p := openDBBench(b, false)
p.wo.Sync = true
p.populate(b.N)
p.writes(1)
p.close()
}
func BenchmarkDBOverwrite(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.writes(1)
p.writes(1)
p.close()
}
func BenchmarkDBOverwriteRandom(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.writes(1)
p.randomize()
p.writes(1)
p.close()
}
func BenchmarkDBPut(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.puts()
p.close()
}
func BenchmarkDBRead(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.gc()
iter := p.newIter()
b.ResetTimer()
for iter.Next() {
}
iter.Release()
b.StopTimer()
b.SetBytes(116)
p.close()
}
func BenchmarkDBReadGC(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
iter := p.newIter()
b.ResetTimer()
for iter.Next() {
}
iter.Release()
b.StopTimer()
b.SetBytes(116)
p.close()
}
func BenchmarkDBReadUncompressed(b *testing.B) {
p := openDBBench(b, true)
p.populate(b.N)
p.fill()
p.gc()
iter := p.newIter()
b.ResetTimer()
for iter.Next() {
}
iter.Release()
b.StopTimer()
b.SetBytes(116)
p.close()
}
func BenchmarkDBReadTable(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.reopen()
p.gc()
iter := p.newIter()
b.ResetTimer()
for iter.Next() {
}
iter.Release()
b.StopTimer()
b.SetBytes(116)
p.close()
}
func BenchmarkDBReadReverse(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.gc()
iter := p.newIter()
b.ResetTimer()
iter.Last()
for iter.Prev() {
}
iter.Release()
b.StopTimer()
b.SetBytes(116)
p.close()
}
func BenchmarkDBReadReverseTable(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.reopen()
p.gc()
iter := p.newIter()
b.ResetTimer()
iter.Last()
for iter.Prev() {
}
iter.Release()
b.StopTimer()
b.SetBytes(116)
p.close()
}
func BenchmarkDBSeek(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.seeks()
p.close()
}
func BenchmarkDBSeekRandom(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.randomize()
p.seeks()
p.close()
}
func BenchmarkDBGet(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.gets()
p.close()
}
func BenchmarkDBGetRandom(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.randomize()
p.gets()
p.close()
}
@@ -1,30 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// +build !go1.2
package cache
import (
"math/rand"
"testing"
)
func BenchmarkLRUCache(b *testing.B) {
c := NewCache(NewLRU(10000))
b.SetParallelism(10)
b.RunParallel(func(pb *testing.PB) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for pb.Next() {
key := uint64(r.Intn(1000000))
c.Get(0, key, func() (int, Value) {
return 1, key
}).Release()
}
})
}
@@ -1,676 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Package cache provides interface and implementation of a cache algorithms.
package cache
import (
"sync"
"sync/atomic"
"unsafe"
"github.com/syndtr/goleveldb/leveldb/util"
)
// Cacher provides interface to implements a caching functionality.
// An implementation must be goroutine-safe.
type Cacher interface {
// Capacity returns cache capacity.
Capacity() int
// SetCapacity sets cache capacity.
SetCapacity(capacity int)
// Promote promotes the 'cache node'.
Promote(n *Node)
// Ban evicts the 'cache node' and prevent subsequent 'promote'.
Ban(n *Node)
// Evict evicts the 'cache node'.
Evict(n *Node)
// EvictNS evicts 'cache node' with the given namespace.
EvictNS(ns uint64)
// EvictAll evicts all 'cache node'.
EvictAll()
// Close closes the 'cache tree'
Close() error
}
// Value is a 'cacheable object'. It may implements util.Releaser, if
// so the the Release method will be called once object is released.
type Value interface{}
type CacheGetter struct {
Cache *Cache
NS uint64
}
func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
return g.Cache.Get(g.NS, key, setFunc)
}
// The hash tables implementation is based on:
// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
const (
mInitialSize = 1 << 4
mOverflowThreshold = 1 << 5
mOverflowGrowThreshold = 1 << 7
)
type mBucket struct {
mu sync.Mutex
node []*Node
frozen bool
}
func (b *mBucket) freeze() []*Node {
b.mu.Lock()
defer b.mu.Unlock()
if !b.frozen {
b.frozen = true
}
return b.node
}
func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
b.mu.Lock()
if b.frozen {
b.mu.Unlock()
return
}
// Scan the node.
for _, n := range b.node {
if n.hash == hash && n.ns == ns && n.key == key {
atomic.AddInt32(&n.ref, 1)
b.mu.Unlock()
return true, false, n
}
}
// Get only.
if noset {
b.mu.Unlock()
return true, false, nil
}
// Create node.
n = &Node{
r: r,
hash: hash,
ns: ns,
key: key,
ref: 1,
}
// Add node to bucket.
b.node = append(b.node, n)
bLen := len(b.node)
b.mu.Unlock()
// Update counter.
grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
if bLen > mOverflowThreshold {
grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
}
// Grow.
if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
nhLen := len(h.buckets) << 1
nh := &mNode{
buckets: make([]unsafe.Pointer, nhLen),
mask: uint32(nhLen) - 1,
pred: unsafe.Pointer(h),
growThreshold: int32(nhLen * mOverflowThreshold),
shrinkThreshold: int32(nhLen >> 1),
}
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
if !ok {
panic("BUG: failed swapping head")
}
go nh.initBuckets()
}
return true, true, n
}
func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
b.mu.Lock()
if b.frozen {
b.mu.Unlock()
return
}
// Scan the node.
var (
n *Node
bLen int
)
for i := range b.node {
n = b.node[i]
if n.ns == ns && n.key == key {
if atomic.LoadInt32(&n.ref) == 0 {
deleted = true
// Call releaser.
if n.value != nil {
if r, ok := n.value.(util.Releaser); ok {
r.Release()
}
n.value = nil
}
// Remove node from bucket.
b.node = append(b.node[:i], b.node[i+1:]...)
bLen = len(b.node)
}
break
}
}
b.mu.Unlock()
if deleted {
// Call OnDel.
for _, f := range n.onDel {
f()
}
// Update counter.
atomic.AddInt32(&r.size, int32(n.size)*-1)
shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
if bLen >= mOverflowThreshold {
atomic.AddInt32(&h.overflow, -1)
}
// Shrink.
if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
nhLen := len(h.buckets) >> 1
nh := &mNode{
buckets: make([]unsafe.Pointer, nhLen),
mask: uint32(nhLen) - 1,
pred: unsafe.Pointer(h),
growThreshold: int32(nhLen * mOverflowThreshold),
shrinkThreshold: int32(nhLen >> 1),
}
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
if !ok {
panic("BUG: failed swapping head")
}
go nh.initBuckets()
}
}
return true, deleted
}
type mNode struct {
buckets []unsafe.Pointer // []*mBucket
mask uint32
pred unsafe.Pointer // *mNode
resizeInProgess int32
overflow int32
growThreshold int32
shrinkThreshold int32
}
func (n *mNode) initBucket(i uint32) *mBucket {
if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
return b
}
p := (*mNode)(atomic.LoadPointer(&n.pred))
if p != nil {
var node []*Node
if n.mask > p.mask {
// Grow.
pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
if pb == nil {
pb = p.initBucket(i & p.mask)
}
m := pb.freeze()
// Split nodes.
for _, x := range m {
if x.hash&n.mask == i {
node = append(node, x)
}
}
} else {
// Shrink.
pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
if pb0 == nil {
pb0 = p.initBucket(i)
}
pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
if pb1 == nil {
pb1 = p.initBucket(i + uint32(len(n.buckets)))
}
m0 := pb0.freeze()
m1 := pb1.freeze()
// Merge nodes.
node = make([]*Node, 0, len(m0)+len(m1))
node = append(node, m0...)
node = append(node, m1...)
}
b := &mBucket{node: node}
if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
if len(node) > mOverflowThreshold {
atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
}
return b
}
}
return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
}
func (n *mNode) initBuckets() {
for i := range n.buckets {
n.initBucket(uint32(i))
}
atomic.StorePointer(&n.pred, nil)
}
// Cache is a 'cache map'.
type Cache struct {
mu sync.RWMutex
mHead unsafe.Pointer // *mNode
nodes int32
size int32
cacher Cacher
closed bool
}
// NewCache creates a new 'cache map'. The cacher is optional and
// may be nil.
func NewCache(cacher Cacher) *Cache {
h := &mNode{
buckets: make([]unsafe.Pointer, mInitialSize),
mask: mInitialSize - 1,
growThreshold: int32(mInitialSize * mOverflowThreshold),
shrinkThreshold: 0,
}
for i := range h.buckets {
h.buckets[i] = unsafe.Pointer(&mBucket{})
}
r := &Cache{
mHead: unsafe.Pointer(h),
cacher: cacher,
}
return r
}
func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
h := (*mNode)(atomic.LoadPointer(&r.mHead))
i := hash & h.mask
b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
if b == nil {
b = h.initBucket(i)
}
return h, b
}
func (r *Cache) delete(n *Node) bool {
for {
h, b := r.getBucket(n.hash)
done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
if done {
return deleted
}
}
return false
}
// Nodes returns number of 'cache node' in the map.
func (r *Cache) Nodes() int {
return int(atomic.LoadInt32(&r.nodes))
}
// Size returns sums of 'cache node' size in the map.
func (r *Cache) Size() int {
return int(atomic.LoadInt32(&r.size))
}
// Capacity returns cache capacity.
func (r *Cache) Capacity() int {
if r.cacher == nil {
return 0
}
return r.cacher.Capacity()
}
// SetCapacity sets cache capacity.
func (r *Cache) SetCapacity(capacity int) {
if r.cacher != nil {
r.cacher.SetCapacity(capacity)
}
}
// Get gets 'cache node' with the given namespace and key.
// If cache node is not found and setFunc is not nil, Get will atomically creates
// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
//
// The returned 'cache handle' should be released after use by calling Release
// method.
func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return nil
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
if done {
if n != nil {
n.mu.Lock()
if n.value == nil {
if setFunc == nil {
n.mu.Unlock()
n.unref()
return nil
}
n.size, n.value = setFunc()
if n.value == nil {
n.size = 0
n.mu.Unlock()
n.unref()
return nil
}
atomic.AddInt32(&r.size, int32(n.size))
}
n.mu.Unlock()
if r.cacher != nil {
r.cacher.Promote(n)
}
return &Handle{unsafe.Pointer(n)}
}
break
}
}
return nil
}
// Delete removes and ban 'cache node' with the given namespace and key.
// A banned 'cache node' will never inserted into the 'cache tree'. Ban
// only attributed to the particular 'cache node', so when a 'cache node'
// is recreated it will not be banned.
//
// If onDel is not nil, then it will be executed if such 'cache node'
// doesn't exist or once the 'cache node' is released.
//
// Delete return true is such 'cache node' exist.
func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return false
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, true)
if done {
if n != nil {
if onDel != nil {
n.mu.Lock()
n.onDel = append(n.onDel, onDel)
n.mu.Unlock()
}
if r.cacher != nil {
r.cacher.Ban(n)
}
n.unref()
return true
}
break
}
}
if onDel != nil {
onDel()
}
return false
}
// Evict evicts 'cache node' with the given namespace and key. This will
// simply call Cacher.Evict.
//
// Evict return true is such 'cache node' exist.
func (r *Cache) Evict(ns, key uint64) bool {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return false
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, true)
if done {
if n != nil {
if r.cacher != nil {
r.cacher.Evict(n)
}
n.unref()
return true
}
break
}
}
return false
}
// EvictNS evicts 'cache node' with the given namespace. This will
// simply call Cacher.EvictNS.
func (r *Cache) EvictNS(ns uint64) {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return
}
if r.cacher != nil {
r.cacher.EvictNS(ns)
}
}
// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
func (r *Cache) EvictAll() {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return
}
if r.cacher != nil {
r.cacher.EvictAll()
}
}
// Close closes the 'cache map' and releases all 'cache node'.
func (r *Cache) Close() error {
r.mu.Lock()
if !r.closed {
r.closed = true
if r.cacher != nil {
if err := r.cacher.Close(); err != nil {
return err
}
}
h := (*mNode)(r.mHead)
h.initBuckets()
for i := range h.buckets {
b := (*mBucket)(h.buckets[i])
for _, n := range b.node {
// Call releaser.
if n.value != nil {
if r, ok := n.value.(util.Releaser); ok {
r.Release()
}
n.value = nil
}
// Call OnDel.
for _, f := range n.onDel {
f()
}
}
}
}
r.mu.Unlock()
return nil
}
// Node is a 'cache node'.
type Node struct {
r *Cache
hash uint32
ns, key uint64
mu sync.Mutex
size int
value Value
ref int32
onDel []func()
CacheData unsafe.Pointer
}
// NS returns this 'cache node' namespace.
func (n *Node) NS() uint64 {
return n.ns
}
// Key returns this 'cache node' key.
func (n *Node) Key() uint64 {
return n.key
}
// Size returns this 'cache node' size.
func (n *Node) Size() int {
return n.size
}
// Value returns this 'cache node' value.
func (n *Node) Value() Value {
return n.value
}
// Ref returns this 'cache node' ref counter.
func (n *Node) Ref() int32 {
return atomic.LoadInt32(&n.ref)
}
// GetHandle returns an handle for this 'cache node'.
func (n *Node) GetHandle() *Handle {
if atomic.AddInt32(&n.ref, 1) <= 1 {
panic("BUG: Node.GetHandle on zero ref")
}
return &Handle{unsafe.Pointer(n)}
}
func (n *Node) unref() {
if atomic.AddInt32(&n.ref, -1) == 0 {
n.r.delete(n)
}
}
func (n *Node) unrefLocked() {
if atomic.AddInt32(&n.ref, -1) == 0 {
n.r.mu.RLock()
if !n.r.closed {
n.r.delete(n)
}
n.r.mu.RUnlock()
}
}
type Handle struct {
n unsafe.Pointer // *Node
}
func (h *Handle) Value() Value {
n := (*Node)(atomic.LoadPointer(&h.n))
if n != nil {
return n.value
}
return nil
}
func (h *Handle) Release() {
nPtr := atomic.LoadPointer(&h.n)
if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
n := (*Node)(nPtr)
n.unrefLocked()
}
}
func murmur32(ns, key uint64, seed uint32) uint32 {
const (
m = uint32(0x5bd1e995)
r = 24
)
k1 := uint32(ns >> 32)
k2 := uint32(ns)
k3 := uint32(key >> 32)
k4 := uint32(key)
k1 *= m
k1 ^= k1 >> r
k1 *= m
k2 *= m
k2 ^= k2 >> r
k2 *= m
k3 *= m
k3 ^= k3 >> r
k3 *= m
k4 *= m
k4 ^= k4 >> r
k4 *= m
h := seed
h *= m
h ^= k1
h *= m
h ^= k2
h *= m
h ^= k3
h *= m
h ^= k4
h ^= h >> 13
h *= m
h ^= h >> 15
return h
}
@@ -1,554 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cache
import (
"math/rand"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"unsafe"
)
type int32o int32
func (o *int32o) acquire() {
if atomic.AddInt32((*int32)(o), 1) != 1 {
panic("BUG: invalid ref")
}
}
func (o *int32o) Release() {
if atomic.AddInt32((*int32)(o), -1) != 0 {
panic("BUG: invalid ref")
}
}
type releaserFunc struct {
fn func()
value Value
}
func (r releaserFunc) Release() {
if r.fn != nil {
r.fn()
}
}
func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle {
return c.Get(ns, key, func() (int, Value) {
if relf != nil {
return charge, releaserFunc{relf, value}
} else {
return charge, value
}
})
}
func TestCacheMap(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
nsx := []struct {
nobjects, nhandles, concurrent, repeat int
}{
{10000, 400, 50, 3},
{100000, 1000, 100, 10},
}
var (
objects [][]int32o
handles [][]unsafe.Pointer
)
for _, x := range nsx {
objects = append(objects, make([]int32o, x.nobjects))
handles = append(handles, make([]unsafe.Pointer, x.nhandles))
}
c := NewCache(nil)
wg := new(sync.WaitGroup)
var done int32
for ns, x := range nsx {
for i := 0; i < x.concurrent; i++ {
wg.Add(1)
go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for j := len(objects) * repeat; j >= 0; j-- {
key := uint64(r.Intn(len(objects)))
h := c.Get(uint64(ns), key, func() (int, Value) {
o := &objects[key]
o.acquire()
return 1, o
})
if v := h.Value().(*int32o); v != &objects[key] {
t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v)
}
if objects[key] != 1 {
t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key])
}
if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) {
h.Release()
}
}
}(ns, i, x.repeat, objects[ns], handles[ns])
}
go func(handles []unsafe.Pointer) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for atomic.LoadInt32(&done) == 0 {
i := r.Intn(len(handles))
h := (*Handle)(atomic.LoadPointer(&handles[i]))
if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) {
h.Release()
}
time.Sleep(time.Millisecond)
}
}(handles[ns])
}
go func() {
handles := make([]*Handle, 100000)
for atomic.LoadInt32(&done) == 0 {
for i := range handles {
handles[i] = c.Get(999999999, uint64(i), func() (int, Value) {
return 1, 1
})
}
for _, h := range handles {
h.Release()
}
}
}()
wg.Wait()
atomic.StoreInt32(&done, 1)
for _, handles0 := range handles {
for i := range handles0 {
h := (*Handle)(atomic.LoadPointer(&handles0[i]))
if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) {
h.Release()
}
}
}
for ns, objects0 := range objects {
for i, o := range objects0 {
if o != 0 {
t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o)
}
}
}
}
func TestCacheMap_NodesAndSize(t *testing.T) {
c := NewCache(nil)
if c.Nodes() != 0 {
t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
}
if c.Size() != 0 {
t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
}
set(c, 0, 1, 1, 1, nil)
set(c, 0, 2, 2, 2, nil)
set(c, 1, 1, 3, 3, nil)
set(c, 2, 1, 4, 1, nil)
if c.Nodes() != 4 {
t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes())
}
if c.Size() != 7 {
t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size())
}
}
func TestLRUCache_Capacity(t *testing.T) {
c := NewCache(NewLRU(10))
if c.Capacity() != 10 {
t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity())
}
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 2, nil).Release()
set(c, 1, 1, 3, 3, nil).Release()
set(c, 2, 1, 4, 1, nil).Release()
set(c, 2, 2, 5, 1, nil).Release()
set(c, 2, 3, 6, 1, nil).Release()
set(c, 2, 4, 7, 1, nil).Release()
set(c, 2, 5, 8, 1, nil).Release()
if c.Nodes() != 7 {
t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes())
}
if c.Size() != 10 {
t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size())
}
c.SetCapacity(9)
if c.Capacity() != 9 {
t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity())
}
if c.Nodes() != 6 {
t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes())
}
if c.Size() != 8 {
t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size())
}
}
func TestCacheMap_NilValue(t *testing.T) {
c := NewCache(NewLRU(10))
h := c.Get(0, 0, func() (size int, value Value) {
return 1, nil
})
if h != nil {
t.Error("cache handle is non-nil")
}
if c.Nodes() != 0 {
t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
}
if c.Size() != 0 {
t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
}
}
func TestLRUCache_GetLatency(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
const (
concurrentSet = 30
concurrentGet = 3
duration = 3 * time.Second
delay = 3 * time.Millisecond
maxkey = 100000
)
var (
set, getHit, getAll int32
getMaxLatency, getDuration int64
)
c := NewCache(NewLRU(5000))
wg := &sync.WaitGroup{}
until := time.Now().Add(duration)
for i := 0; i < concurrentSet; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for time.Now().Before(until) {
c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) {
time.Sleep(delay)
atomic.AddInt32(&set, 1)
return 1, 1
}).Release()
}
}(i)
}
for i := 0; i < concurrentGet; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for {
mark := time.Now()
if mark.Before(until) {
h := c.Get(0, uint64(r.Intn(maxkey)), nil)
latency := int64(time.Now().Sub(mark))
m := atomic.LoadInt64(&getMaxLatency)
if latency > m {
atomic.CompareAndSwapInt64(&getMaxLatency, m, latency)
}
atomic.AddInt64(&getDuration, latency)
if h != nil {
atomic.AddInt32(&getHit, 1)
h.Release()
}
atomic.AddInt32(&getAll, 1)
} else {
break
}
}
}(i)
}
wg.Wait()
getAvglatency := time.Duration(getDuration) / time.Duration(getAll)
t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v",
set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency)
if getAvglatency > delay/3 {
t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency)
}
}
func TestLRUCache_HitMiss(t *testing.T) {
cases := []struct {
key uint64
value string
}{
{1, "vvvvvvvvv"},
{100, "v1"},
{0, "v2"},
{12346, "v3"},
{777, "v4"},
{999, "v5"},
{7654, "v6"},
{2, "v7"},
{3, "v8"},
{9, "v9"},
}
setfin := 0
c := NewCache(NewLRU(1000))
for i, x := range cases {
set(c, 0, x.key, x.value, len(x.value), func() {
setfin++
}).Release()
for j, y := range cases {
h := c.Get(0, y.key, nil)
if j <= i {
// should hit
if h == nil {
t.Errorf("case '%d' iteration '%d' is miss", i, j)
} else {
if x := h.Value().(releaserFunc).value.(string); x != y.value {
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
}
}
} else {
// should miss
if h != nil {
t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string))
}
}
if h != nil {
h.Release()
}
}
}
for i, x := range cases {
finalizerOk := false
c.Delete(0, x.key, func() {
finalizerOk = true
})
if !finalizerOk {
t.Errorf("case %d delete finalizer not executed", i)
}
for j, y := range cases {
h := c.Get(0, y.key, nil)
if j > i {
// should hit
if h == nil {
t.Errorf("case '%d' iteration '%d' is miss", i, j)
} else {
if x := h.Value().(releaserFunc).value.(string); x != y.value {
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
}
}
} else {
// should miss
if h != nil {
t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string))
}
}
if h != nil {
h.Release()
}
}
}
if setfin != len(cases) {
t.Errorf("some set finalizer may not be executed, want=%d got=%d", len(cases), setfin)
}
}
func TestLRUCache_Eviction(t *testing.T) {
c := NewCache(NewLRU(12))
o1 := set(c, 0, 1, 1, 1, nil)
set(c, 0, 2, 2, 1, nil).Release()
set(c, 0, 3, 3, 1, nil).Release()
set(c, 0, 4, 4, 1, nil).Release()
set(c, 0, 5, 5, 1, nil).Release()
if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2
h.Release()
}
set(c, 0, 9, 9, 10, nil).Release() // 5,2,9
for _, key := range []uint64{9, 2, 5, 1} {
h := c.Get(0, key, nil)
if h == nil {
t.Errorf("miss for key '%d'", key)
} else {
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
h.Release()
}
}
o1.Release()
for _, key := range []uint64{1, 2, 5} {
h := c.Get(0, key, nil)
if h == nil {
t.Errorf("miss for key '%d'", key)
} else {
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
h.Release()
}
}
for _, key := range []uint64{3, 4, 9} {
h := c.Get(0, key, nil)
if h != nil {
t.Errorf("hit for key '%d'", key)
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
h.Release()
}
}
}
func TestLRUCache_Evict(t *testing.T) {
c := NewCache(NewLRU(6))
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 1, nil).Release()
set(c, 1, 1, 4, 1, nil).Release()
set(c, 1, 2, 5, 1, nil).Release()
set(c, 2, 1, 6, 1, nil).Release()
set(c, 2, 2, 7, 1, nil).Release()
for ns := 0; ns < 3; ns++ {
for key := 1; key < 3; key++ {
if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
h.Release()
} else {
t.Errorf("Cache.Get on #%d.%d return nil", ns, key)
}
}
}
if ok := c.Evict(0, 1); !ok {
t.Error("first Cache.Evict on #0.1 return false")
}
if ok := c.Evict(0, 1); ok {
t.Error("second Cache.Evict on #0.1 return true")
}
if h := c.Get(0, 1, nil); h != nil {
t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value())
}
c.EvictNS(1)
if h := c.Get(1, 1, nil); h != nil {
t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value())
}
if h := c.Get(1, 2, nil); h != nil {
t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value())
}
c.EvictAll()
for ns := 0; ns < 3; ns++ {
for key := 1; key < 3; key++ {
if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value())
}
}
}
}
func TestLRUCache_Delete(t *testing.T) {
delFuncCalled := 0
delFunc := func() {
delFuncCalled++
}
c := NewCache(NewLRU(2))
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 1, nil).Release()
if ok := c.Delete(0, 1, delFunc); !ok {
t.Error("Cache.Delete on #1 return false")
}
if h := c.Get(0, 1, nil); h != nil {
t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value())
}
if ok := c.Delete(0, 1, delFunc); ok {
t.Error("Cache.Delete on #1 return true")
}
h2 := c.Get(0, 2, nil)
if h2 == nil {
t.Error("Cache.Get on #2 return nil")
}
if ok := c.Delete(0, 2, delFunc); !ok {
t.Error("(1) Cache.Delete on #2 return false")
}
if ok := c.Delete(0, 2, delFunc); !ok {
t.Error("(2) Cache.Delete on #2 return false")
}
set(c, 0, 3, 3, 1, nil).Release()
set(c, 0, 4, 4, 1, nil).Release()
c.Get(0, 2, nil).Release()
for key := 2; key <= 4; key++ {
if h := c.Get(0, uint64(key), nil); h != nil {
h.Release()
} else {
t.Errorf("Cache.Get on #%d return nil", key)
}
}
h2.Release()
if h := c.Get(0, 2, nil); h != nil {
t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value())
}
if delFuncCalled != 4 {
t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled)
}
}
func TestLRUCache_Close(t *testing.T) {
relFuncCalled := 0
relFunc := func() {
relFuncCalled++
}
delFuncCalled := 0
delFunc := func() {
delFuncCalled++
}
c := NewCache(NewLRU(2))
set(c, 0, 1, 1, 1, relFunc).Release()
set(c, 0, 2, 2, 1, relFunc).Release()
h3 := set(c, 0, 3, 3, 1, relFunc)
if h3 == nil {
t.Error("Cache.Get on #3 return nil")
}
if ok := c.Delete(0, 3, delFunc); !ok {
t.Error("Cache.Delete on #3 return false")
}
c.Close()
if relFuncCalled != 3 {
t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled)
}
if delFuncCalled != 1 {
t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled)
}
}
-195
View File
@@ -1,195 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cache
import (
"sync"
"unsafe"
)
type lruNode struct {
n *Node
h *Handle
ban bool
next, prev *lruNode
}
func (n *lruNode) insert(at *lruNode) {
x := at.next
at.next = n
n.prev = at
n.next = x
x.prev = n
}
func (n *lruNode) remove() {
if n.prev != nil {
n.prev.next = n.next
n.next.prev = n.prev
n.prev = nil
n.next = nil
} else {
panic("BUG: removing removed node")
}
}
type lru struct {
mu sync.Mutex
capacity int
used int
recent lruNode
}
func (r *lru) reset() {
r.recent.next = &r.recent
r.recent.prev = &r.recent
r.used = 0
}
func (r *lru) Capacity() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.capacity
}
func (r *lru) SetCapacity(capacity int) {
var evicted []*lruNode
r.mu.Lock()
r.capacity = capacity
for r.used > r.capacity {
rn := r.recent.prev
if rn == nil {
panic("BUG: invalid LRU used or capacity counter")
}
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) Promote(n *Node) {
var evicted []*lruNode
r.mu.Lock()
if n.CacheData == nil {
if n.Size() <= r.capacity {
rn := &lruNode{n: n, h: n.GetHandle()}
rn.insert(&r.recent)
n.CacheData = unsafe.Pointer(rn)
r.used += n.Size()
for r.used > r.capacity {
rn := r.recent.prev
if rn == nil {
panic("BUG: invalid LRU used or capacity counter")
}
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
}
} else {
rn := (*lruNode)(n.CacheData)
if !rn.ban {
rn.remove()
rn.insert(&r.recent)
}
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) Ban(n *Node) {
r.mu.Lock()
if n.CacheData == nil {
n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
} else {
rn := (*lruNode)(n.CacheData)
if !rn.ban {
rn.remove()
rn.ban = true
r.used -= rn.n.Size()
r.mu.Unlock()
rn.h.Release()
rn.h = nil
return
}
}
r.mu.Unlock()
}
func (r *lru) Evict(n *Node) {
r.mu.Lock()
rn := (*lruNode)(n.CacheData)
if rn == nil || rn.ban {
r.mu.Unlock()
return
}
n.CacheData = nil
r.mu.Unlock()
rn.h.Release()
}
func (r *lru) EvictNS(ns uint64) {
var evicted []*lruNode
r.mu.Lock()
for e := r.recent.prev; e != &r.recent; {
rn := e
e = e.prev
if rn.n.NS() == ns {
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) EvictAll() {
r.mu.Lock()
back := r.recent.prev
for rn := back; rn != &r.recent; rn = rn.prev {
rn.n.CacheData = nil
}
r.reset()
r.mu.Unlock()
for rn := back; rn != &r.recent; rn = rn.prev {
rn.h.Release()
}
}
func (r *lru) Close() error {
return nil
}
// NewLRU create a new LRU-cache.
func NewLRU(capacity int) Cacher {
r := &lru{capacity: capacity}
r.reset()
return r
}
-75
View File
@@ -1,75 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package leveldb
import "github.com/syndtr/goleveldb/leveldb/comparer"
type iComparer struct {
ucmp comparer.Comparer
}
func (icmp *iComparer) uName() string {
return icmp.ucmp.Name()
}
func (icmp *iComparer) uCompare(a, b []byte) int {
return icmp.ucmp.Compare(a, b)
}
func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte {
return icmp.ucmp.Separator(dst, a, b)
}
func (icmp *iComparer) uSuccessor(dst, b []byte) []byte {
return icmp.ucmp.Successor(dst, b)
}
func (icmp *iComparer) Name() string {
return icmp.uName()
}
func (icmp *iComparer) Compare(a, b []byte) int {
x := icmp.ucmp.Compare(iKey(a).ukey(), iKey(b).ukey())
if x == 0 {
if m, n := iKey(a).num(), iKey(b).num(); m > n {
x = -1
} else if m < n {
x = 1
}
}
return x
}
func (icmp *iComparer) Separator(dst, a, b []byte) []byte {
ua, ub := iKey(a).ukey(), iKey(b).ukey()
dst = icmp.ucmp.Separator(dst, ua, ub)
if dst == nil {
return nil
}
if len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 {
dst = append(dst, kMaxNumBytes...)
} else {
// Did not close possibilities that n maybe longer than len(ub).
dst = append(dst, a[len(a)-8:]...)
}
return dst
}
func (icmp *iComparer) Successor(dst, b []byte) []byte {
ub := iKey(b).ukey()
dst = icmp.ucmp.Successor(dst, ub)
if dst == nil {
return nil
}
if len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 {
dst = append(dst, kMaxNumBytes...)
} else {
// Did not close possibilities that n maybe longer than len(ub).
dst = append(dst, b[len(b)-8:]...)
}
return dst
}
@@ -1,51 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package comparer
import "bytes"
type bytesComparer struct{}
func (bytesComparer) Compare(a, b []byte) int {
return bytes.Compare(a, b)
}
func (bytesComparer) Name() string {
return "leveldb.BytewiseComparator"
}
func (bytesComparer) Separator(dst, a, b []byte) []byte {
i, n := 0, len(a)
if n > len(b) {
n = len(b)
}
for ; i < n && a[i] == b[i]; i++ {
}
if i >= n {
// Do not shorten if one string is a prefix of the other
} else if c := a[i]; c < 0xff && c+1 < b[i] {
dst = append(dst, a[:i+1]...)
dst[i]++
return dst
}
return nil
}
func (bytesComparer) Successor(dst, b []byte) []byte {
for i, c := range b {
if c != 0xff {
dst = append(dst, b[:i+1]...)
dst[i]++
return dst
}
}
return nil
}
// DefaultComparer are default implementation of the Comparer interface.
// It uses the natural ordering, consistent with bytes.Compare.
var DefaultComparer = bytesComparer{}
@@ -1,57 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Package comparer provides interface and implementation for ordering
// sets of data.
package comparer
// BasicComparer is the interface that wraps the basic Compare method.
type BasicComparer interface {
// Compare returns -1, 0, or +1 depending on whether a is 'less than',
// 'equal to' or 'greater than' b. The two arguments can only be 'equal'
// if their contents are exactly equal. Furthermore, the empty slice
// must be 'less than' any non-empty slice.
Compare(a, b []byte) int
}
// Comparer defines a total ordering over the space of []byte keys: a 'less
// than' relationship.
type Comparer interface {
BasicComparer
// Name returns name of the comparer.
//
// The Level-DB on-disk format stores the comparer name, and opening a
// database with a different comparer from the one it was created with
// will result in an error.
//
// An implementation to a new name whenever the comparer implementation
// changes in a way that will cause the relative ordering of any two keys
// to change.
//
// Names starting with "leveldb." are reserved and should not be used
// by any users of this package.
Name() string
// Bellow are advanced functions used used to reduce the space requirements
// for internal data structures such as index blocks.
// Separator appends a sequence of bytes x to dst such that a <= x && x < b,
// where 'less than' is consistent with Compare. An implementation should
// return nil if x equal to a.
//
// Either contents of a or b should not by any means modified. Doing so
// may cause corruption on the internal state.
Separator(dst, a, b []byte) []byte
// Successor appends a sequence of bytes x to dst such that x >= b, where
// 'less than' is consistent with Compare. An implementation should return
// nil if x equal to b.
//
// Contents of b should not by any means modified. Doing so may cause
// corruption on the internal state.
Successor(dst, b []byte) []byte
}
@@ -1,500 +0,0 @@
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package leveldb
import (
"bytes"
"fmt"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
"io"
"math/rand"
"testing"
)
const ctValSize = 1000
type dbCorruptHarness struct {
dbHarness
}
func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
h := new(dbCorruptHarness)
h.init(t, o)
return h
}
func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
return newDbCorruptHarnessWopt(t, &opt.Options{
BlockCacheCapacity: 100,
Strict: opt.StrictJournalChecksum,
})
}
func (h *dbCorruptHarness) recover() {
p := &h.dbHarness
t := p.t
var err error
p.db, err = Recover(h.stor, h.o)
if err != nil {
t.Fatal("Repair: got error: ", err)
}
}
func (h *dbCorruptHarness) build(n int) {
p := &h.dbHarness
t := p.t
db := p.db
batch := new(Batch)
for i := 0; i < n; i++ {
batch.Reset()
batch.Put(tkey(i), tval(i, ctValSize))
err := db.Write(batch, p.wo)
if err != nil {
t.Fatal("write error: ", err)
}
}
}
func (h *dbCorruptHarness) buildShuffled(n int, rnd *rand.Rand) {
p := &h.dbHarness
t := p.t
db := p.db
batch := new(Batch)
for i := range rnd.Perm(n) {
batch.Reset()
batch.Put(tkey(i), tval(i, ctValSize))
err := db.Write(batch, p.wo)
if err != nil {
t.Fatal("write error: ", err)
}
}
}
func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
p := &h.dbHarness
t := p.t
db := p.db
batch := new(Batch)
for i := 0; i < n; i++ {
batch.Reset()
batch.Delete(tkey(rnd.Intn(max)))
err := db.Write(batch, p.wo)
if err != nil {
t.Fatal("write error: ", err)
}
}
}
func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) {
p := &h.dbHarness
t := p.t
ff, _ := p.stor.GetFiles(ft)
sff := files(ff)
sff.sort()
if fi < 0 {
fi = len(sff) - 1
}
if fi >= len(sff) {
t.Fatalf("no such file with type %q with index %d", ft, fi)
}
file := sff[fi]
r, err := file.Open()
if err != nil {
t.Fatal("cannot open file: ", err)
}
x, err := r.Seek(0, 2)
if err != nil {
t.Fatal("cannot query file size: ", err)
}
m := int(x)
if _, err := r.Seek(0, 0); err != nil {
t.Fatal(err)
}
if offset < 0 {
if -offset > m {
offset = 0
} else {
offset = m + offset
}
}
if offset > m {
offset = m
}
if offset+n > m {
n = m - offset
}
buf := make([]byte, m)
_, err = io.ReadFull(r, buf)
if err != nil {
t.Fatal("cannot read file: ", err)
}
r.Close()
for i := 0; i < n; i++ {
buf[offset+i] ^= 0x80
}
err = file.Remove()
if err != nil {
t.Fatal("cannot remove old file: ", err)
}
w, err := file.Create()
if err != nil {
t.Fatal("cannot create new file: ", err)
}
_, err = w.Write(buf)
if err != nil {
t.Fatal("cannot write new file: ", err)
}
w.Close()
}
func (h *dbCorruptHarness) removeAll(ft storage.FileType) {
ff, err := h.stor.GetFiles(ft)
if err != nil {
h.t.Fatal("get files: ", err)
}
for _, f := range ff {
if err := f.Remove(); err != nil {
h.t.Error("remove file: ", err)
}
}
}
func (h *dbCorruptHarness) removeOne(ft storage.FileType) {
ff, err := h.stor.GetFiles(ft)
if err != nil {
h.t.Fatal("get files: ", err)
}
f := ff[rand.Intn(len(ff))]
h.t.Logf("removing file @%d", f.Num())
if err := f.Remove(); err != nil {
h.t.Error("remove file: ", err)
}
}
func (h *dbCorruptHarness) check(min, max int) {
p := &h.dbHarness
t := p.t
db := p.db
var n, badk, badv, missed, good int
iter := db.NewIterator(nil, p.ro)
for iter.Next() {
k := 0
fmt.Sscanf(string(iter.Key()), "%d", &k)
if k < n {
badk++
continue
}
missed += k - n
n = k + 1
if !bytes.Equal(iter.Value(), tval(k, ctValSize)) {
badv++
} else {
good++
}
}
err := iter.Error()
iter.Release()
t.Logf("want=%d..%d got=%d badkeys=%d badvalues=%d missed=%d, err=%v",
min, max, good, badk, badv, missed, err)
if good < min || good > max {
t.Errorf("good entries number not in range")
}
}
func TestCorruptDB_Journal(t *testing.T) {
h := newDbCorruptHarness(t)
h.build(100)
h.check(100, 100)
h.closeDB()
h.corrupt(storage.TypeJournal, -1, 19, 1)
h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1)
h.openDB()
h.check(36, 36)
h.close()
}
func TestCorruptDB_Table(t *testing.T) {
h := newDbCorruptHarness(t)
h.build(100)
h.compactMem()
h.compactRangeAt(0, "", "")
h.compactRangeAt(1, "", "")
h.closeDB()
h.corrupt(storage.TypeTable, -1, 100, 1)
h.openDB()
h.check(99, 99)
h.close()
}
func TestCorruptDB_TableIndex(t *testing.T) {
h := newDbCorruptHarness(t)
h.build(10000)
h.compactMem()
h.closeDB()
h.corrupt(storage.TypeTable, -1, -2000, 500)
h.openDB()
h.check(5000, 9999)
h.close()
}
func TestCorruptDB_MissingManifest(t *testing.T) {
rnd := rand.New(rand.NewSource(0x0badda7a))
h := newDbCorruptHarnessWopt(t, &opt.Options{
BlockCacheCapacity: 100,
Strict: opt.StrictJournalChecksum,
WriteBuffer: 1000 * 60,
})
h.build(1000)
h.compactMem()
h.buildShuffled(1000, rnd)
h.compactMem()
h.deleteRand(500, 1000, rnd)
h.compactMem()
h.buildShuffled(1000, rnd)
h.compactMem()
h.deleteRand(500, 1000, rnd)
h.compactMem()
h.buildShuffled(1000, rnd)
h.compactMem()
h.closeDB()
h.stor.SetIgnoreOpenErr(storage.TypeManifest)
h.removeAll(storage.TypeManifest)
h.openAssert(false)
h.stor.SetIgnoreOpenErr(0)
h.recover()
h.check(1000, 1000)
h.build(1000)
h.compactMem()
h.compactRange("", "")
h.closeDB()
h.recover()
h.check(1000, 1000)
h.close()
}
func TestCorruptDB_SequenceNumberRecovery(t *testing.T) {
h := newDbCorruptHarness(t)
h.put("foo", "v1")
h.put("foo", "v2")
h.put("foo", "v3")
h.put("foo", "v4")
h.put("foo", "v5")
h.closeDB()
h.recover()
h.getVal("foo", "v5")
h.put("foo", "v6")
h.getVal("foo", "v6")
h.reopenDB()
h.getVal("foo", "v6")
h.close()
}
func TestCorruptDB_SequenceNumberRecoveryTable(t *testing.T) {
h := newDbCorruptHarness(t)
h.put("foo", "v1")
h.put("foo", "v2")
h.put("foo", "v3")
h.compactMem()
h.put("foo", "v4")
h.put("foo", "v5")
h.compactMem()
h.closeDB()
h.recover()
h.getVal("foo", "v5")
h.put("foo", "v6")
h.getVal("foo", "v6")
h.reopenDB()
h.getVal("foo", "v6")
h.close()
}
func TestCorruptDB_CorruptedManifest(t *testing.T) {
h := newDbCorruptHarness(t)
h.put("foo", "hello")
h.compactMem()
h.compactRange("", "")
h.closeDB()
h.corrupt(storage.TypeManifest, -1, 0, 1000)
h.openAssert(false)
h.recover()
h.getVal("foo", "hello")
h.close()
}
func TestCorruptDB_CompactionInputError(t *testing.T) {
h := newDbCorruptHarness(t)
h.build(10)
h.compactMem()
h.closeDB()
h.corrupt(storage.TypeTable, -1, 100, 1)
h.openDB()
h.check(9, 9)
h.build(10000)
h.check(10000, 10000)
h.close()
}
func TestCorruptDB_UnrelatedKeys(t *testing.T) {
h := newDbCorruptHarness(t)
h.build(10)
h.compactMem()
h.closeDB()
h.corrupt(storage.TypeTable, -1, 100, 1)
h.openDB()
h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
h.compactMem()
h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
h.close()
}
func TestCorruptDB_Level0NewerFileHasOlderSeqnum(t *testing.T) {
h := newDbCorruptHarness(t)
h.put("a", "v1")
h.put("b", "v1")
h.compactMem()
h.put("a", "v2")
h.put("b", "v2")
h.compactMem()
h.put("a", "v3")
h.put("b", "v3")
h.compactMem()
h.put("c", "v0")
h.put("d", "v0")
h.compactMem()
h.compactRangeAt(1, "", "")
h.closeDB()
h.recover()
h.getVal("a", "v3")
h.getVal("b", "v3")
h.getVal("c", "v0")
h.getVal("d", "v0")
h.close()
}
func TestCorruptDB_RecoverInvalidSeq_Issue53(t *testing.T) {
h := newDbCorruptHarness(t)
h.put("a", "v1")
h.put("b", "v1")
h.compactMem()
h.put("a", "v2")
h.put("b", "v2")
h.compactMem()
h.put("a", "v3")
h.put("b", "v3")
h.compactMem()
h.put("c", "v0")
h.put("d", "v0")
h.compactMem()
h.compactRangeAt(0, "", "")
h.closeDB()
h.recover()
h.getVal("a", "v3")
h.getVal("b", "v3")
h.getVal("c", "v0")
h.getVal("d", "v0")
h.close()
}
func TestCorruptDB_MissingTableFiles(t *testing.T) {
h := newDbCorruptHarness(t)
h.put("a", "v1")
h.put("b", "v1")
h.compactMem()
h.put("c", "v2")
h.put("d", "v2")
h.compactMem()
h.put("e", "v3")
h.put("f", "v3")
h.closeDB()
h.removeOne(storage.TypeTable)
h.openAssert(false)
h.close()
}
func TestCorruptDB_RecoverTable(t *testing.T) {
h := newDbCorruptHarnessWopt(t, &opt.Options{
WriteBuffer: 112 * opt.KiB,
CompactionTableSize: 90 * opt.KiB,
Filter: filter.NewBloomFilter(10),
})
h.build(1000)
h.compactMem()
h.compactRangeAt(0, "", "")
h.compactRangeAt(1, "", "")
seq := h.db.seq
h.closeDB()
h.corrupt(storage.TypeTable, 0, 1000, 1)
h.corrupt(storage.TypeTable, 3, 10000, 1)
// Corrupted filter shouldn't affect recovery.
h.corrupt(storage.TypeTable, 3, 113888, 10)
h.corrupt(storage.TypeTable, -1, 20000, 1)
h.recover()
if h.db.seq != seq {
t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq)
}
h.check(985, 985)
h.close()
}

Some files were not shown because too many files have changed in this diff Show More