Compare commits
3446 Commits
v0.9.0-beta2
...
v0.14.19
| Author | SHA1 | Date | |
|---|---|---|---|
| c953cdc375 | |||
| c20c17e3c6 | |||
| 1a1e35d998 | |||
| 8d2a31e38e | |||
| fe377a166a | |||
| 5770d80bf9 | |||
| 7a16dbd31d | |||
| 926c88cfc4 | |||
| 29d010ec0e | |||
| 920274bce4 | |||
| 2ebd6ad77f | |||
| 79dd6918f2 | |||
| 79f7f50c4d | |||
| 987718baf8 | |||
| 4fb9c143ac | |||
| ec62888539 | |||
| 8c34a76f7a | |||
| 6809d38cde | |||
| 69ae4aa024 | |||
| 8e8b867fba | |||
| 0a118d2979 | |||
| 8daaa5d0d2 | |||
| eb14f85a57 | |||
| c69c3c7c36 | |||
| 54911d44c5 | |||
| c765f7be8d | |||
| 44bdaf3ac2 | |||
| fc16e49cb0 | |||
| f5a310ad64 | |||
| 01e50eb3fa | |||
| 722b81c6f0 | |||
| f0efa2b974 | |||
| bab7c8ebbf | |||
| 0725e3af38 | |||
| dd7bb6c4b8 | |||
| d41c131364 | |||
| 47f22ff3e5 | |||
| 744c2e82b5 | |||
| ead7281c20 | |||
| aa3ef49dd7 | |||
| 5c067661f4 | |||
| 226da976dc | |||
| ba17cc0a11 | |||
| 9e0afb7d8a | |||
| 9e7d50bc76 | |||
| d7d5687faa | |||
| 21eb098dd2 | |||
| 2f770f8bfb | |||
| f09698d845 | |||
| 3fbcb024a7 | |||
| b8c1c0e048 | |||
| 2d47242d54 | |||
| 66a7829eee | |||
| 9c67bd2550 | |||
| f67c5a2fd6 | |||
| 0437f6dd66 | |||
| 11b35d650d | |||
| b279e261a1 | |||
| 263402f80a | |||
| 920a83ec7a | |||
| 3c2ac3522c | |||
| 9fdaa637a8 | |||
| 81a9d7f2b9 | |||
| d8d3f05164 | |||
| 653be136ee | |||
| 398c356f22 | |||
| 542b76f687 | |||
| abb8a1914a | |||
| 163d335078 | |||
| 0582836820 | |||
| bb15776ae6 | |||
| dde9d4c9eb | |||
| 1ef75be1c6 | |||
| 3582783972 | |||
| 5070d52f2f | |||
| 7b07ed6580 | |||
| f6a2b6252a | |||
| a9b03de99a | |||
| a7f7058636 | |||
| 8ce9b026e9 | |||
| 0dcf2f1bc8 | |||
| 99922feb3b | |||
| 2fd1dca905 | |||
| e3cf718998 | |||
| 7157917a16 | |||
| 3266aae1c3 | |||
| 63194a37f6 | |||
| cabe94552a | |||
| 48a229a0cd | |||
| e4db86836b | |||
| 913a85c571 | |||
| ed4f6fc4b3 | |||
| 9da422f1c5 | |||
| ab1739ba34 | |||
| fc1430aa92 | |||
| 3cde608eda | |||
| 2898552f4b | |||
| 911c148c71 | |||
| 91568a173a | |||
| e57f5499a1 | |||
| 9abb7b71a9 | |||
| 724c354d62 | |||
| c44779094d | |||
| eeedab4091 | |||
| 8559e20237 | |||
| 26730eb083 | |||
| 4160ce674d | |||
| 2dbeea21c4 | |||
| a2b8485a89 | |||
| 5bb74ee61c | |||
| 462fde5e7d | |||
| f1e83a57cd | |||
| cc9a9fb390 | |||
| 8fbcceb742 | |||
| d3a251e6d9 | |||
| be80b26c18 | |||
| 1574b7d834 | |||
| 51e10e344d | |||
| 0d55d8c5b0 | |||
| 1392589d36 | |||
| 548a324256 | |||
| a8a0bc356a | |||
| faee1d5a8d | |||
| 3088dac33b | |||
| 2641062c17 | |||
| 95c738ea28 | |||
| 562d2f67a6 | |||
| ba6aff4a1b | |||
| bb23e3940e | |||
| 94e4370c7e | |||
| 38d28c3f4a | |||
| f60b424d70 | |||
| a1a91d5ef4 | |||
| bfb48b5dde | |||
| 2860813a8e | |||
| 72538e350d | |||
| 59f3d1445f | |||
| 0b88cf1d03 | |||
| 56e2ba29d0 | |||
| 6ec9b84674 | |||
| afd15392b1 | |||
| ae4cc94a9d | |||
| 3f9b75b7b3 | |||
| ec2b097313 | |||
| caaab462bc | |||
| da413b823b | |||
| 14937e7dd2 | |||
| 3418497f3d | |||
| e408f1061a | |||
| c08fe4e2c5 | |||
| b7e21984a1 | |||
| 7fba8cf759 | |||
| 0296c23685 | |||
| 1cdfef4d6a | |||
| cead20ec91 | |||
| 74dd051d51 | |||
| 5473285010 | |||
| 4b3adfa21c | |||
| 7c37301c91 | |||
| d9040f8038 | |||
| f41606c0b0 | |||
| 31d9750579 | |||
| 173fb97832 | |||
| 81248c3f56 | |||
| 2914a0a0a5 | |||
| 815588daba | |||
| 6152eb6d6d | |||
| ff0ebc196c | |||
| 4e8c8d7e2c | |||
| b8a90b7eaa | |||
| 60e7ca4a4c | |||
| 3a3c8ec6b8 | |||
| 05c37e58c1 | |||
| d203dd4770 | |||
| 29ccf10d0b | |||
| b49df09fec | |||
| ce3e117976 | |||
| 309795198d | |||
| 7db00132b2 | |||
| 76a2862b7a | |||
| 215503b4f7 | |||
| 54d4010f1a | |||
| cb1b53cfb5 | |||
| 96e8f94833 | |||
| 1e54a3e801 | |||
| fe9c2b9857 | |||
| 2a2177e7fa | |||
| d1d565e58b | |||
| 891ff383ec | |||
| d322ebd0b9 | |||
| 50190236bb | |||
| d5a0f91cb4 | |||
| 467c1b26fb | |||
| 3cabecda04 | |||
| 6d3160b0ab | |||
| d328e0fb75 | |||
| 5f01afb7ea | |||
| 6fe2fa5ff0 | |||
| b371b1fe34 | |||
| 90c0a39df8 | |||
| 70c5a5dff1 | |||
| da0b7cc7f2 | |||
| 139e9b144e | |||
| 77c0a19451 | |||
| 58cbd19742 | |||
| 9bf6917ae8 | |||
| 897cca0a82 | |||
| 6af09c61be | |||
| c3c7798446 | |||
| 06dc91fadf | |||
| 526cab538a | |||
| 81d19a00aa | |||
| ca755ec9e0 | |||
| 4f6206cb2d | |||
| 7fb53ec954 | |||
| d8b5070ca8 | |||
| 5e99d38412 | |||
| 3990014073 | |||
| 3e51206a6b | |||
| 7569b75d61 | |||
| 8fcabac518 | |||
| abb0cfde72 | |||
| 7990ffcc60 | |||
| 49910a1d85 | |||
| 46a143e80e | |||
| 69b7f26e4c | |||
| 5b37d0356c | |||
| 1188ebbb7b | |||
| 76b903b2e0 | |||
| be38c2111f | |||
| 1de787fab8 | |||
| 81f683a61c | |||
| db6f68d031 | |||
| d0a1c805e9 | |||
| 00a654845f | |||
| 04dad8485a | |||
| 0b1475169f | |||
| 6ec4fbc82b | |||
| 18cc7a663b | |||
| cf5febad47 | |||
| 42849af5a8 | |||
| e6364407a9 | |||
| 480b78f2c8 | |||
| fa8f339478 | |||
| 7776839c82 | |||
| 7114cacb85 | |||
| e52be3d83e | |||
| c9cf01e0b6 | |||
| dcbf68e104 | |||
| c2d8c07137 | |||
| a4ed50ca85 | |||
| b3788c8ea0 | |||
| 946c074a41 | |||
| 19f79afb0f | |||
| af3b6f9c83 | |||
| fbe42c156d | |||
| a1f6cbd354 | |||
| a4f052ad31 | |||
| ea87bcefd6 | |||
| 0655991a19 | |||
| f368d2278f | |||
| 1eb6db6ca8 | |||
| a25b63e2df | |||
| ffe7a2fcd7 | |||
| 08b5a7908f | |||
| a8cd9d0154 | |||
| 297240facf | |||
| a022b0cfff | |||
| 72026db599 | |||
| aafc96f58f | |||
| 7c7e8648ff | |||
| 24e2ce0764 | |||
| d7cb4d407b | |||
| 66a506e72b | |||
| 25a7b0a6f8 | |||
| 7aaa1dd8a3 | |||
| 2a6f164923 | |||
| 0f28626bb4 | |||
| 6ed22d0885 | |||
| 6715b91a6c | |||
| 694da60659 | |||
| 47fa4b0a2c | |||
| 8ab6b60778 | |||
| e1a4f81e50 | |||
| 7b7e35d339 | |||
| 3176629410 | |||
| e3ccc45d19 | |||
| beec9e834e | |||
| f6f0486ff9 | |||
| 518f446d31 | |||
| fbbd510088 | |||
| e440d30028 | |||
| 44d30c83bf | |||
| 7ff7b55732 | |||
| 44346b3a5a | |||
| 23a538d61a | |||
| dcb5026f33 | |||
| 778ff9daa9 | |||
| ce9dc809bc | |||
| 59370588dd | |||
| 7d434aa9c4 | |||
| 59ce7c0424 | |||
| 9a0e5a7c18 | |||
| 8d0019595f | |||
| 6ff74cfcab | |||
| aa50ef4069 | |||
| fa0101bd60 | |||
| 21f5b16e47 | |||
| 223a835f33 | |||
| 223e14b0d0 | |||
| a58f69be04 | |||
| e194eb1f69 | |||
| 672824641b | |||
| 6d357211b2 | |||
| 8e39e2889d | |||
| a9ee4bb9f1 | |||
| 80fd6c2400 | |||
| 3cbe7d40d1 | |||
| af0bc95de5 | |||
| 4bf3e7485b | |||
| b701de60ce | |||
| 7ef2743964 | |||
| a165838cbd | |||
| 3c77b8388c | |||
| 9d16f4545d | |||
| d57e6808cc | |||
| b71cc8a580 | |||
| ac3b03881a | |||
| b0d03d1f1c | |||
| a2dcffcca2 | |||
| 9323f0faf8 | |||
| f343c8ba36 | |||
| 502bee9a09 | |||
| 379e2119a8 | |||
| 89a29946f9 | |||
| 20a94fafa7 | |||
| 99ddf1e4ab | |||
| fb778218f5 | |||
| 55fc3cb2c5 | |||
| b779e22205 | |||
| bb5b1f8f01 | |||
| c1a96d4900 | |||
| de298da532 | |||
| 6f5ca53f99 | |||
| d507126101 | |||
| 9a25df01fe | |||
| 11b9212948 | |||
| b4e2914b70 | |||
| 09b7348595 | |||
| d2bb6e0c0a | |||
| 8632a03662 | |||
| e71c78ae84 | |||
| 03a8027efc | |||
| b7e186b370 | |||
| 4a69f3987f | |||
| 343dc486e0 | |||
| 5aacfd1639 | |||
| 06e63aedea | |||
| 0320194757 | |||
| 1753771356 | |||
| bc794e7c15 | |||
| eefcecc7ce | |||
| 3795a786c9 | |||
| 855a1bef89 | |||
| 6a67921e40 | |||
| 8709fec517 | |||
| 48245effdf | |||
| 16063933d1 | |||
| d317f197be | |||
| 8ac862f50a | |||
| 0e996c4664 | |||
| 287cfee73c | |||
| a6c465e929 | |||
| becb5ab1dc | |||
| 49170bf2d8 | |||
| b1205db7ac | |||
| ff0cd413e6 | |||
| 7a56e4a0e5 | |||
| d17608d0a0 | |||
| 0af216fea0 | |||
| 1287433a99 | |||
| 56a9964101 | |||
| 532b4383bf | |||
| f9e2623fdc | |||
| eacae83886 | |||
| 5fc53f59c7 | |||
| 7035ea3ab7 | |||
| d67c0a1eda | |||
| 36c6a1955f | |||
| f792989d9b | |||
| ee398f17e1 | |||
| 8c4723ff43 | |||
| 01ae866d58 | |||
| 3b8ae33fe3 | |||
| 6f63909c65 | |||
| 1612baca92 | |||
| 4970bd7f65 | |||
| a775dd2b79 | |||
| 137894348b | |||
| ac40b27c79 | |||
| 9d756525ce | |||
| 6361172bea | |||
| 56b6383407 | |||
| 46fa5a374b | |||
| 7373d2eb3c | |||
| 4453236949 | |||
| c2dc4a8e06 | |||
| 92a23da3ec | |||
| 242db26343 | |||
| 87701339fe | |||
| 4669ce0766 | |||
| 9bb5988b4e | |||
| c513171014 | |||
| da5010d37a | |||
| e6b78e5d56 | |||
| 410d700ae3 | |||
| fc173bf679 | |||
| 72154aa668 | |||
| 31b5156191 | |||
| ebce5d07ac | |||
| 915e1ac7de | |||
| b78bfc0a43 | |||
| 30436741a7 | |||
| 98734375f2 | |||
| 37816e3818 | |||
| 4bc2b3f369 | |||
| 00be2bf18d | |||
| 44290a66b7 | |||
| f6cc344623 | |||
| a89d487510 | |||
| a0ec4467fd | |||
| 7dddc0de9e | |||
| e7280f1eb5 | |||
| bf7fcc612d | |||
| cff9bbc9c5 | |||
| fddca3d2d6 | |||
| 9db49fb45e | |||
| 891409aedf | |||
| 77e47066ed | |||
| 852759f904 | |||
| 1dbc310c9b | |||
| 86ca58e2a9 | |||
| 22280db5db | |||
| 8e060e23e3 | |||
| 6e07742fe9 | |||
| 04d5032055 | |||
| 73ae87fad1 | |||
| cd05282369 | |||
| ee94d53bda | |||
| 922e1407c2 | |||
| 2ea22b1850 | |||
| 2c1323ece6 | |||
| adb7fb43cb | |||
| d59fd9c22d | |||
| 6f743f3138 | |||
| 5a7fad0bcd | |||
| 5d2414dfa9 | |||
| bef2425025 | |||
| e8b4286c93 | |||
| 2e9bf0b67c | |||
| 935c273c8f | |||
| b993b41847 | |||
| 1be40cc4fa | |||
| d628b731d1 | |||
| 21e116aa45 | |||
| d77d8ff803 | |||
| 31f64186ae | |||
| 1a703efa78 | |||
| 8b7b0a03eb | |||
| 0761d804a4 | |||
| 3ad42d9279 | |||
| bd41e21c26 | |||
| 10fe23b8f2 | |||
| 39899e40bf | |||
| 5d337bb24f | |||
| dd5909568f | |||
| 38166e976f | |||
| d6a7ffe0d4 | |||
| 2ebc6996a2 | |||
| 2e840134d2 | |||
| 66e1be33cf | |||
| 591959261c | |||
| 459930df09 | |||
| 674fc566bb | |||
| 09832abe50 | |||
| eabd2fc936 | |||
| 6720906ee5 | |||
| abb96802cb | |||
| 29fa05ae05 | |||
| 7b43ba809b | |||
| 49387f9494 | |||
| 953482de53 | |||
| 8cf3a7aeda | |||
| 175f65aabc | |||
| b8c5cf1142 | |||
| 236f121c4e | |||
| 94a392144b | |||
| 2467678bd4 | |||
| e87c1abd4e | |||
| dffc34559b | |||
| 80f2a9a6bf | |||
| 4aa6ecb122 | |||
| ccfcdf7f48 | |||
| 4eb23a38b1 | |||
| cb38213444 | |||
| 842b6111db | |||
| ea54525a33 | |||
| 893cc025f9 | |||
| b81c8d2e1b | |||
| 4b07535e86 | |||
| 0d2fe320a7 | |||
| f294113d01 | |||
| e9063c639a | |||
| 8d6dedc15b | |||
| 1bc4c1a8ac | |||
| 1c7af1a72e | |||
| e61f424ade | |||
| 6d3aae32bc | |||
| fa1cfd94d0 | |||
| 0155b6f841 | |||
| f6953624dd | |||
| 1a5f524ae4 | |||
| a4cd4cc253 | |||
| 1a35c440e8 | |||
| 2c6c84ac61 | |||
| bd666daf82 | |||
| ca3831c4f5 | |||
| bbe0d34f43 | |||
| dd364c962f | |||
| 50068b0b0f | |||
| c49453c519 | |||
| 52c7804f32 | |||
| 19b4f3bfb4 | |||
| f3ac421266 | |||
| 7533a61203 | |||
| 6355a7019b | |||
| 490464e170 | |||
| 467d338fe4 | |||
| 6130578d18 | |||
| 4389bb037d | |||
| 2eb8a9ef56 | |||
| 393798098c | |||
| 668eb7c398 | |||
| 0937f85534 | |||
| cf64376dca | |||
| 5a98af622d | |||
| 4f5d0b46f7 | |||
| 492e92d65d | |||
| 181939c841 | |||
| b678b4e048 | |||
| 1934b3a5b6 | |||
| cc1d122352 | |||
| a4f0b85462 | |||
| 7b4e1e9055 | |||
| 4c3cd4c9e3 | |||
| 46e913dc23 | |||
| 8f580b13df | |||
| a551686d37 | |||
| 432c78079b | |||
| f5f0e46016 | |||
| b6f32b6e45 | |||
| 66f480519b | |||
| 8044522691 | |||
| c6a67bd203 | |||
| c6881b6d02 | |||
| 4489bec6ef | |||
| 3d71e68696 | |||
| 783d2da4a8 | |||
| 6be4b49999 | |||
| 68185dd93c | |||
| d01ea9d6fb | |||
| d91e6023eb | |||
| 17ed01a0c9 | |||
| 4b6c2d0d3d | |||
| 46c07bb207 | |||
| eaa805b9f0 | |||
| 436fd0b88e | |||
| f706d3c393 | |||
| c58eb1d47a | |||
| b4f9a55e6e | |||
| 1d17891286 | |||
| 6a3f3f5577 | |||
| 29913dd1e4 | |||
| 690837dbe5 | |||
| 82e80a479a | |||
| bc508aee7b | |||
| 95247f7740 | |||
| e5731229c7 | |||
| 52c74ad866 | |||
| a28f890e83 | |||
| 31362dfc17 | |||
| a492cfba13 | |||
| 96afcd90e3 | |||
| ea61f8f597 | |||
| 894ccd18ff | |||
| 9dec6f1324 | |||
| 2e44473ce4 | |||
| 26d6969384 | |||
| 2dbde224d9 | |||
| 8d7ed9f8bf | |||
| aba2cc4db2 | |||
| 2df001fe5c | |||
| a49b8a2608 | |||
| bea272c40b | |||
| a455e32adf | |||
| 9d522bd626 | |||
| 0427396f50 | |||
| c952468e13 | |||
| 94b3ce44e6 | |||
| c439c543d0 | |||
| 78120bd989 | |||
| f66c1c3c9c | |||
| 6f82d83bd6 | |||
| 3e218b146e | |||
| 17517bcc3d | |||
| d8fba47870 | |||
| e9c5261a49 | |||
| 8d53175c20 | |||
| ba5231dc89 | |||
| 032365d57c | |||
| e9aed494f8 | |||
| 16c3d39fd2 | |||
| 1875f7287e | |||
| d619031f68 | |||
| 4ef759dba8 | |||
| 0d16c8eab4 | |||
| de7d176edf | |||
| d37ed65f42 | |||
| 710ddf7906 | |||
| 3abb80885e | |||
| fd962c5e99 | |||
| 07f944bf48 | |||
| 012423338e | |||
| 64cfebc63c | |||
| 28d74f5d9b | |||
| 8418fae82b | |||
| 9b1bebc9b2 | |||
| 8d888bb756 | |||
| 83c29e1945 | |||
| 09ebc33b30 | |||
| ff9bfae722 | |||
| 3b146eda0d | |||
| a8ffde6f21 | |||
| dd9a4e044a | |||
| b8c72ade4c | |||
| 5dd55d3811 | |||
| f00b133eee | |||
| a117b0c723 | |||
| 65aaa607ab | |||
| 9259425a9a | |||
| 6816e2436b | |||
| c8b6e6fd9b | |||
| ac2343ea57 | |||
| a6a9af4f02 | |||
| 35dc173c80 | |||
| a686be8ba4 | |||
| d61b03701c | |||
| 81d9857888 | |||
| cd9e142db3 | |||
| 8682a33ab1 | |||
| 0631e4395a | |||
| d78425eab4 | |||
| 0b03a640fb | |||
| 9d277ac2ac | |||
| 54c1ffe5f3 | |||
| e11302172e | |||
| bf353a42cd | |||
| d8e19b776e | |||
| cf96bb464f | |||
| 3c7164846d | |||
| 4fa4668ed6 | |||
| 0ce21aea08 | |||
| 6f2de31146 | |||
| e1ac740ac4 | |||
| 4feeaf1641 | |||
| a08bbabd4d | |||
| a7a9d7d85c | |||
| e93c766c42 | |||
| 5d4bfdabd6 | |||
| 39c16d1cc4 | |||
| eb55d19786 | |||
| ae36fada6b | |||
| 60ca7784ba | |||
| 7e8db13854 | |||
| 3e7d0ec14f | |||
| 5971c00a4f | |||
| 8ff7531f89 | |||
| f59e1ad854 | |||
| 1a0a8a1655 | |||
| 24023ff9e8 | |||
| 016f799983 | |||
| fae68a5396 | |||
| 79680b1d5e | |||
| 0ce45c20b8 | |||
| fed374fcb6 | |||
| 374202ac45 | |||
| 56db1d3dfa | |||
| d4796261d7 | |||
| 452c5d5e91 | |||
| cc5f93e717 | |||
| 49601a63c8 | |||
| 6c33188af3 | |||
| 1353e3916f | |||
| 11d4986517 | |||
| e267bf3e09 | |||
| 39c5c8c1d1 | |||
| 6cce073da5 | |||
| 99372c69e5 | |||
| 042b703fe4 | |||
| 1880284bde | |||
| 6c1faa4bdb | |||
| 33f97d7d8f | |||
| 82e033942e | |||
| 693e1c93f1 | |||
| 2919b76947 | |||
| 42b94561a2 | |||
| acaf134dfe | |||
| ab9109e0dc | |||
| f88b2c11fe | |||
| d5d330413b | |||
| 61d7e11001 | |||
| c4c6df179b | |||
| b04b7bf357 | |||
| 97b1c66d4a | |||
| 74a210f198 | |||
| 31861052e5 | |||
| 7c42b5cb17 | |||
| df7fea4412 | |||
| 5fa8b42fac | |||
| d3fa67fe2e | |||
| 357089a438 | |||
| 8b3d75b339 | |||
| f741066466 | |||
| 1e45111bde | |||
| 9595687bce | |||
| 7427b9de35 | |||
| acdddc0b79 | |||
| 01c70caa8f | |||
| ff4bab4c07 | |||
| 5c36029274 | |||
| cd54186113 | |||
| 353689857e | |||
| d74377350b | |||
| 6c0a973ac3 | |||
| 3ca46c29c3 | |||
| 837fde70ae | |||
| 1e52cc474f | |||
| 76807006be | |||
| 0d35fe0f21 | |||
| 370b0fc5da | |||
| 25b3c09f6a | |||
| 576c365753 | |||
| bc2ed60b92 | |||
| a4385100c4 | |||
| 4c01709cdf | |||
| 1df924f4f8 | |||
| 400bfe9251 | |||
| 6e1d364d60 | |||
| 504ad86648 | |||
| 096b2d73cd | |||
| 5754d31d0f | |||
| ed3ed1f90a | |||
| adb1227b2e | |||
| ac190b2e39 | |||
| 18ae87962d | |||
| 4673862981 | |||
| 212d7257e9 | |||
| 9df8d0848a | |||
| ded7abb1f6 | |||
| 44d5a61cfe | |||
| 0db80710aa | |||
| 7c47eff112 | |||
| d81849ab3b | |||
| dbb3f80995 | |||
| 80b9a3e00b | |||
| be5e5d837b | |||
| ffc4a60bc6 | |||
| 1edfa4474f | |||
| 18e70f4e79 | |||
| 62a81cfdd1 | |||
| 6365a026c1 | |||
| 7c89193398 | |||
| 4a6f1718b8 | |||
| 7b3fa8da60 | |||
| 2fae7ccf5c | |||
| f36f48c2cf | |||
| 8b726c7e8b | |||
| 8eb0687407 | |||
| 9b9912ba9e | |||
| 22d0ed8225 | |||
| b7a58d2f87 | |||
| 219ece22fc | |||
| 4874301615 | |||
| 1827dda0c6 | |||
| d088b01f75 | |||
| 00c5062eab | |||
| 84eacde63a | |||
| d98290c17f | |||
| 6d94a3be05 | |||
| a2833d18ed | |||
| 6f95afdc59 | |||
| aaa75a32a5 | |||
| c300015ac2 | |||
| eb4f5e9faa | |||
| c7aec839ae | |||
| 37aa89b19d | |||
| 3f94e70488 | |||
| 435afa0eea | |||
| fb82a5e086 | |||
| 71d98c2f26 | |||
| 4a97aa12d6 | |||
| c9e67fb460 | |||
| cec87be4e3 | |||
| eb1a234a77 | |||
| 394c2b67d6 | |||
| aeb3af7105 | |||
| e5cf99e31c | |||
| 8120535a35 | |||
| ba01433381 | |||
| 6b99cdb83a | |||
| e7b71f8743 | |||
| 03935b2d64 | |||
| 2b89f33765 | |||
| 9c82a4ca60 | |||
| 7eb9b9b1c6 | |||
| 69b35b2ede | |||
| e8016abd97 | |||
| e54036be25 | |||
| 1dc894087c | |||
| b014967550 | |||
| 490962ccdb | |||
| d10e81fb3d | |||
| 33a87f54bb | |||
| 2cdfa59fbe | |||
| bebe74fa4a | |||
| 4098f97735 | |||
| a0b7ac402d | |||
| 0ca4482977 | |||
| 5cf15db6e4 | |||
| 6379d50f07 | |||
| 5a2792ae24 | |||
| 9b26f3cd84 | |||
| 316be5ee34 | |||
| f208e6f0b6 | |||
| 7c8c131e1a | |||
| 81e71d7275 | |||
| fc6c4b8e16 | |||
| 2280566bca | |||
| 33173e76a0 | |||
| cc0b9e5088 | |||
| ecc72d7693 | |||
| 4ab4aeacb0 | |||
| 348d12bd60 | |||
| 7b686c1103 | |||
| 403583cfbb | |||
| 29bff06cd6 | |||
| 2b80057ac9 | |||
| c3625e16d7 | |||
| f4642e9e66 | |||
| 5bdf4c6143 | |||
| 1250850492 | |||
| 80aaf6a065 | |||
| 3025caf932 | |||
| 95cfc50fbd | |||
| ded0925155 | |||
| ebfef15fb0 | |||
| 931408037c | |||
| ad418abf91 | |||
| c7d51a26f6 | |||
| 2c01cc000e | |||
| 3318651565 | |||
| 2ab07f3aac | |||
| 1340e54327 | |||
| 1b6e4645b1 | |||
| a6a573f5dc | |||
| 415415b5b2 | |||
| 175769b53e | |||
| 22f193f042 | |||
| 38e9b92c42 | |||
| 345d727936 | |||
| 86e8e5199e | |||
| 739979a116 | |||
| 7cbc81adca | |||
| 737d0fa23c | |||
| ab1962934d | |||
| 192455702d | |||
| cb0d739daf | |||
| a937fcc477 | |||
| 9503e60444 | |||
| 9f2dc4554d | |||
| 55da600433 | |||
| 96b5c2ae00 | |||
| 3008dc76d3 | |||
| 596c4b77e8 | |||
| 05a31c2686 | |||
| 45535c0f5a | |||
| 4bd0dd2123 | |||
| 9b2a643626 | |||
| 1ebc9a9a88 | |||
| f0c8b7ce40 | |||
| ec54550f21 | |||
| 4474d200b0 | |||
| b24a9e57fd | |||
| 07722dc33d | |||
| f062e35641 | |||
| 321ef9816c | |||
| be01e925c7 | |||
| 6d11006b54 | |||
| ed792b97c0 | |||
| a4b8c2298a | |||
| e5b33ce9f6 | |||
| 30374d46c9 | |||
| 9edf8233f7 | |||
| bd4a14519c | |||
| f12bf8c09a | |||
| 53cc45a0d5 | |||
| fa4b4dece1 | |||
| 02f044a2a1 | |||
| f39f816a98 | |||
| bc5b95be8a | |||
| 431d51f5c4 | |||
| 45c1357bab | |||
| 845f31b98f | |||
| 5136675fae | |||
| 89b6c32cee | |||
| 6ee36fe361 | |||
| db4f23f377 | |||
| 77572d0aee | |||
| 6ff02761a9 | |||
| d46f267663 | |||
| 6a98534d5d | |||
| eeb5d99942 | |||
| dcc5f333c1 | |||
| ff8a66d22f | |||
| f7ad97918a | |||
| 37b79735bf | |||
| 9d9ad6de88 | |||
| 20b925abec | |||
| 9047d56aa0 | |||
| 945ddc2403 | |||
| 92158b0611 | |||
| bed6155c79 | |||
| f2459e61dd | |||
| 69dcdafb3d | |||
| 6dbb072d11 | |||
| dc96849718 | |||
| d7a3cc505c | |||
| c6936ced6c | |||
| 91048f655c | |||
| 66bf524394 | |||
| ba9448bdd7 | |||
| c5cb5cba18 | |||
| aa853ac833 | |||
| 8b759d0e1e | |||
| a8a2192cf9 | |||
| 37f866b47f | |||
| 9cf653d673 | |||
| 0167b4b4a3 | |||
| dcb773e446 | |||
| b1a86fbc98 | |||
| cdd0cf7f18 | |||
| 9ae419201d | |||
| e37cefdbee | |||
| ad8c266f76 | |||
| 807c3bdcc7 | |||
| 62efbd17de | |||
| d99350fd61 | |||
| 40c70a9a2a | |||
| 8fb7f40a6a | |||
| 2f12d41d9d | |||
| 9fbdb6b305 | |||
| 73285cadb6 | |||
| 7d00722bbf | |||
| 4ea600d34e | |||
| 56f1c295b6 | |||
| 7f76ed8413 | |||
| f7edd36931 | |||
| c39f2b7c05 | |||
| 342036408e | |||
| f4904fce17 | |||
| 2abb2de753 | |||
| b61d7c2428 | |||
| bcc5d7c00f | |||
| ef0a0db07e | |||
| a45795efec | |||
| 88ae353aef | |||
| 97b9690711 | |||
| 48ce356d5c | |||
| 92451f94bc | |||
| 593632045d | |||
| e3c55ef307 | |||
| edf1730fd2 | |||
| 34cd8e3f95 | |||
| 242cce022a | |||
| 4a36cca703 | |||
| 19bf51cefb | |||
| 74a2e80142 | |||
| b9b630e3b6 | |||
| f0bdf833d1 | |||
| 81bc6bf34b | |||
| 7de736e8d0 | |||
| cd097e0fce | |||
| cc81a7ccfe | |||
| f83ae630c1 | |||
| 58d320c270 | |||
| 5894f35364 | |||
| c5acbf7e22 | |||
| 59565fd1d1 | |||
| 55592137a2 | |||
| 58523060f0 | |||
| 567aaf87c6 | |||
| e660d683a0 | |||
| 685306c386 | |||
| 5e04274d84 | |||
| 07d53be9fc | |||
| d4b0235a8b | |||
| 34aa41e17b | |||
| 3357fded14 | |||
| 618fc54ac2 | |||
| 339e058b64 | |||
| 102027a343 | |||
| 36f6a9347c | |||
| d49d386ef2 | |||
| 00c363829c | |||
| a9691dbdf4 | |||
| 9df701906f | |||
| 283671fa9d | |||
| 435c29755d | |||
| 686f91777c | |||
| 0d1df6bec3 | |||
| 925f60d9c3 | |||
| 8b3f5fda07 | |||
| 2aa028facb | |||
| b4bbd050c2 | |||
| 2a4fc28318 | |||
| 313485e406 | |||
| faf4267c73 | |||
| e6277d799f | |||
| cdbc8004fb | |||
| 1fac2f686d | |||
| 08c8d679ac | |||
| 48c34b7234 | |||
| ac17b2c584 | |||
| c67c861dc6 | |||
| 28603f0d2c | |||
| b2855f02fe | |||
| bef3d88076 | |||
| e1a8ea7dec | |||
| c4ad97136f | |||
| eab1d6782b | |||
| fd7b8ec77e | |||
| e28c991331 | |||
| a52811dfa3 | |||
| 9e210d705d | |||
| c42f1b53ab | |||
| d171173e90 | |||
| 679f0f9363 | |||
| 724c1e297f | |||
| 83154569b1 | |||
| e3c0fba34b | |||
| 2b6a6b91f3 | |||
| 09a555fdd2 | |||
| dc32f7f0a3 | |||
| 1efd8d6c75 | |||
| 898fc72313 | |||
| 21c5806cbf | |||
| 464e6bec95 | |||
| 2ae832d919 | |||
| 5b03c2d949 | |||
| f629a998a0 | |||
| fe88781bc8 | |||
| e725c97967 | |||
| 63caf22671 | |||
| 44790b1333 | |||
| b40bb64612 | |||
| 09ba9e6259 | |||
| 7775166477 | |||
| 7b5ab29a6d | |||
| 4fd614be09 | |||
| 73236e58c5 | |||
| 0e167f5c24 | |||
| a310a32371 | |||
| c00e26be81 | |||
| 32414853c6 | |||
| f3dc78d457 | |||
| a32ac62208 | |||
| d7a934cf0e | |||
| 503491392d | |||
| b3a2bf367b | |||
| c19eff4872 | |||
| 2941a813c2 | |||
| 0a022d38fa | |||
| 7ed3b3dd3a | |||
| ce52963d2b | |||
| 9a1922fdc6 | |||
| 967424a538 | |||
| 83131103cf | |||
| 9f4a0d3216 | |||
| c1591a5efd | |||
| 918ef4dff8 | |||
| 0d9a04c713 | |||
| 0c0c69f0cf | |||
| 943e80e26c | |||
| 058a327584 | |||
| 1eca4170f7 | |||
| c268e4ad1b | |||
| d4f81e8791 | |||
| 4f0680c3c8 | |||
| 8c26fe44c3 | |||
| 8c7d9f3dd2 | |||
| f241b7e79a | |||
| dc1f3503be | |||
| c2a5e180b8 | |||
| 7351217489 | |||
| aa42aafe33 | |||
| b2da0120d6 | |||
| 9e84e09c26 | |||
| 32c1a9bc45 | |||
| a7e95922c1 | |||
| 1392d0bc14 | |||
| 0f9fa9507e | |||
| 1087535d8f | |||
| 0e51f51979 | |||
| 90e0141ac5 | |||
| 8435a8678e | |||
| bd2888fc3b | |||
| ce1a5cd2ce | |||
| 5c8a28d717 | |||
| 6578ffe2c9 | |||
| 175340522f | |||
| afc917b582 | |||
| 9f4cd7716e | |||
| 29b0017445 | |||
| 910a7c619a | |||
| 273fac2028 | |||
| 59c5d984af | |||
| c885903ff2 | |||
| a323d85d32 | |||
| 491a33de0b | |||
| d6a0a44432 | |||
| 752533489a | |||
| e4e3c19e96 | |||
| 4ddb066728 | |||
| 958bbbc8cb | |||
| e15be5c2bf | |||
| cc436dc8cb | |||
| abbcd1f436 | |||
| db494f2afc | |||
| 985ea29940 | |||
| 76359da58e | |||
| 368cd44558 | |||
| cc1387ec0c | |||
| 7c79985a29 | |||
| 2b56961b54 | |||
| ff9920cbdc | |||
| 953a67bc3a | |||
| 29343aec3a | |||
| 2972472179 | |||
| 240e7b0835 | |||
| baf5191433 | |||
| 2645e87766 | |||
| ec8bc02d33 | |||
| 054bc970e2 | |||
| e4403ca396 | |||
| 04912ea888 | |||
| 103238066d | |||
| 7e4f08c033 | |||
| d47d82d8e1 | |||
| c1c41242bb | |||
| 9b9b44dd65 | |||
| dc5627a2ef | |||
| c1dfae1a6e | |||
| 7b5e4ab426 | |||
| 169ff73d26 | |||
| d985ed553a | |||
| dea1ef24d9 | |||
| 49f29a0453 | |||
| 76af9ba53d | |||
| 2de364414f | |||
| 5ae84970e7 | |||
| 141b0d38a6 | |||
| 44891b6924 | |||
| f008588307 | |||
| e481d03b5e | |||
| a8a73b60c4 | |||
| 7e8b76e8ea | |||
| 36c746bd9f | |||
| 7476c583e7 | |||
| 89928ca8e4 | |||
| 38a3bf3ada | |||
| 96b3d31b42 | |||
| 362ae5c4bb | |||
| dc303c2a71 | |||
| be2ca0ea22 | |||
| 460cb19839 | |||
| ddfebb17cf | |||
| 375c9dd116 | |||
| 15716a0772 | |||
| 6f6c1cd330 | |||
| 36ac757c3a | |||
| b614cfffcb | |||
| b58a52c7b4 | |||
| a80fc1b062 | |||
| 90d18189da | |||
| 22a2e95126 | |||
| 11e1a99e14 | |||
| 3c6bfb880d | |||
| ad2c05c3f5 | |||
| fa75f54a05 | |||
| 6405fd4770 | |||
| f42f20c70c | |||
| 209bc59e8c | |||
| 84ee86f6b3 | |||
| ed7791d824 | |||
| 0580b10385 | |||
| 47c7351b16 | |||
| b158072a15 | |||
| b6486c26e6 | |||
| da0ac96704 | |||
| af1fbda892 | |||
| 2234c45c19 | |||
| fd38fb684a | |||
| e0a16e08dd | |||
| 43189dfe3a | |||
| 46d4f6037d | |||
| e522811a52 | |||
| 6124bbb12a | |||
| fbf911cf7e | |||
| 7fdfa81fb8 | |||
| a4673f3007 | |||
| 26a44068d8 | |||
| 602b12dcf5 | |||
| a91a836224 | |||
| 969d7c802d | |||
| 4e196d408a | |||
| 8450ab8dab | |||
| 24c499d282 | |||
| 4581c57478 | |||
| f177924629 | |||
| 633e888ba7 | |||
| 4316992d95 | |||
| 5ecb8bdd8a | |||
| 168889d999 | |||
| e1339628d9 | |||
| 1ee190e844 | |||
| aadcfed17d | |||
| 8f99f6eb66 | |||
| a51b948f45 | |||
| 425f61cf34 | |||
| 87cc2d2313 | |||
| 0e2132ad3e | |||
| 7d9df5abc6 | |||
| 118cba4d9b | |||
| 3b81d4b8a5 | |||
| 6e3b3dc4e7 | |||
| 3b2adc9a3e | |||
| 8d421a62d2 | |||
| 185b0690c8 | |||
| bd0e97023e | |||
| 34ff0706a3 | |||
| acba61babb | |||
| f91191218b | |||
| 009b5bc72b | |||
| 05c79ac8c2 | |||
| 24d2a93c0d | |||
| 9b541a28e6 | |||
| de43080228 | |||
| b0cd7be39b | |||
| a7169a6348 | |||
| 6e126fb97e | |||
| 3533429563 | |||
| 22783d8f6c | |||
| 500230af51 | |||
| 1d710bdcd9 | |||
| 5feffba1ff | |||
| 6c56586a6b | |||
| a32b66c7c3 | |||
| 7e3c06191e | |||
| 372c96c6b2 | |||
| 4a2cbc1715 | |||
| 7073b8721a | |||
| 61f8fdd9e8 | |||
| fccb9c0bf4 | |||
| cfdca9f702 | |||
| 3d09090c4e | |||
| 596a49c112 | |||
| 95fc253d6b | |||
| e6d5372029 | |||
| 8e5c692244 | |||
| e694c664e5 | |||
| 8779f93746 | |||
| 1f0f5c1e23 | |||
| cbe24d0c61 | |||
| f9f12131ae | |||
| 50f0da6793 | |||
| 7e0106da0c | |||
| 0b7ab0a095 | |||
| aaf6bf3cd2 | |||
| fa95c82daf | |||
| 0da7142a59 | |||
| 446a938b06 | |||
| 21c3994650 | |||
| f90f3a5ca6 | |||
| a34d2b72c0 | |||
| 9d617dcfec | |||
| 55506a0fc3 | |||
| 19c03f504f | |||
| 985a3436e2 | |||
| 9dae87c80c | |||
| 46364a38c6 | |||
| 148b2b9d02 | |||
| 64354b51c9 | |||
| 3cacb48f3c | |||
| f6a58151cb | |||
| d180bc794b | |||
| 3404393974 | |||
| eab5fd5bdd | |||
| 6965812d79 | |||
| 78fb7fe9f9 | |||
| 24bcf6a088 | |||
| e3ca797dad | |||
| f2db1c8ab2 | |||
| 36b8a75ede | |||
| 25d0a363a8 | |||
| 11b2815b88 | |||
| d7c8075862 | |||
| 0a42b85c06 | |||
| baf231e3b6 | |||
| e26e85b6d6 | |||
| b2193b23e5 | |||
| 2af3a92833 | |||
| d2af6dcf38 | |||
| dd6b66167e | |||
| 51a4a88a81 | |||
| 516efb21cf | |||
| 2d4397af53 | |||
| 06f319a380 | |||
| 37ed5a01e0 | |||
| 4a9997e449 | |||
| 54269553c8 | |||
| 041b97dd25 | |||
| 541d05df1b | |||
| 3d6ea23511 | |||
| 9b85a6fb7c | |||
| c0554c9fbf | |||
| 61130ea191 | |||
| 2581e56503 | |||
| fea0ae7f2f | |||
| 3299438cbd | |||
| 5c160048df | |||
| e3e1036dda | |||
| 876d7ac85e | |||
| 70b37dc469 | |||
| 6393f69138 | |||
| 0ca39f6c65 | |||
| 02493251d5 | |||
| ff04648112 | |||
| 55002d7adf | |||
| 0664c6b5b0 | |||
| a4ebac147b | |||
| cf802dc67e | |||
| 84365882de | |||
| 37fb6473b3 | |||
| ba676f2810 | |||
| b3d7c622c3 | |||
| bc016e360e | |||
| 594918dd3c | |||
| ec5feb41e8 | |||
| 94c52e3a77 | |||
| 875de4f637 | |||
| 72fa5a69f1 | |||
| d63e54237b | |||
| e256d93b43 | |||
| 4d12df5424 | |||
| cae120fd4d | |||
| be332a6223 | |||
| bda0bb6f13 | |||
| 68c5dcd83d | |||
| 9bdcadf634 | |||
| 037be433f8 | |||
| 2bed62dd9e | |||
| a27bc4ebea | |||
| d6e34761dc | |||
| 98effcd8e3 | |||
| baa87bc823 | |||
| 944d9c84a0 | |||
| 1e447741ee | |||
| 4405ac7386 | |||
| e1190f0f0f | |||
| a7f2416c0c | |||
| 40d0100132 | |||
| 37f7c48cfc | |||
| 21adf752c8 | |||
| aec143b882 | |||
| f691040936 | |||
| 42acf0ed60 | |||
| ca4a3589e5 | |||
| 2eead17224 | |||
| 626b26a227 | |||
| c46db0761e | |||
| cfed06697d | |||
| a0d9183b14 | |||
| d3eb674b30 | |||
| f407ff8861 | |||
| 7fe1fdd8c7 | |||
| a413b83c01 | |||
| 37cbe68204 | |||
| 81f4de965f | |||
| f76a66fc55 | |||
| 030b1f3467 | |||
| d7949aa58e | |||
| b7a180114e | |||
| f0c0c5483f | |||
| 4c9a26dbca | |||
| e611828249 | |||
| 7d444021bb | |||
| 388a29bbe2 | |||
| a03dd1bd41 | |||
| 4b366f2857 | |||
| c87faace6b | |||
| e2be051558 | |||
| 1e8b185377 | |||
| 031804827f | |||
| 6cccd9b6fc | |||
| 687fbb0a7e | |||
| 8f2db99c86 | |||
| 2c0f8dc546 | |||
| a388fb0bb7 | |||
| 27465353c1 | |||
| c2ccab4361 | |||
| bb876eac82 | |||
| 34c04babbe | |||
| e80a9b0075 | |||
| 7c6a310179 | |||
| 50702eda94 | |||
| 59eeafbdfa | |||
| abc606608c | |||
| 1487552b48 | |||
| c7dbe18df6 | |||
| c2bc3358cc | |||
| 47a1494d68 | |||
| dbb388719e | |||
| 283c91548a | |||
| 38b93bd310 | |||
| 8dcc30ac83 | |||
| 0ee123375d | |||
| be18cbef8b | |||
| 6a36ec63d7 | |||
| 9c8b907ff1 | |||
| f769df16e8 | |||
| c6f5075721 | |||
| 8eb494c13e | |||
| b6b6375f70 | |||
| 8783688391 | |||
| 98afc3e99c | |||
| 50a1858367 | |||
| f3f586773b | |||
| 61a182077f | |||
| 1c9513e770 | |||
| 5e5eb9bf8e | |||
| 7a9bb65e03 | |||
| a5345ac71e | |||
| ae5079f7b4 | |||
| ea1ecfbc38 | |||
| a84b6b4bcc | |||
| 93023128fd | |||
| 77157f16a1 | |||
| 99736e5066 | |||
| 681306b7a1 | |||
| 5f36c9d4de | |||
| 6cbc8791b1 | |||
| c08de67b0d | |||
| b21e18dfad | |||
| 1e497915be | |||
| 3704c41dda | |||
| 6ff31ac666 | |||
| a2f73a7d35 | |||
| 1492e57676 | |||
| 7504fc53b6 | |||
| daa2bcefad | |||
| 49aa9399be | |||
| a71090df81 | |||
| 0bfcafc5c6 | |||
| 161d5c8379 | |||
| 5cfb578170 | |||
| 9b0d47e9eb | |||
| 13f4706067 | |||
| 7ebdb1736f | |||
| 2bcb57c994 | |||
| a2df691c7d | |||
| 58f1191f2d | |||
| dfaa999291 | |||
| a693698279 | |||
| 9370f9cae4 | |||
| 6a58033f2b | |||
| 7705a6c1f1 | |||
| 0a803891a4 | |||
| 7d620a93b9 | |||
| 3f3b2f4c99 | |||
| 9eb4089710 | |||
| 257d1afdf8 | |||
| dad1fb7805 | |||
| e312fdd4f8 | |||
| 9b6681d543 | |||
| bcc04623c1 | |||
| d75d162058 | |||
| 8f2294bbd4 | |||
| d1b91c7619 | |||
| 1b6b481fcc | |||
| dd64ba1910 | |||
| 82a5d1cd79 | |||
| c26c172d01 | |||
| a7b7aaa7cb | |||
| 4e5c02c05c | |||
| 2baf61fda3 | |||
| 219a25fe80 | |||
| b1dd704819 | |||
| 9513e91d66 | |||
| 26d52bedb3 | |||
| 19451e0654 | |||
| fa922d7792 | |||
| 4949e3ba41 | |||
| bbe1de3119 | |||
| f87e9b596d | |||
| 917e12952e | |||
| 1977c526e4 | |||
| b63351074c | |||
| ebcdea63c0 | |||
| d78eb1247e | |||
| 9b9fe0d65c | |||
| 5a2db802d9 | |||
| d3972b88f2 | |||
| e62cf13760 | |||
| a5f6e3bba0 | |||
| d170660c25 | |||
| 2202aaed51 | |||
| cbefcd50cf | |||
| 1acfa291a0 | |||
| 21accd534c | |||
| 604f2c9161 | |||
| eb29989dff | |||
| 4d9ca822a7 | |||
| de89d7a976 | |||
| d1f3d95c96 | |||
| 78ef42daa1 | |||
| 319abebd70 | |||
| 76480adda5 | |||
| e205f8afbb | |||
| 42dc51784e | |||
| a4a46f480d | |||
| e34be16237 | |||
| dfcc166918 | |||
| 895d56ed04 | |||
| 12eab4a8ba | |||
| 3eb2b1f7a2 | |||
| 6ecc9bf93a | |||
| 22e24fc387 | |||
| 516d88b072 | |||
| da4ebb6535 | |||
| efa0a06947 | |||
| 77457e91e9 | |||
| 6e4d33c741 | |||
| d3387e2a28 | |||
| 491452a19d | |||
| 7d3257b222 | |||
| 1836ef2884 | |||
| 43d6322d0f | |||
| f0684d83e9 | |||
| 3f3170818d | |||
| 7683096fe1 | |||
| bb438bfb17 | |||
| a11aa295de | |||
| 11eb241c8f | |||
| 049d92b525 | |||
| ebef239a06 | |||
| f9bd59f031 | |||
| 3d5507451b | |||
| 6d0d5bd566 | |||
| 98a13204b2 | |||
| 35d20a19bc | |||
| c318fdc94b | |||
| dab1c4cfc9 | |||
| 9a50f4ac1f | |||
| 59e829e595 | |||
| d0229b62da | |||
| f86946c6df | |||
| 37ad20a71b | |||
| e97f75cad5 | |||
| fcd6ebb06e | |||
| 2505f82ce5 | |||
| 78dca5fe8b | |||
| 8c816f64e4 | |||
| 22c525e3fe | |||
| f3f6b03d85 | |||
| 00bebc317e | |||
| 8f38e83aaf | |||
| 8fab7ec5e3 | |||
| 50eb968109 | |||
| 569314be45 | |||
| 909d60464e | |||
| d855abf8b0 | |||
| b611f72e08 | |||
| 44e3bec42e | |||
| a04b005e93 | |||
| 1b837116e6 | |||
| d16b04b683 | |||
| d2e7a8004d | |||
| 7996ef0d45 | |||
| 0c28216ee5 | |||
| b05c1a5bb9 | |||
| 5bb8ea7449 | |||
| b78c515724 | |||
| cb1a7a7bdc | |||
| b8dcb7c884 | |||
| 9dd6f848bd | |||
| 1ded554a15 | |||
| bc0ce7b820 | |||
| 1da3a57fe7 | |||
| 8697982302 | |||
| c4168cf855 | |||
| 7023d3ca2b | |||
| 57a5d13c47 | |||
| 500b96240b | |||
| 13d961d41d | |||
| fddc4c2fc0 | |||
| 061ec7369f | |||
| 8366dbd8e0 | |||
| b02047e4b5 | |||
| e9545c4961 | |||
| 966a2b1df5 | |||
| dec6540967 | |||
| 3fe1673ce9 | |||
| e9e13474c9 | |||
| aee9093848 | |||
| 76822c7c34 | |||
| 5c18d34d89 | |||
| 970a9c7552 | |||
| 37a42dc408 | |||
| a03c9f9457 | |||
| 60004ebff1 | |||
| 2d9fcf6828 | |||
| dc9c86e3a1 | |||
| e1959afb6b | |||
| 6bc6ae2d28 | |||
| c68c78d412 | |||
| c8ac9721d7 | |||
| f8bedc55e5 | |||
| b72d31f87f | |||
| b1b68b58fe | |||
| 95e15c95f2 | |||
| ca21db9481 | |||
| 93ad803073 | |||
| 6cc7f70a65 | |||
| 9f871a3726 | |||
| e19e2c123e | |||
| cbe44e1fff | |||
| 2b0c33f74d | |||
| dae1d36a23 | |||
| 824fa8f17a | |||
| 31cd0b943c | |||
| f376c79f7f | |||
| 8e191c8e6b | |||
| 070eced2f6 | |||
| 986f8dfb2e | |||
| a98824b4cf | |||
| 19d742b9e4 | |||
| 8c0c03eb38 | |||
| fd9bc20bc5 | |||
| 089fca2319 | |||
| e936890927 | |||
| 0450d48f89 | |||
| 2463819a3d | |||
| 2b2cae2d50 | |||
| 0f1b40da71 | |||
| f73d5a9ab2 | |||
| 4eb0e24c6e | |||
| 1d2235abe7 | |||
| d347e54acb | |||
| b5198d8119 | |||
| b8b5c5ff34 | |||
| a738490a3b | |||
| 54a8de2059 | |||
| cb2c0e7ac5 | |||
| 510d309b8a | |||
| a7ce2a7aa5 | |||
| cfe24ecdd9 | |||
| c5e9cb025c | |||
| c3d07d60ca | |||
| a0897a7456 | |||
| c50ba9267c | |||
| 423e69916c | |||
| b56c76f8ad | |||
| cb2d2f000f | |||
| 69af77a3bd | |||
| 7767746d3e | |||
| 7219aaeb89 | |||
| 7af1863e81 | |||
| 4beb42bf45 | |||
| 12a3086a9e | |||
| 198725216f | |||
| 08647f1267 | |||
| 87811efc30 | |||
| 82c3e6f87f | |||
| 1ac40a3043 | |||
| 127b0c3332 | |||
| a6d9150b14 | |||
| 7e5197c566 | |||
| 2d217e72bd | |||
| 12331cc62b | |||
| 2449f1e1b6 | |||
| 6a6593c656 | |||
| ad220d61f9 | |||
| 1e35383b4d | |||
| c8457ab005 | |||
| 070a308217 | |||
| c1f4477376 | |||
| d728320ece | |||
| fee0d7168a | |||
| 7c23b32de3 | |||
| 1437952aee | |||
| a162157301 | |||
| 6316bf3582 | |||
| 1d856b4723 | |||
| 7f56d5c23a | |||
| a778763851 | |||
| 983d7ec265 | |||
| 297769ef57 | |||
| 885d050e5f | |||
| 8fb4ce6cad | |||
| 42738ab54d | |||
| 7d1250620e | |||
| 5c49b93c67 | |||
| 9a11f81fd3 | |||
| cba2e972fd | |||
| 76ad925842 | |||
| ef6f52f688 | |||
| 197bfa9f11 | |||
| 145f8c7435 | |||
| a8b43ae598 | |||
| 567f19bf68 | |||
| 5cd4cd2271 | |||
| 4180569443 | |||
| 5f4a92c8e6 | |||
| ccf3fed950 | |||
| 3626003f68 | |||
| 11cb040ad1 | |||
| c1761cab49 | |||
| 25b25b5434 | |||
| 8bdf66d9c0 | |||
| ccebdd142a | |||
| e952da7f91 | |||
| 5bd1e4a167 | |||
| 6d3de41751 | |||
| f11bac6705 | |||
| c23a601cc6 | |||
| 860fbe48dd | |||
| 36d4c69fd6 | |||
| ca7e7fa0c4 | |||
| 1f6dd5dbb9 | |||
| db52646655 | |||
| 8bb18fa988 | |||
| f0edaf2f8c | |||
| d632e3aa1b | |||
| f0e58fa804 | |||
| ceced09d02 | |||
| 7d48115b90 | |||
| d2205228fb | |||
| 5417fb7287 | |||
| 3f59d6daff | |||
| 0d659933aa | |||
| 3e8eabe833 | |||
| badfc77339 | |||
| c51d3e59ea | |||
| c0d02a65c3 | |||
| 3a203b8d83 | |||
| feecdcc7a4 | |||
| 51ad533be6 | |||
| 29da0bc8f5 | |||
| bccf7fc2a8 | |||
| 7b6b5981c4 | |||
| 9463192224 | |||
| d1689f0012 | |||
| 8ed67fe349 | |||
| 90a1d99785 | |||
| e215cf6fb8 | |||
| e827c0bd94 | |||
| 8dd7e4e6b5 | |||
| a2b94f4e06 | |||
| 8b0037ffab | |||
| 4ab03f3bef | |||
| 3c3db52f49 | |||
| 5b1e884659 | |||
| 7ec6740e26 | |||
| f12b8c19be | |||
| 76174d31ce | |||
| 5042248260 | |||
| 1df6589533 | |||
| 12f76b448c | |||
| f112ef34f6 | |||
| e4b57a978f | |||
| b51e09e0e8 | |||
| a3ba3f895c | |||
| 947a129e12 | |||
| f3fe6a6cbd | |||
| 9d150bef9f | |||
| 65be18cc93 | |||
| e27ea63900 | |||
| aa96f7b660 | |||
| 053690d885 | |||
| b9fc6397a3 | |||
| 19b9f15da3 | |||
| 0b9441e1a4 | |||
| 2324d7420c | |||
| c6b2ca8b19 | |||
| 83ea8dc577 | |||
| d898277f62 | |||
| a289cfb986 | |||
| d603998617 | |||
| 8bf9f4f5ab | |||
| 2c4e6f2926 | |||
| 21fbbc50cd | |||
| 99512418da | |||
| c2f2d8771f | |||
| 179c9ee8cc | |||
| eb8a505287 | |||
| a7482a3644 | |||
| a692348336 | |||
| 285dcc30cf | |||
| bcf51aed83 | |||
| 1a11ce6211 | |||
| 10021c97a6 | |||
| e869e3c534 | |||
| f6416285db | |||
| 7f0593cd2d | |||
| e4c41718d8 | |||
| 7234553990 | |||
| 5761efdb73 | |||
| 1133192a86 | |||
| af3288043a | |||
| 24a348f6a8 | |||
| 5528b6c231 | |||
| 245bd1eb17 | |||
| 03506db76c | |||
| cb5ef26020 | |||
| 2493ae4c2c | |||
| 2bd88344ad | |||
| 4950980d69 | |||
| 2361a0dd6e | |||
| 152cdd1750 | |||
| 31797a5831 | |||
| 9308c42cff | |||
| c096cd34b9 | |||
| 5fc0808f28 | |||
| e6866ee980 | |||
| 45631d30b0 | |||
| 0f432a0844 | |||
| df59bc7194 | |||
| ff4706e450 | |||
| fc013bd04c | |||
| 7e29a8b927 | |||
| 67ae7a0b6c | |||
| bd5a64bac0 | |||
| 1bd85d8baf | |||
| fe34b08ece | |||
| 33048f88b8 | |||
| 0ec01f4e78 | |||
| d0ebf06ff8 | |||
| 687b249034 | |||
| d1528dcff0 | |||
| 1c31cf6319 | |||
| 67f0c9bef0 | |||
| d54c366150 | |||
| 4ff535f883 | |||
| 58b15f9452 | |||
| dedca59ac9 | |||
| 3cbddfe545 | |||
| e3cae69495 | |||
| aee40316f8 | |||
| d754f9ae89 | |||
| 3c50b3a9e0 | |||
| fb312a71f7 | |||
| 136d79eaa3 | |||
| 834336499a | |||
| c9da8237df | |||
| 9638dcda0a | |||
| 756c5a2604 | |||
| a9c31652b6 | |||
| 32a76901a9 | |||
| a5e11c7489 | |||
| f2b12014e1 | |||
| 60fcaebfdb | |||
| 32fe2cb659 | |||
| 1207d54fdd | |||
| 3932884688 | |||
| 19a2042746 | |||
| 4c6eb137da | |||
| 57ec2ff915 | |||
| 77a161a087 | |||
| 0642402449 | |||
| 50d377d9fe | |||
| f5211b0697 | |||
| fd4ea46fd7 | |||
| 8d8546868d | |||
| 8ce547edeb | |||
| a17c48aed6 | |||
| d12db3e7b8 | |||
| 15b87ae297 | |||
| 02fdf59839 | |||
| d9da02b7a8 | |||
| 8f2ad6418d | |||
| ff984425a3 | |||
| ac1058359f | |||
| 9afbca3001 | |||
| ec3f17cb9c | |||
| 73b9d5c5f9 | |||
| ecc8591c95 | |||
| 696b67e4b1 | |||
| 266a5116a1 | |||
| 131f2be857 | |||
| be7b3a9952 | |||
| bb31b1785b | |||
| 2a60f4b1e9 | |||
| 33a4fb5a1a | |||
| aece6e8b6c | |||
| 7bf55dd14f | |||
| e158f17c2b | |||
| c5027d9478 | |||
| 36c1d82146 | |||
| bd4f404d45 | |||
| 43d39844f7 | |||
| e041a4d212 | |||
| 433b923ea7 | |||
| f8f1c72b44 | |||
| 542716e216 | |||
| b35958d024 | |||
| 9ee3541655 | |||
| bf7d84c12a | |||
| 34c691087e | |||
| 08c383012f | |||
| e2420495f3 | |||
| d530c5eda7 | |||
| ef7420ecf6 | |||
| c905a41e2a | |||
| 42ff4b5bf0 | |||
| 4fb74a32cc | |||
| c741465328 | |||
| fbca537a40 | |||
| 83420b0199 | |||
| 33d3ba1b45 | |||
| 497f85a236 | |||
| a624c302ab | |||
| cebe21a3af | |||
| 9eb679d70a | |||
| 6d84443db8 | |||
| da8a1f242c | |||
| 946d98b71f | |||
| dff51fc707 | |||
| 7d954dd5d1 | |||
| c6300a5da8 | |||
| 9359daa0d9 | |||
| 2322e9cff7 | |||
| a876e1e348 | |||
| 6a863c8f71 | |||
| 392b006b06 | |||
| 96289f42b7 | |||
| 1b69c2441c | |||
| 8ca85a4918 | |||
| 2a31031cbc | |||
| d148cd8ccc | |||
| d1cc1828b8 | |||
| 069e8cf122 | |||
| 45cbcaca6d | |||
| 102a2db1f3 | |||
| 9f81c85ca7 | |||
| ba4a6fc0c5 | |||
| aa803ce2ff | |||
| a027a60f5d | |||
| 270649535e | |||
| cf80ba71f4 | |||
| b74df18a4a | |||
| 5cd2906a39 | |||
| bc37b69d17 | |||
| 94f6e400ad | |||
| b95a6ccf80 | |||
| 7df9c1b6e4 | |||
| 75348c0158 | |||
| 75fb14acaf | |||
| 5350315b68 | |||
| 658e39c270 | |||
| ef7ce6c7e1 | |||
| 509e2411bf | |||
| 65c906f951 | |||
| 1f159e8233 | |||
| 936c76119d | |||
| f45865606a | |||
| cfc9776bae | |||
| e7db264803 | |||
| 0cb7ed9e4e | |||
| 4b07609458 | |||
| e41e58e781 | |||
| f5030f1c2c | |||
| 2a48fb8e87 | |||
| 3d8a71fdb2 | |||
| df6dbc5fa4 | |||
| 4b1d2839e8 | |||
| a892f80e86 | |||
| b2a79855ae | |||
| ff4974178a | |||
| d7100fd9bc | |||
| 0bfb40ae51 | |||
| 11c83670d6 | |||
| 68ff4f3842 | |||
| ab25cd09ed | |||
| 8f05b8f982 | |||
| 63ae2f64cf | |||
| 105103fae0 | |||
| 70f4792ab1 | |||
| defd9fa322 | |||
| e884d0fda6 | |||
| 5f6a8fdc20 | |||
| 196a9ddbb0 | |||
| 746140bd11 | |||
| 7b99a5fbac | |||
| b74c31e520 | |||
| 221f43e4bd | |||
| a17333d73e | |||
| 207b43499c | |||
| 0c0de17b38 | |||
| 77882e6086 | |||
| 19a9834843 | |||
| 23dab30ca5 | |||
| b5d7ce8ebe | |||
| bf4eb4b269 | |||
| a5edb6807e | |||
| ecadf30fe7 | |||
| 515f0db5b4 | |||
| c2f367cf70 | |||
| f21dfea965 | |||
| b84cad4db0 | |||
| 16ae019c8c | |||
| bd2051febd | |||
| 6fb1e03ed4 | |||
| 8d41a762b6 | |||
| 6d3003716c | |||
| 2c87c3bac3 | |||
| 739c525a98 | |||
| ab287ebf40 | |||
| ab6bcab78a | |||
| 6e317896e9 | |||
| b08ee3ff81 | |||
| 17fd09102e | |||
| a598cd2b18 | |||
| 55e434d67a | |||
| 04d4b5d8a0 | |||
| 7ea00bcb78 | |||
| 0ab56ffde8 | |||
| e7e945533e | |||
| 7fd1047832 | |||
| 19dfa88258 | |||
| 65923b5c20 | |||
| ac731aa50c | |||
| 2aa3182476 | |||
| 529c386943 | |||
| b659da8a4b | |||
| eba98717c9 | |||
| e4dba99cc0 | |||
| 454e688c3d | |||
| 54752deaa1 | |||
| a3cf37cb2e | |||
| 6459d11d32 | |||
| 15f2fabaaf | |||
| d6030b8d68 | |||
| e1757ee726 | |||
| 9fed75d59c | |||
| 5fe15475a4 | |||
| c7f6f4f48d | |||
| 747c6c2714 | |||
| 2951f128f6 | |||
| 53cb66eeaf | |||
| bcf8f798e2 | |||
| 7406176fad | |||
| 34ba5678c3 | |||
| da0b78c67a | |||
| 47e64ae503 | |||
| 520bb74626 | |||
| 4beef5cc66 | |||
| ba575f55ec | |||
| 2012ce02e8 | |||
| c67e2c2a5a | |||
| 4b1ce250c1 | |||
| 0401a07507 | |||
| c12265499a | |||
| 0e341832e0 | |||
| 489e2e6ad5 | |||
| 941f637bca | |||
| 6277c0595c | |||
| aa9eda1979 | |||
| 1d76efcbcd | |||
| 34c2c1ec16 | |||
| bf7fea9a0a | |||
| fdf15f3ca3 | |||
| 1cb5875b20 | |||
| 1a59a5478f | |||
| fc0cb704f2 | |||
| 960c0cbddf | |||
| 5750443371 | |||
| 9f67d86b30 | |||
| 66f7d83baa | |||
| b44e87c6e8 | |||
| 6da7f17c4a | |||
| b4f45d1e79 | |||
| 7d766bf7c7 | |||
| f9132cae85 | |||
| 75dc7e6671 | |||
| 3fd887fc57 | |||
| 128447a681 | |||
| 17149741a7 | |||
| 23bae932c7 | |||
| d2ec40bb67 | |||
| 9701998f82 | |||
| 9d06132743 | |||
| 0289c50ad9 | |||
| 51eea3f90b | |||
| 50490f5b26 | |||
| d12f802027 | |||
| ac7097b4d0 | |||
| 66087e4332 | |||
| 3706f9bcb8 | |||
| c505218896 | |||
| 6186a746e0 | |||
| 3e98bae5ec | |||
| 2e1e8f764e | |||
| a7492f8612 | |||
| 123b1f01e4 | |||
| 865f62e3eb | |||
| 3ea93f52ee | |||
| 27c70bdf07 | |||
| b53e545ebc | |||
| 157a4c891c | |||
| ad9ea07309 | |||
| fc483cdfc6 | |||
| 9033838cf2 | |||
| 8e5d2d5905 | |||
| 18aa66dabb | |||
| a2f7b78453 | |||
| 5c026cbe1d | |||
| dc51476897 | |||
| 39eaa577e0 | |||
| 1f006481ee | |||
| 60faabcbe2 | |||
| d3f1eaf1a3 | |||
| f568e76fd4 | |||
| e947223aaa | |||
| 8311162be3 | |||
| 75523556e8 | |||
| c82b5d4982 | |||
| 1c3158099c | |||
| bdbca75dfa | |||
| 124b189cc0 | |||
| de38b46392 | |||
| e1975644d6 | |||
| d9fd27a9e8 | |||
| 32425c5561 | |||
| 8d20923881 | |||
| 3a35b8b26c | |||
| 36c93b755a | |||
| ea8c3debea | |||
| b2425b2a25 | |||
| 49bc74e7a0 | |||
| 51c932164f | |||
| 19e82e93b1 | |||
| 3f785eaecf | |||
| e59c0f38d9 | |||
| 64004c6bc0 | |||
| 422332de7e | |||
| 5a15ba7451 | |||
| d3686bb1e2 | |||
| 3a6eeef580 | |||
| 1dc5c6b8a8 | |||
| 3532a560d8 | |||
| d2d894d808 | |||
| 2aa38bfc4b | |||
| 9c3cee9ae4 | |||
| fc521b5f9d | |||
| 1a4398cc55 | |||
| 5253368acc | |||
| 51cfc3d4be | |||
| 80bffd93e7 | |||
| df4f22e899 | |||
| 529d91fb9d | |||
| 8cc70843a5 | |||
| 70c841f23a | |||
| 108b4e2e10 | |||
| 7b22e09805 | |||
| c5838c143c | |||
| eaf71db7c9 | |||
| b322b527b3 | |||
| cc4b231875 | |||
| 05642a3e17 | |||
| 7dcc6bb579 | |||
| f15c416e59 | |||
| 6fa97eeec7 | |||
| 03bbf273b3 | |||
| cd0cce4195 | |||
| 9f1a72ec88 | |||
| 1e376cd3a6 | |||
| 49cf939c04 | |||
| 338394f8c3 | |||
| 575b62d77b | |||
| 57fc0eb5b1 | |||
| 3a19fe3663 | |||
| a10c621e33 | |||
| 0c049179b4 | |||
| e22c873ec4 | |||
| d644ebab09 | |||
| 2fdc578a88 | |||
| aeb3a3f7b5 | |||
| 044b7ce070 | |||
| 815e538f10 | |||
| 758233f001 | |||
| f4f4fda520 | |||
| 1d77aeb69c | |||
| 29dbfc647d | |||
| 55d9514e83 | |||
| 46bd7956a3 | |||
| e1ee394c26 | |||
| 19884ade99 | |||
| f0a88061db | |||
| 6057138466 | |||
| aaaa6556f3 | |||
| 4745431cda | |||
| 0455a948a9 | |||
| bf3e249237 | |||
| fb649e9525 | |||
| 9d1e2d9f46 | |||
| 9876d93b60 | |||
| b3dd05580b | |||
| 32847f33fd | |||
| bff9723fe3 | |||
| d114648c16 | |||
| 44d0da02d0 | |||
| c25107eff3 | |||
| af5c36d2a8 | |||
| 0828a67145 | |||
| 617fb84983 | |||
| 6f8ac2b61c | |||
| 6f2b4b96cf | |||
| 3cc288a169 | |||
| 0bbbf3eb3b | |||
| 4b1b56fee8 | |||
| 154fc59e93 | |||
| 218c4c128c | |||
| 53f1af0cab | |||
| f9577a38dc | |||
| fadc7d9ba5 | |||
| 1e4b2133f6 | |||
| bfefa6d016 | |||
| 8b66472949 | |||
| 3b3aa94c4e | |||
| dc05275670 | |||
| 7921082ece | |||
| efd6a29909 | |||
| 88c44b303d | |||
| e7dbb8ccdc | |||
| fe2a743c8d | |||
| 2e2d479103 | |||
| 1f5c124ac4 | |||
| 64a5bc038a | |||
| 5d9396334c | |||
| 35f0e355bf | |||
| ec160f1f0a | |||
| bbaeca96eb | |||
| 3ab779895f | |||
| 203c7360e7 | |||
| a831f174ef | |||
| 153091f52f | |||
| f8abb8e541 | |||
| c8346d0581 | |||
| 35d3af5039 | |||
| 57e8cd6eab | |||
| fc123a71af | |||
| f14836cf02 | |||
| 4178feb65f | |||
| 2edaf22590 | |||
| acd3dab957 | |||
| 6bbd74adcd | |||
| 9bb928bb38 | |||
| c87a6c5969 | |||
| c482c13dcb | |||
| b87ed97402 | |||
| ee000dabfd | |||
| 4395711d26 | |||
| a73a011ee0 | |||
| 2a8e5e2c14 | |||
| 5d9a41f712 | |||
| ebcf4b60f6 | |||
| f976b78917 | |||
| 078790bd0f | |||
| b88c5a89a8 | |||
| 57028e3acc | |||
| c586a17926 | |||
| 9d078bac54 | |||
| 38eaefcabd | |||
| ba8cadc2f1 | |||
| 380d5dfa6d | |||
| 32af626630 | |||
| 8358fedaf4 | |||
| aba915037f | |||
| ec82b0c648 | |||
| 81a87f873f | |||
| d91b8ac444 | |||
| 952e51ac75 | |||
| 11267cd44f | |||
| 0e59e0aebd | |||
| ae1d3b3dd3 | |||
| 1a91dbee5f | |||
| fd507e3e41 | |||
| 9c1a67cf47 | |||
| 69e3824840 | |||
| fcb1a98129 | |||
| 6d942635af | |||
| cda2c5d459 | |||
| 969bb5a742 | |||
| 4bccc611c3 | |||
| d18c4ece0c | |||
| 25c664b13a | |||
| 7c680c955f | |||
| 442e93d3fc | |||
| f037d1b6ca | |||
| 2c8b627008 | |||
| 221e3eddd5 | |||
| 74c39c677b | |||
| e6558832bf | |||
| 9c50625c55 | |||
| d372435e92 | |||
| a53facf709 | |||
| ffc39dfbcb | |||
| cba38b15a9 | |||
| 3450b5f80c | |||
| 5ac7564bfe | |||
| 53cd289b90 | |||
| 8dc13bcf1a | |||
| 261825a89b | |||
| f47a5a309d | |||
| a40f2b9fa0 | |||
| cfcd3892f7 | |||
| 7aaea6d005 | |||
| 703987f61c | |||
| e50a8917ec | |||
| 74d7c8e625 | |||
| a5d1383fe8 | |||
| bf2bcf515c | |||
| 4c5e94c64b | |||
| b4043216b6 | |||
| 4371014667 | |||
| fbb3222d29 | |||
| f76b5d8002 | |||
| 7e54868206 | |||
| 4ca3889bed | |||
| d84a8e6404 | |||
| eef1aebe8c | |||
| 4833b6085c | |||
| 2ceaca8828 | |||
| 48382c4b59 | |||
| d02158c0ef | |||
| 9a45f0b31c | |||
| 25c26e2f81 | |||
| 6213c4f2cd | |||
| e4837f14b1 | |||
| ce86131d12 | |||
| e6c9baf6ef | |||
| 8d6db7be31 | |||
| a2548b1fd0 | |||
| e4658bb99d | |||
| bf2e4a561a | |||
| 1816320124 | |||
| f09bfe293d | |||
| 7b4e8fda4b | |||
| c95812353f | |||
| cd34eea017 | |||
| b622ec7a28 | |||
| 7a0a702ec0 | |||
| d8fbe7b77f | |||
| dbcac37d91 | |||
| d4d391b34f | |||
| 571cf7d490 | |||
| e18b19ca5a | |||
| 48651bf482 | |||
| 5034a41c08 | |||
| 6795173e77 | |||
| 219ef996f5 | |||
| 00af1db275 | |||
| 5935ea896f | |||
| 459983c05e | |||
| ebf4f029ac | |||
| 0eec945df1 | |||
| 8824b9d68f | |||
| d2862814c5 | |||
| 25fece2d50 | |||
| beb4239d1b | |||
| 2b78e37d92 | |||
| a2070d9ce4 | |||
| 5827a686b8 | |||
| dec479532e | |||
| 5d173168cc | |||
| 2aac1cde04 | |||
| 3676f0268f | |||
| a7b75a54bb | |||
| 961a87b743 | |||
| e03d59e381 | |||
| d46ce5003c | |||
| d9ed8e125e | |||
| 4c4143d9be | |||
| 36708a5067 | |||
| 8bc7d259f4 | |||
| 2d047fa428 | |||
| 735d420d40 | |||
| 8c32955da1 | |||
| bc9fc1aece | |||
| c111ed4b20 | |||
| b88e3c99c1 | |||
| a3ea9427d1 | |||
| ce3e6e084c | |||
| 2a58ca7697 | |||
| af96f7a0cd | |||
| 7d39d1a925 | |||
| 1b6c700e18 | |||
| 6304bd60ee | |||
| 4ad4417740 | |||
| 6a4c259a73 | |||
| 12eabb220d | |||
| d68ce2d68c | |||
| 8e02c040eb | |||
| a7a317c284 | |||
| 9d6ef24660 | |||
| 14014408fb | |||
| b933e9666a | |||
| 7aff59bcce | |||
| 8e2760cb3d | |||
| 7a9fc6dbd3 | |||
| 75d0dc251e | |||
| 9a50c4d93f | |||
| 010d5a0192 | |||
| cf1594829a | |||
| 854d720ce0 | |||
| 2f43c74ece | |||
| b9817ac6b4 | |||
| 1e8da0d494 | |||
| c47be7b415 | |||
| d3f6cb860f | |||
| 83d25f09a3 | |||
| ed747a2d3d | |||
| 3a8ee4ce2e | |||
| 5ac01a3af4 | |||
| 46343f2f9e | |||
| 56ccb5b2ab | |||
| 9a946eed80 | |||
| 9c6cb0f630 | |||
| 1b066d6965 | |||
| 54c3caad53 | |||
| 9b5e8aaf83 | |||
| 5143c09bcf | |||
| 09b534b8a3 | |||
| 2496185629 | |||
| 34deb82aea | |||
| 8f72ae9da2 | |||
| b753f01ac1 | |||
| fd0a147ae6 | |||
| e94bd90782 | |||
| ce4b897d0e | |||
| a7694029e2 | |||
| 1e9110b763 | |||
| 6f3fbbbe49 | |||
| d346ec7bfe | |||
| 26a3613397 | |||
| e6318bddf3 | |||
| 514bb0beda | |||
| 41b1bd2f05 | |||
| bf40dadf04 | |||
| cb1678ebec | |||
| 0c1ac568b5 | |||
| 0f9550c747 | |||
| b13ae17a47 | |||
| f762a12d18 | |||
| 20d30a80be | |||
| 229b218203 | |||
| 4b668aaca8 | |||
| 8c7f1421c6 | |||
| d90b2c1d52 | |||
| 22f39be197 | |||
| 2fa45436c2 | |||
| cadbb6bbce | |||
| 2c89f04be7 | |||
| 597011e3a9 | |||
| 0d433b58ba | |||
| cde8ef56e5 | |||
| 110816c7aa | |||
| fbb1e168f7 | |||
| 23085eb5ae | |||
| 7344a6205f | |||
| 4b76ec40c0 | |||
| 90101d0269 | |||
| 7ac84c0660 | |||
| 2090530bbb | |||
| b6cb7ddbaf | |||
| 3422d9335c | |||
| e91f9a944e | |||
| e7ddc7cf0f | |||
| 40dfa48756 | |||
| 579f92cf5f | |||
| 4565125da9 | |||
| ce13a01e65 | |||
| 618a8682b7 | |||
| 963077f918 | |||
| 3704d2d86b | |||
| fc6a029311 | |||
| 7c7b1e6c2d | |||
| 892920039d | |||
| 51cdd38c3e | |||
| 80977bd4c0 | |||
| d8022f94ef | |||
| 1c43587d7d | |||
| b2ed32b118 | |||
| 0cc815d816 | |||
| d452b7593f | |||
| 5346bdc683 | |||
| e3911bacde | |||
| dc5c1e2002 | |||
| 2e48e298a2 | |||
| 7a1aaaf5c4 | |||
| bde92d5cfe | |||
| 691f0f4845 | |||
| fdd458d2fe | |||
| d2c0b8374a | |||
| c96c78892d | |||
| 957643f523 | |||
| 749bbec566 | |||
| 25e363c5fb | |||
| febeed3277 | |||
| 9d07aa006d | |||
| 12d69e25dd | |||
| 0c9f1efc75 | |||
| 665b4506e7 | |||
| cb5548ceb8 | |||
| c9492e54f7 | |||
| 12490eafff | |||
| 6e83d11d5f | |||
| 9c6aedc91b | |||
| a9339d0627 | |||
| 4d9aa10532 | |||
| b00264b594 | |||
| e329c7015e | |||
| 1392cfc72d | |||
| c6688d8f89 | |||
| 87abea0ba3 | |||
| f9fcb44f3c | |||
| 996cbbca38 | |||
| 581f4b89bd | |||
| 88a347dce0 | |||
| 3e7b197a1d | |||
| 94ab06e92f | |||
| d38c81fcff | |||
| 3e26fdfb67 | |||
| e1be73232d | |||
| 190c61ba2f | |||
| 06fd2268d9 | |||
| 15251dfae1 | |||
| f62812a8dc | |||
| c6041d2590 | |||
| c7e779107c | |||
| 7cd25c919f | |||
| 47d67d3985 | |||
| 1a7921b46c | |||
| 43d569741b | |||
| 52c6869eab | |||
| 6dff9097a2 | |||
| 4ff211662a | |||
| 05eab51a0d | |||
| 604a4e7dbc | |||
| 80dca96ee8 | |||
| 7f97037190 | |||
| b658afd857 | |||
| 992ad97ad5 | |||
| 5af6cbae2c | |||
| 2abe792f36 | |||
| 1ff9bb8fdc | |||
| 5cb1039daf | |||
| 591c5dabf4 | |||
| 770fff287e | |||
| cea7a179ae | |||
| d80c40cfbf | |||
| 12e83374e9 | |||
| 98344d2e5e | |||
| 99dc1eec50 | |||
| 2a886576a6 | |||
| 919d005550 | |||
| 97abdaca5a | |||
| 9cc8b7c858 | |||
| 0726472b91 | |||
| 3cbe92d797 | |||
| 72a278c9ed | |||
| e567c8adce | |||
| dde8045109 | |||
| c922c4c383 | |||
| bc8907e90d | |||
| a8ba7786ae | |||
| c734e48ad0 | |||
| d30d0b29a9 | |||
| 2912defb97 | |||
| 69f8ac6b56 | |||
| e7441ff6e8 | |||
| 8a34158fa4 | |||
| 8d2a6d96f2 | |||
| bb50b677c7 | |||
| 0fde4b3b2e | |||
| ee9c109f07 | |||
| 1219423091 | |||
| cf00ab854f | |||
| c417dcb7e2 | |||
| 7ad711f554 | |||
| 2d7b0cf94d | |||
| d669c07e8a | |||
| 8bd52946b4 | |||
| 27e81637be | |||
| 5c67e27a30 | |||
| 59af9809fe | |||
| a564510c49 | |||
| 285b614927 | |||
| fd2d2c035e | |||
| dc71ec734d | |||
| 78981862be | |||
| 7f1253ff83 | |||
| e0265aed05 | |||
| 9d36d88a65 | |||
| 5dd5602229 | |||
| 126c4e9a06 | |||
| 5dbaf6ceb0 | |||
| 90de5659ea | |||
| 367e50edab | |||
| 42b8dafafe | |||
| 577aaf8ad6 | |||
| 07cdf0364c | |||
| 7f829f0159 | |||
| a918aa97d9 | |||
| 4fdecc9b85 | |||
| 7af25c785d | |||
| 4de39b205d | |||
| 2748a2e97f | |||
| 2926bbfe15 | |||
| 254c63763a | |||
| 2de834f1f4 | |||
| 7273eab80e | |||
| 13e79c777a | |||
| 8aa7d4b463 | |||
| 5251f1c9db | |||
| 82e923dfc8 | |||
| decf16b92c | |||
| b84d960a81 | |||
| 34cb305755 | |||
| ed85bfa915 | |||
| 06ef33ff5e | |||
| 57f121178c | |||
| 3b88ee623b | |||
| 8588625937 | |||
| 3417839726 | |||
| c1069052ae | |||
| ea17542e4b | |||
| c7d779fe88 | |||
| a70f3f12c5 | |||
| 90a31589bb | |||
| b48d9a3a82 | |||
| 0255311bbe | |||
| bd91519df9 | |||
| 58fe8b0cf1 | |||
| 064aa64f20 | |||
| d9f79853fb | |||
| 2e68ee5c8b | |||
| a9544ca890 | |||
| 9a549a853b | |||
| 2dad769a00 | |||
| 0ceb14dbf6 | |||
| bab1e26d9b | |||
| 9a91cc232c | |||
| 5a46cf1d48 | |||
| f1e241940b | |||
| 47b344ba12 | |||
| afbb06a72f | |||
| e336cd463f | |||
| 3a8315971e | |||
| 4ccfa98771 | |||
| 1db120bf06 | |||
| 262cf63956 | |||
| fe2ae4c6c3 | |||
| 16d9944dbb | |||
| e9956cc71e | |||
| 3af96e50bd | |||
| 59a85c1d75 | |||
| 4427149a38 | |||
| 20dee618ea | |||
| 37ebbb53be | |||
| ba019efaf1 | |||
| ce948fc512 | |||
| 2cd9e7fb55 | |||
| 1e2d151684 | |||
| ce5651f5fa | |||
| 20ba0bf4ed | |||
| ddc56c8a0d | |||
| c325ffd0f8 | |||
| e0da2764c9 | |||
| 6e88d9688b | |||
| bf898f10fb | |||
| c891999e1d | |||
| 938e287501 | |||
| edcfc32b1a | |||
| 904b211d98 | |||
| af08567f24 | |||
| 75ef658962 | |||
| fe2dd79838 | |||
| bbe7e6525d | |||
| ef20df719c | |||
| 68399601ce | |||
| aa637fd942 | |||
| 601c97c015 | |||
| 6b47052491 | |||
| 297da94319 | |||
| 64f101f534 | |||
| 45917f278a | |||
| ddd2759cec | |||
| 70d8903d3c | |||
| f66c7dc09c | |||
| ad29093ac1 | |||
| 82c6caef85 | |||
| 46ec72412a | |||
| ead09395d9 | |||
| 7106fc5304 | |||
| d16dcb9f19 | |||
| 1aaf34b0ed | |||
| 39a3b8922d | |||
| 9b78582475 | |||
| 3a84224b93 | |||
| 2592ba7399 | |||
| 1795e0a290 | |||
| c959f59581 | |||
| 2449723a1c | |||
| ae0e56e98d | |||
| 6efe521e44 | |||
| bccd21ac14 | |||
| 8449a65cdf | |||
| fc47562983 | |||
| 76900ae291 | |||
| ec55559ff1 | |||
| 3daa26e1f7 | |||
| 9ea8b6f659 | |||
| 387f2f0a94 | |||
| b0d95d02be | |||
| 3a98f01d31 | |||
| d305752749 | |||
| 2ba4b235fc | |||
| 6820c0a5d7 | |||
| 048883ad27 | |||
| 08e7ada242 | |||
| d3ddfa31f7 | |||
| 4b899a813e | |||
| 28610a9a42 | |||
| 15ee9a5cac | |||
| 58945a429f | |||
| 9f4111015e | |||
| 04b960b415 | |||
| 33267f2178 | |||
| 970e50d1f1 | |||
| d4199c2d08 | |||
| cf4ca7b6a8 | |||
| d8b335ce65 | |||
| 39a2934b05 | |||
| 7d1c720b84 | |||
| 51743461ee | |||
| 53cf5ca762 | |||
| b5ef42b0a1 | |||
| 0521ddd858 | |||
| b7bb3bfee2 | |||
| 4183044e96 | |||
| 27448bde20 | |||
| 87b9e8fbaf | |||
| 9d79859ba6 | |||
| 25bb55491a | |||
| 198cbacc3e | |||
| 3f842221f7 | |||
| 5d0183a9ed | |||
| 99df4d660b | |||
| 65eb528e2d | |||
| f2adfde1a8 | |||
| 1e915a2903 | |||
| 962eaa8a4b | |||
| e2dc3e9ff3 | |||
| f1bb8daaab | |||
| ec9d68960f | |||
| b0fcbebdae | |||
| 34f72ecf8f | |||
| 9d348319fd | |||
| c55fee69de | |||
| ce31cb072b | |||
| 6b91fc9c91 | |||
| e34f77ba0e | |||
| bc3b7401a1 | |||
| 85677eaf1a | |||
| 75d5e74059 | |||
| c4d15b3b95 | |||
| aa168ec2d6 | |||
| 4ae0efe887 | |||
| 86a57d8b56 | |||
| 9dda7485eb | |||
| 8b9670add9 | |||
| 978aebd79c | |||
| ac079f0f83 | |||
| e82e912151 | |||
| 5488ae5b89 | |||
| 15b875b116 | |||
| dedf835aa6 | |||
| e62b9c6009 | |||
| 53da778506 | |||
| 4360b2c815 | |||
| 1e15b1e0be | |||
| c618eba9a9 | |||
| 0bc50f7284 | |||
| 435f9113e8 | |||
| 8818c4785b | |||
| 2e003e5404 | |||
| e791a8ea07 | |||
| 2fb8eb755b | |||
| 1ef8378a30 | |||
| d031f958a9 | |||
| 9bbadac9dc | |||
| b012f77475 | |||
| 3cf36b1773 | |||
| 8f93c046a9 | |||
| 90af68901a | |||
| c17507b216 | |||
| b110b7c3f7 | |||
| 36431b3dcd | |||
| 609294deee | |||
| fffae9a741 | |||
| 598ce4bb5f | |||
| 212f6dc9e0 | |||
| cd05f1c3d7 | |||
| d7a0691c99 | |||
| 86346aa332 | |||
| b162b1fa34 | |||
| ea9f8b0ceb | |||
| 6210b9e746 | |||
| a778b410b9 | |||
| 8a674c8bc3 | |||
| aaf625c624 | |||
| d4079a3273 | |||
| 8d94fe3346 | |||
| ce510e55ae | |||
| 5419ff9a71 | |||
| ade437d625 | |||
| 87780a5b7e | |||
| f6f6f261ed | |||
| 65acc7c9ad | |||
| 31d95ac9e6 | |||
| 964d17d05a | |||
| 665c5992f0 | |||
| 5f52e0581d | |||
| 870e3a45ef | |||
| a5fe4a3694 | |||
| 838670ccbc | |||
| baf4cc225e | |||
| 93ac1605bd | |||
| c8a68001c1 | |||
| 244a22755c | |||
| 79c3ea82c7 | |||
| 4b92960975 | |||
| ef616ff25b | |||
| 7fb1a470ce | |||
| fc6b2d9193 | |||
| e5dc66e7e5 | |||
| bb01b76582 | |||
| f1aff0fd96 | |||
| 5656be5206 | |||
| 484ce8e488 | |||
| dcadefd133 | |||
| 28366677b0 | |||
| d65302742c | |||
| b2cf28efdd | |||
| 41e20bb6b7 | |||
| 6e670a2499 | |||
| 481b2186cb | |||
| 1bc1c0b14f | |||
| e9d27b9d2b | |||
| 828bbc407f | |||
| e50469d84e | |||
| d3a9b126a6 | |||
| 9eb185ec39 | |||
| fcf60e7f7c | |||
| 0ebee92f7d | |||
| 64b42bba2e | |||
| d297f9e032 | |||
| 30aabf1da9 | |||
| bfba18fdcb | |||
| eebdaa2f27 | |||
| c3c9c4cde5 | |||
| 640d5135df | |||
| cbbd20a687 | |||
| 1a2a27b988 | |||
| d819151020 | |||
| d089436546 | |||
| 289d604690 | |||
| 2979e0e964 | |||
| 5338f1cfbd | |||
| 214f18cbfd | |||
| 9b11609b63 | |||
| d476c2b613 | |||
| 590afebc0a | |||
| 02bd1af293 | |||
| 2fde82528d | |||
| 6c383e279f | |||
| 5c07477de4 | |||
| 146a284315 | |||
| a8faeeac73 | |||
| 69e385e4cd | |||
| 41b8dd2863 | |||
| 493dc8fcd5 | |||
| 87764445e8 | |||
| 0bb31e16c9 | |||
| 72c90abe36 | |||
| c4d8d33a60 | |||
| a267bca8fb | |||
| 32d2e78e3c | |||
| 555e70ebec | |||
| cd1b2aab46 | |||
| 43289103cb | |||
| 9edce23e76 | |||
| 756a8a35e3 | |||
| f3057c61a7 | |||
| 25345f08e7 | |||
| 2091e12e82 | |||
| 3eb000fa60 | |||
| 175669c61e | |||
| 3059b36118 | |||
| 35b1887e17 | |||
| 8f9b8a8550 | |||
| 174befe729 | |||
| e212b64823 | |||
| 3599b98dca | |||
| 991dc32a0b | |||
| a76efd4166 | |||
| 8a768baaaa | |||
| 997692b494 | |||
| 59ffec4e39 | |||
| 56d0ecc253 | |||
| e863746bd7 | |||
| d4dc7911eb | |||
| f561d3261a | |||
| 1bc5632771 | |||
| fdf8ee7015 | |||
| 5ec95086f2 | |||
| 26e4669316 | |||
| 4b488a2d28 | |||
| 6c352dca74 | |||
| 9d816694ba | |||
| 39ef35db0c | |||
| b8ed135183 | |||
| 6f750582dd | |||
| 5f93fbd471 | |||
| 0e2653b7dd | |||
| 47554b562d | |||
| 99427d649e | |||
| 7bc4589d4d | |||
| 9af586d4ac | |||
| d1c3be3251 | |||
| 87e68cac6c | |||
| 7d5a98409b | |||
| f28367bcfc | |||
| 14817e31f6 | |||
| fbdbd722b1 | |||
| b34102cd11 | |||
| 3c51cd6626 | |||
| b0b34236e3 | |||
| a502836002 | |||
| 09417d4b83 | |||
| 83ef2fa84c | |||
| e596a45e9f | |||
| 24e5000c37 | |||
| e3bcfa17f8 | |||
| 3b512676b7 | |||
| 928198bbfe | |||
| 1fb56f0ad2 | |||
| 9797f62cb8 | |||
| 4ddd87e773 | |||
| 7fd2e4d2db | |||
| 55c7d86205 | |||
| 737a28050c | |||
| 434ecdac6b | |||
| 709570afcc | |||
| b084b4faaf | |||
| d96ce23451 | |||
| 760a9d6d35 | |||
| 8e624cedb1 | |||
| 39cf269d6b | |||
| b9f83c7780 | |||
| 6a00b5a79e | |||
| 2ce674e3fd | |||
| 0fcc25d7c9 | |||
| 63bd0136fb | |||
| 80a2a934dd | |||
| e13976a3b3 | |||
| 5144330807 | |||
| bb29639183 | |||
| 4667cb9de9 | |||
| c34f3defe1 | |||
| eb0d742672 | |||
| d9b0a73787 | |||
| 4810879b2f | |||
| f4b6704aad | |||
| b1a31d3b30 | |||
| bf909db3f9 | |||
| 9c68be4d5e | |||
| d7956dd495 | |||
| 37a473e7d6 | |||
| 5a1c885e8f | |||
| 0b1136ad82 | |||
| 45af549897 | |||
| 97844603fc | |||
| c07b39e58b | |||
| 384c543ab9 | |||
| 592b13d7db | |||
| 6fdba3c02e | |||
| cbf758ead9 | |||
| d1ad778a64 | |||
| ce5ad296ae | |||
| 797e105786 | |||
| d17d80747e | |||
| 55ea207a55 | |||
| 6384d1e5a3 | |||
| aba01cdace | |||
| 517b7a14b4 | |||
| 2927de7cf9 | |||
| 6471ba70e4 | |||
| 9f9de01c51 | |||
| 3662decb8b | |||
| 583bcfb3c7 | |||
| c45e3fa4d5 | |||
| 24cbcef620 | |||
| e2a520ff49 | |||
| a5e3317e28 | |||
| 5638c4ba87 | |||
| bf7a128142 | |||
| c5243cd4d5 | |||
| db868ed29d | |||
| 450c7d80f8 | |||
| abbb001975 | |||
| f35d83ae48 | |||
| a2315dc95e | |||
| 4e2feb6fbc | |||
| 13602b6769 | |||
| 85dba25246 | |||
| 66432672b3 | |||
| e6d96e4c18 | |||
| 9812305bb9 | |||
| f680a63a1f | |||
| 781d63cb2a | |||
| 9ff04ee3d8 | |||
| 5d85a24977 | |||
| 1e51fca0b0 | |||
| 5537d53f9a | |||
| 50a4170541 | |||
| 3a8255bda1 | |||
| a617846f0f | |||
| 5772588c29 | |||
| 9d0dc45f74 | |||
| c6aefbc9a0 | |||
| dbbafb0cc9 | |||
| 6e8272f78f | |||
| cbf73ef29e | |||
| db6d3b495b | |||
| baf8a63121 | |||
| fc4a76ee50 | |||
| 2117d1d035 | |||
| 0a70e0b7b6 | |||
| 64ffac5671 | |||
| ac384e8a9c | |||
| f97c8222c7 | |||
| 728289ee3a | |||
| 5faa16f9ee | |||
| 4e608b116a | |||
| 521b49166e | |||
| 8f32decf2d | |||
| 0d51f83d2d | |||
| 78c6a68db9 | |||
| 2949ab73e2 | |||
| 223741820d | |||
| 4b57821f52 | |||
| 74271a479f | |||
| c377177108 | |||
| 84eb729bd4 | |||
| 14aea365c5 | |||
| 97cb3fa5a5 | |||
| b5368db704 | |||
| 8c442b72f3 | |||
| f8f6791d39 | |||
| 0c09f077aa | |||
| af2831d7b6 | |||
| 64d5d4aec7 | |||
| 619a6b2adb | |||
| 33a26bc0cf | |||
| b445a7c4d3 | |||
| e6892d0c3e | |||
| 33e9a88b56 | |||
| df00a2251e | |||
| 92c44c8abe | |||
| 8e4f7bbd3e | |||
| a40217cf07 | |||
| e586fda5f2 | |||
| a58564ff88 | |||
| 89885b9fb9 | |||
| 5c7d977ae0 | |||
| 2cd3ee9698 | |||
| dd3080e018 | |||
| 5915e8e86a | |||
| 3c67c06654 | |||
| 76232ca573 | |||
| 5235e82bda | |||
| 10f0713257 | |||
| e9c7970ea4 | |||
| 1a6ac4aeb1 | |||
| f633bdddf0 | |||
| de0b91d157 | |||
| 2e77e498f5 | |||
| 4ac67eb1f9 | |||
| 2b536de37f | |||
| 2ffa92ba1b | |||
| 6ecddd8388 | |||
| bd2772ea4c | |||
| 92bf79d53b | |||
| eebe0eeb71 | |||
| 1068eaa0b9 | |||
| faac3e7d7c | |||
| dab4340207 | |||
| fd2567748f | |||
| c2daedbd11 | |||
| 7c604beb73 | |||
| 8c42aea827 | |||
| cf1bfdfb61 | |||
| 75b26513e1 | |||
| 6c09a77a97 | |||
| 67389c39fb | |||
| c326103e6e | |||
| c2120a16da | |||
| 258ad4352e | |||
| 435d3958f4 | |||
| b0408ef5c6 | |||
| 1c41b0bc2f | |||
| aa827f3042 | |||
| f44f5964bb | |||
| 91ba93bd7a | |||
| 0abe4cefb4 | |||
| bccd460f3b | |||
| d1023004e1 | |||
| 04a5f9cb04 | |||
| 9818e2b550 | |||
| fe43e3b89d | |||
| e1f1ae041f | |||
| 5bcf26e324 | |||
| 5f47a8149f | |||
| 00b662b53a | |||
| faf519ab1b | |||
| fce73f6f17 | |||
| 887890baf5 | |||
| c66b24feeb | |||
| 84c6f147ad | |||
| 0cdb0daa8c | |||
| eee702f299 | |||
| df65247325 | |||
| 1a174e75d3 | |||
| 9e1fd3454f | |||
| 3b1603cadf | |||
| 8803bac708 | |||
| 3a01eaa4a6 | |||
| 9f84c1c448 | |||
| dda0390156 | |||
| c74509dd5f | |||
| f61bbb2ff4 | |||
| e7f60161a3 | |||
| ebec4fbc24 | |||
| 1d4105ae3d | |||
| 586d49f0c3 | |||
| 5b0fab0697 | |||
| 2b3359dff3 | |||
| 63203aa14c | |||
| 716a8329c2 | |||
| dab0aec85e | |||
| 6ea8e2525a | |||
| 1f1ab017c0 | |||
| b6912ef95e | |||
| db54dca694 | |||
| 0e751b983c | |||
| 997b20a975 | |||
| 386f9c42c2 | |||
| cfae06db65 | |||
| 44260b7b5c | |||
| 13063b957f | |||
| ee05e12480 | |||
| 5538545fb0 | |||
| bc1167c2c5 | |||
| c57656e4c3 | |||
| 264400a984 | |||
| 408db4eb1d | |||
| 9347f223ef | |||
| 518aa30c9c | |||
| 6bbf1f9355 | |||
| b221e4d445 | |||
| 580fccbfca | |||
| 045916efcc | |||
| 4f92482294 | |||
| 2f055a75a0 | |||
| f0621207e3 | |||
| d657bc4e3d | |||
| a1fd07b27c | |||
| 52219c5f3f | |||
| 1a66461e07 | |||
| d20df12168 | |||
| 668b429615 | |||
| 7db528be39 | |||
| 60f760ee49 | |||
| 884aaab751 | |||
| e968560ea4 | |||
| 07caaa96e4 | |||
| e8a679c280 | |||
| bc885f1d08 | |||
| f2f051d6de | |||
| 49a0bfccba | |||
| 0c1e60894f | |||
| ace87ad7bb | |||
| 50f0097843 | |||
| 32a9466277 | |||
| 1ee3407946 | |||
| f1120d7aa9 | |||
| 2e7d6b2f99 | |||
| dfef929187 | |||
| e78d9ad592 | |||
| 9f2948f595 | |||
| 198da910ed | |||
| 5f1bf9d9d6 | |||
| 798c4aef9a | |||
| f80f5b3bda | |||
| cbb07b0d67 | |||
| 7cc9921615 | |||
| 7555fe065e | |||
| d977f4278e | |||
| 870e3ca893 | |||
| 213acaee3b | |||
| 29296ec998 | |||
| 58381496a2 | |||
| 5981e42aed | |||
| 3c9165d295 | |||
| 60d0ef93ac | |||
| f45d5b0066 | |||
| b71306480f | |||
| 0c7771ccc5 | |||
| dc9df0a79a | |||
| 17cd49fbdc | |||
| ad273adb78 | |||
| 150e7daf2d | |||
| b004155e8f | |||
| 92eed3b33b | |||
| fe7b77198c | |||
| f51b775698 | |||
| 939dd5cb31 | |||
| adcbe13ecd | |||
| 8976e53998 | |||
| 97dda6a4bb | |||
| 9e395eb883 | |||
| 60da59623e | |||
| 9752ea9ac3 | |||
| 279693078a | |||
| 19b93045a4 | |||
| 5231a09820 | |||
| ab952e6103 | |||
| a418771c04 | |||
| b41590ce38 | |||
| c7dde9499f | |||
| 528cbf62ec | |||
| 1be4b8bb5d | |||
| c832fc9917 | |||
| 4797a94689 | |||
| 6948903084 | |||
| 94164611ae | |||
| ae298e8902 | |||
| 3d8771ecb0 | |||
| 28db264e90 | |||
| 6af9fa4b81 | |||
| 60b4d05860 | |||
| 7b93839ed1 | |||
| fdb11d7c06 | |||
| 5651847877 | |||
| e1442290b6 | |||
| c45b18cc75 | |||
| bb2ad77987 | |||
| 68b1ffec19 | |||
| bc2bb22673 | |||
| 83d707fc4b | |||
| 175b32e56c | |||
| 97b4a6553b | |||
| 4ade30e681 | |||
| 4e03b4f191 | |||
| bfe1d1d4ca | |||
| 8918de85fd | |||
| 5e237aecae | |||
| 13291ad481 | |||
| a47ee86bee | |||
| 62d703f967 | |||
| b2c196e5c7 | |||
| 4be6a54bc0 | |||
| 8ce8476547 | |||
| d82caf6bd4 | |||
| 8ea1e302c3 | |||
| a8799efa94 | |||
| 0cfac4e021 | |||
| f6c9642d72 | |||
| 5a07f9ddee | |||
| 9db75e91ac | |||
| f288e00c37 | |||
| c9edd31993 | |||
| 5a7780ab5f | |||
| ac0fba99ad | |||
| 6f724a113c | |||
| 327cd4cb87 | |||
| 25de3a2590 | |||
| 06208a703a | |||
| bdd265a1b1 | |||
| 56afba6606 | |||
| 2c9df7aad1 | |||
| d65bbf2113 | |||
| b8bfc9b732 | |||
| cec3bad373 | |||
| 9312e3c7de | |||
| 43e7435c41 | |||
| f34f5e41a4 | |||
| 47a70a536b | |||
| bbeddfe522 | |||
| 28220310a5 | |||
| 3e82a0a259 | |||
| c860ad23a0 | |||
| 4e36dd2943 | |||
| 13d77f1557 | |||
| cc619f6b53 | |||
| d425794665 | |||
| 32da1c8d58 | |||
| 830be1035b | |||
| e9e45d0e29 | |||
| d3ca265a25 | |||
| 244f0ffaf1 | |||
| 73f5c47fe2 | |||
| e8b9600ddb | |||
| d2c813ffac | |||
| 8e699f8243 | |||
| 3f6cdc829b | |||
| 1fca248d4c | |||
| c5c9ee92ac | |||
| 7f1fcc9cfc | |||
| 9de45c3be4 | |||
| 144a881ae5 | |||
| 4566690617 | |||
| e8fe1590b6 | |||
| 25f4fd5a19 | |||
| 7b8c126aa1 | |||
| 86b3ff3099 | |||
| fa9df4dc5e | |||
| fbd22e7b94 | |||
| e35411d90f | |||
| be15e48074 | |||
| 99081ea2a0 | |||
| 2be1218aa3 | |||
| c47aebdd2a | |||
| f4d1632506 | |||
| 8bfe4374de | |||
| 4afe02cb21 | |||
| 115b967e5b | |||
| ea4524024a | |||
| 4ff6cd9105 | |||
| 96c17d8292 | |||
| bc6faaffc4 | |||
| 51e9839237 | |||
| 6115631746 | |||
| ee005fbc8e | |||
| e27d42935c | |||
| 9c99d65716 | |||
| 5b9469eed3 | |||
| 6805ac915b | |||
| 7148cf99f7 | |||
| 67a3fb8bf2 | |||
| 933b61f99f | |||
| 6c5c14f35f | |||
| 6a441d5013 | |||
| 6b46465c77 | |||
| 75388caeed | |||
| 2546930a1a | |||
| 135e29a3bb | |||
| 3b65a58f59 | |||
| 49cb931572 | |||
| b7176d2204 | |||
| 5bf7d372f6 | |||
| 073775e461 | |||
| fbf8f3dc68 | |||
| e8c8cc550b | |||
| 87c3790fa8 | |||
| 0d9dcb2f4f | |||
| 6188185b22 | |||
| f762bd5e25 | |||
| b676264fca | |||
| 3640c3b66a | |||
| 5087d02fba | |||
| 2aa4340551 | |||
| 3b34895ae6 | |||
| 91cc84c4e6 | |||
| 797e53c5ba | |||
| c714a12ad7 | |||
| 08ce9b09ec | |||
| 3152152ed9 | |||
| 544fea51b0 | |||
| 08ca9f9378 | |||
| 978f68b744 | |||
| 680896e4c4 | |||
| 975627af2e | |||
| b208102b98 | |||
| 88a063434c | |||
| 58cc108c0c | |||
| 1f62247c7e | |||
| 6415d1a6a5 | |||
| 50b37f1366 | |||
| a7b6e35467 | |||
| 37d83a4e2e | |||
| 926b08c197 | |||
| aff41d0b08 | |||
| 5d9c968614 | |||
| c020cf05e1 | |||
| 09e8d85b1e | |||
| 4d3eb134a2 | |||
| b92df85893 | |||
| 545025ed2b | |||
| 3158962506 | |||
| c314f74de6 | |||
| 65615385e7 | |||
| 727f35b35b | |||
| 07ddf7e87b |
@@ -0,0 +1,8 @@
|
||||
# Text files use LF line endings in this repository
|
||||
* text=auto
|
||||
|
||||
# Except the dependencies, which we leave alone
|
||||
vendor/** -text=auto
|
||||
|
||||
# Diffs on these files are meaningless
|
||||
*.svg -diff
|
||||
+12
-3
@@ -1,10 +1,19 @@
|
||||
syncthing
|
||||
/syncthing
|
||||
/stdiscosrv
|
||||
syncthing.exe
|
||||
stdiscosrv.exe
|
||||
*.tar.gz
|
||||
*.zip
|
||||
*.asc
|
||||
*.sublime*
|
||||
discosrv
|
||||
*.deb
|
||||
.jshintrc
|
||||
coverage.out
|
||||
files/pidx
|
||||
bin
|
||||
perfstats*.csv
|
||||
coverage.xml
|
||||
syncthing.sig
|
||||
RELEASE
|
||||
deb
|
||||
lib/auto/gui.files.go
|
||||
snapcraft.yaml
|
||||
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- tip
|
||||
|
||||
install:
|
||||
- export PATH=$PATH:$HOME/gopath/bin
|
||||
- ./build.sh setup
|
||||
- go get code.google.com/p/go.tools/cmd/cover
|
||||
- go get github.com/mattn/goveralls
|
||||
|
||||
script:
|
||||
- ./build.sh test-cov
|
||||
|
||||
after_success:
|
||||
- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/syncthing -repotoken="$COVERALS_TOKEN"
|
||||
|
||||
env:
|
||||
global:
|
||||
secure: "zEV2h2XtKHNLVdXJjM4LA/VjMfLVydm6goF+ARit+nOSGxGoH7f7jIdzJzhxgh7shKG93q61eLO1Tug+WBMYB2EpBuYnTB5AIMYhCDwNI8C4uBV6c3brHfcrie7MASNao8TID2QScASKNFFWvjv/i1Ccn5ztxdcQuhSsNjGZp8A="
|
||||
@@ -0,0 +1,110 @@
|
||||
# This is the official list of Syncthing authors for copyright purposes.
|
||||
# The format is:
|
||||
#
|
||||
# Name Name Name (nickname) <email1@example.com> <email2@example.com>
|
||||
#
|
||||
# The NICKS list is auto generated from this file.
|
||||
|
||||
Aaron Bieber (qbit) <qbit@deftly.net>
|
||||
Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
|
||||
Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
|
||||
Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
|
||||
Alexander Graf (alex2108) <register-github@alex-graf.de>
|
||||
Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
|
||||
Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
|
||||
Andrew Dunham (andrew-d) <andrew@du.nham.ca>
|
||||
Andrey D (scienmind) <scintertech@cryptolab.net>
|
||||
Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
|
||||
Antony Male (canton7) <antony.male@gmail.com>
|
||||
Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
|
||||
Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
|
||||
Bart De Vries (mogwa1) <devriesb@gmail.com>
|
||||
Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
|
||||
Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
|
||||
Ben Sidhom (bsidhom) <bsidhom@gmail.com>
|
||||
Benny Ng (tpng) <benny.tpng@gmail.com>
|
||||
Brandon Philips (philips) <brandon@ifup.org>
|
||||
Brendan Long (brendanlong) <self@brendanlong.com>
|
||||
Brian R. Becker (brbecker) <brbecker@gmail.com>
|
||||
Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
|
||||
Carsten Hagemann (Moter8) <moter8@gmail.com>
|
||||
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
|
||||
Cedric Staniewski (xduugu) <cedric@gmx.ca>
|
||||
Chris Howie (cdhowie) <me@chrishowie.com>
|
||||
Chris Joel (cdata) <chris@scriptolo.gy>
|
||||
Colin Kennedy (moshen) <moshen.colin@gmail.com>
|
||||
Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
|
||||
Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
|
||||
Daniel Martí (mvdan) <mvdan@mvdan.cc>
|
||||
David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
|
||||
Denis A. (dva) <denisva@gmail.com>
|
||||
Dennis Wilson (snnd) <dw@risu.io>
|
||||
Dominik Heidler (asdil12) <dominik@heidler.eu>
|
||||
Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
|
||||
Emil Hessman (ceh) <emil@hessman.se>
|
||||
Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
|
||||
Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
|
||||
Felix Ableitner (Nutomic) <me@nutomic.com>
|
||||
Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
|
||||
Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
|
||||
Frank Isemann (fti7) <frank@isemann.name>
|
||||
Gilli Sigurdsson (gillisig) <gilli@vx.is>
|
||||
Heiko Zuerker (Smiley73) <heiko@zuerker.org>
|
||||
Jaakko Hannikainen (jgke) <jgke@jgke.fi>
|
||||
Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
|
||||
Jake Peterson (acogdev) <jake@acogdev.com>
|
||||
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
|
||||
James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
|
||||
Jaroslav Malec (dzarda) <dzardacz@gmail.com>
|
||||
Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
|
||||
Jochen Voss (seehuhn) <voss@seehuhn.de>
|
||||
Johan Vromans (sciurius) <jvromans@squirrel.nl>
|
||||
Karol Różycki (krozycki) <rozycki.karol@gmail.com>
|
||||
Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
|
||||
Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
|
||||
Kevin Allen (ironmig) <kma1660@gmail.com>
|
||||
Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
|
||||
Kurt Fitzner (Kudalufi) <kurt@va1der.ca>
|
||||
Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
|
||||
Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
|
||||
Leo Arias (elopio) <yo@elopio.net>
|
||||
Lode Hoste (Zillode) <zillode@zillode.be>
|
||||
Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
|
||||
Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
|
||||
Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
|
||||
Marc Pujol (kilburn) <kilburn@la3.org>
|
||||
Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
|
||||
Mark Pulford (mpx) <mark@kyne.com.au>
|
||||
Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
|
||||
Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
|
||||
Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
|
||||
Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
|
||||
Michael Ploujnikov (plouj) <ploujj@gmail.com>
|
||||
Michael Tilli (pyfisch) <pyfisch@gmail.com>
|
||||
Nate Morrison (nrm21) <natemorrison@gmail.com>
|
||||
Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
|
||||
Peter Hoeg (peterhoeg) <peter@speartail.com>
|
||||
Philippe Schommers (filoozoom) <philippe@schommers.be>
|
||||
Phill Luby (pluby) <phill.luby@newredo.com>
|
||||
Piotr Bejda (piobpl) <piotrb10@gmail.com>
|
||||
Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
|
||||
Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
|
||||
Scott Klupfel (kluppy) <kluppy@going2blue.com>
|
||||
Sergey Mishin (ralder) <ralder@yandex.ru>
|
||||
Simon Frei (imsodin) <freisim93@gmail.com>
|
||||
Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
|
||||
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
|
||||
Tim Abell (timabell) <tim@timwise.co.uk>
|
||||
Tim Howes (timhowes) <timhowes@berkeley.edu>
|
||||
Tobias Nygren (tnn2) <tnn@nygren.pp.se>
|
||||
Tomas Cerveny (kozec) <kozec@kozec.com>
|
||||
Tully Robinson (tojrobinson) <tully@tojr.org>
|
||||
Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
|
||||
Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
|
||||
Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
|
||||
Victor Buinsky (buinsky) <vix_booja@tut.by>
|
||||
Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
|
||||
William A. Kennington III (wkennington) <william@wkennington.com>
|
||||
Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de>
|
||||
Xavier O. (damajor) <damajor@gmail.com>
|
||||
Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
## Conduct
|
||||
|
||||
* We are committed to providing a friendly, safe and welcoming
|
||||
environment for all, regardless of gender, sexual orientation,
|
||||
disability, ethnicity, religion, or similar personal characteristic.
|
||||
|
||||
* On IRC, please avoid using overtly sexual nicknames or other nicknames
|
||||
that might detract from a friendly, safe and welcoming environment for
|
||||
all.
|
||||
|
||||
* Please be kind and courteous. There's no need to be mean or rude.
|
||||
|
||||
* Respect that people have differences of opinion and that every design
|
||||
or implementation choice carries a trade-off and numerous costs. There
|
||||
is seldom a right answer.
|
||||
|
||||
* Please keep unstructured critique to a minimum. If you have solid
|
||||
ideas you want to experiment with, make a fork and see how it works.
|
||||
|
||||
* We will exclude you from interaction if you insult, demean or harass
|
||||
anyone. That is not welcome behaviour. We interpret the term
|
||||
"harassment" as including the definition in the <a
|
||||
href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
|
||||
if you have any lack of clarity about what might be included in that
|
||||
concept, please read their definition. In particular, we don't
|
||||
tolerate behavior that excludes people in socially marginalized
|
||||
groups.
|
||||
|
||||
* Private harassment is also unacceptable. No matter who you are, if you
|
||||
feel you have been or are being harassed or made uncomfortable by a
|
||||
community member, please contact one of the channel ops or any of the
|
||||
Syncthing core team immediately. Whether you're a regular contributor
|
||||
or a newcomer, we care about making this community a safe place for
|
||||
you and we've got your back.
|
||||
|
||||
* Likewise any spamming, trolling, flaming, baiting or other
|
||||
attention-stealing behaviour is not welcome.
|
||||
|
||||
## Moderation
|
||||
|
||||
These are the policies for upholding our community's standards of
|
||||
conduct in our communication channels, most notably in Syncthing-related
|
||||
IRC channels and on the web forum.
|
||||
|
||||
1. Remarks that violate the Syncthing standards of conduct, including
|
||||
hateful, hurtful, oppressive, or exclusionary remarks, are not
|
||||
allowed. (Cursing is allowed, but never targeting another user, and
|
||||
never in a hateful manner.)
|
||||
|
||||
2. Remarks that moderators find inappropriate, whether listed in the
|
||||
code of conduct or not, are also not allowed.
|
||||
|
||||
3. Moderators will first respond to such remarks with a warning.
|
||||
|
||||
4. If the warning is unheeded, the user will be "kicked," i.e., kicked
|
||||
out of the communication channel to cool off.
|
||||
|
||||
5. If the user comes back and continues to make trouble, they will be
|
||||
banned, i.e., indefinitely excluded.
|
||||
|
||||
6. Moderators may choose at their discretion to un-ban the user if it
|
||||
was a first offense and they offer the offended party a genuine
|
||||
apology.
|
||||
|
||||
7. If a moderator bans someone and you think it was unjustified, please
|
||||
take it up with that moderator, or with a different moderator, **in
|
||||
private**. Complaints about bans in-channel are not allowed.
|
||||
|
||||
8. Moderators are held to a higher standard than other community
|
||||
members. If a moderator creates an inappropriate situation, they
|
||||
should expect less leeway than others.
|
||||
|
||||
In the Syncthing community we strive to go the extra step to look out
|
||||
for each other. Don't just aim to be technically unimpeachable, try to
|
||||
be your best self. In particular, avoid flirting with offensive or
|
||||
sensitive issues, particularly if they're off-topic; this all too
|
||||
often leads to unnecessary fights, hurt feelings, and damaged trust;
|
||||
worse, it can drive people away from the community entirely.
|
||||
|
||||
And if someone takes issue with something you said or did, resist the
|
||||
urge to be defensive. Just stop doing what it was they complained about
|
||||
and apologize. Even if you feel you were misinterpreted or unfairly
|
||||
accused, chances are good there was something you could've communicated
|
||||
better — remember that it's your responsibility to make your fellow
|
||||
community members comfortable. Everyone wants to get along and we are
|
||||
all here first and foremost because we want to talk about cool
|
||||
technology. You will find that people will be eager to assume good
|
||||
intent and forgive as long as you earn their trust.
|
||||
|
||||
*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
|
||||
|
||||
*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*
|
||||
+55
-48
@@ -1,56 +1,63 @@
|
||||
Please do contribute! If you want to contribute but are unsure where to
|
||||
start, the [Contributions Needed
|
||||
topic](http://discourse.syncthing.net/t/contributions-needed/49)
|
||||
lists areas in need of attention.
|
||||
## Reporting Bugs
|
||||
|
||||
Please file bugs in the [Github Issue
|
||||
Tracker](https://github.com/syncthing/syncthing/issues). Include at
|
||||
least the following:
|
||||
|
||||
- What happened
|
||||
|
||||
- What did you expect to happen instead of what *did* happen, if it's
|
||||
not crazy obvious
|
||||
|
||||
- What operating system, operating system version and version of
|
||||
Syncthing you are running
|
||||
|
||||
- The same for other connected devices, where relevant
|
||||
|
||||
- Screenshot if the issue concerns something visible in the GUI
|
||||
|
||||
- Console log entries, where possible and relevant
|
||||
|
||||
If you're not sure whether something is relevant, erring on the side of
|
||||
too much information will never get you yelled at. :)
|
||||
|
||||
## Contributing Translations
|
||||
|
||||
All translations are done via
|
||||
[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
|
||||
wish to contribute to a translation, just head over there and sign up.
|
||||
Before every release, the language resources are updated from the
|
||||
latest info on Transifex.
|
||||
|
||||
## Contributing Code
|
||||
|
||||
Every contribution is welcome. If you want to contribute but are unsure
|
||||
where to start, any open issues are fair game! See the [Contribution
|
||||
Guidelines](https://docs.syncthing.net/dev/contributing.html) for the full
|
||||
story on committing code.
|
||||
|
||||
## Contributing Documentation
|
||||
|
||||
Updates to the [documentation site](https://docs.syncthing.net/) can be
|
||||
made as pull requests on the [documentation
|
||||
repository](https://github.com/syncthing/docs).
|
||||
|
||||
## Licensing
|
||||
|
||||
All contributions are made under the same MIT License as the rest of the
|
||||
project, except documentation which is licensed under the Creative
|
||||
Commons Attribution 4.0 International License. You retain the copyright
|
||||
to code you have written.
|
||||
All contributions are made available under the same license as the already
|
||||
existing material being contributed to. For most of the project and unless
|
||||
otherwise stated this means MPLv2, but there are exceptions:
|
||||
|
||||
When accepting your first contribution, the maintainer of the project
|
||||
will ensure that you are added to the CONTRIBUTORS file. You are welcome
|
||||
to add yourself as a separate commit in your first pull request.
|
||||
- Certain commands (under cmd/...) may have a separate license, indicated by
|
||||
the presence of a LICENSE file in the corresponding directory.
|
||||
|
||||
## Building
|
||||
- The documentation (man/...) is licensed under the Creative Commons
|
||||
Attribution 4.0 International License.
|
||||
|
||||
[See the
|
||||
documentation](http://discourse.syncthing.net/t/building-syncthing/44)
|
||||
- Projects under vendor/... are copyright by and licensed from their
|
||||
respective original authors. Contributions should be made to the original
|
||||
project, not here.
|
||||
|
||||
## Branches
|
||||
|
||||
- `master` is the main branch containing good code that will end up in
|
||||
the next release. You should base your work on it. It won't ever be
|
||||
rebased or force-pushed to.
|
||||
|
||||
- `vx.y` branches exist to make patch releases on otherwise obsolete
|
||||
minor releases. Should only contain fixes cherry picked from master.
|
||||
Don't base any work on them.
|
||||
|
||||
- Other branches are probably topic branches and may be subject to
|
||||
rebasing. Don't base any work on them unless you specifically know
|
||||
otherwise.
|
||||
|
||||
## Tags
|
||||
|
||||
All releases are tagged semver style as `vx.y.z`. Release tags are
|
||||
signed by GPG key BCE524C7.
|
||||
|
||||
## Tests
|
||||
|
||||
Yes please!
|
||||
|
||||
## Style
|
||||
|
||||
`go fmt`
|
||||
|
||||
## Documentation
|
||||
|
||||
[Over here!](http://discourse.syncthing.net/category/documentation)
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
Regardless of the license in effect, you retain the copyright to your
|
||||
contribution.
|
||||
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
Aaron Bieber <qbit@deftly.net>
|
||||
Andrew Dunham <andrew@du.nham.ca>
|
||||
Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com>
|
||||
Ben Sidhom <bsidhom@gmail.com>
|
||||
Brandon Philips <brandon@ifup.org>
|
||||
James Patterson <jamespatterson@operamail.com>
|
||||
Jens Diemer <github.com@jensdiemer.de>
|
||||
Philippe Schommers <philippe@schommers.be>
|
||||
Ryan Sullivan <kayoticsully@gmail.com>
|
||||
Veeti Paananen <veeti.paananen@rojekti.fi>
|
||||
Generated
-65
@@ -1,65 +0,0 @@
|
||||
{
|
||||
"ImportPath": "github.com/calmh/syncthing",
|
||||
"GoVersion": "go1.3",
|
||||
"Packages": [
|
||||
"./cmd/syncthing",
|
||||
"./cmd/assets",
|
||||
"./discover/cmd/discosrv"
|
||||
],
|
||||
"Deps": [
|
||||
{
|
||||
"ImportPath": "bitbucket.org/kardianos/osext",
|
||||
"Comment": "null-13",
|
||||
"Rev": "5d3ddcf53a508cc2f7404eaebf546ef2cb5cdb6e"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.crypto/bcrypt",
|
||||
"Comment": "null-212",
|
||||
"Rev": "1064b89a6fb591df0dd65422295b8498916b092f"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.crypto/blowfish",
|
||||
"Comment": "null-212",
|
||||
"Rev": "1064b89a6fb591df0dd65422295b8498916b092f"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.text/transform",
|
||||
"Comment": "null-87",
|
||||
"Rev": "c59e4f2f93824f81213799e64c3eead7be24660a"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.text/unicode/norm",
|
||||
"Comment": "null-87",
|
||||
"Rev": "c59e4f2f93824f81213799e64c3eead7be24660a"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/snappy-go/snappy",
|
||||
"Comment": "null-15",
|
||||
"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/golang/groupcache/lru",
|
||||
"Rev": "a531d51b7f9f3dd13c1c2b50d42d739b70442dbb"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/juju/ratelimit",
|
||||
"Rev": "f9f36d11773655c0485207f0ad30dc2655f69d56"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
|
||||
"Rev": "e1f2d2bdccd7c62f4d4a29aaf081bf1fc4404f91"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/vitrun/qart/coding",
|
||||
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/vitrun/qart/gf256",
|
||||
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/vitrun/qart/qr",
|
||||
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
|
||||
}
|
||||
]
|
||||
}
|
||||
Generated
-5
@@ -1,5 +0,0 @@
|
||||
This directory tree is generated automatically by godep.
|
||||
|
||||
Please do not edit.
|
||||
|
||||
See https://github.com/tools/godep for more information.
|
||||
Generated
-2
@@ -1,2 +0,0 @@
|
||||
/pkg
|
||||
/bin
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
Copyright (c) 2012 Daniel Theophanes
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any damages
|
||||
arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not
|
||||
claim that you wrote the original software. If you use this software
|
||||
in a product, an acknowledgment in the product documentation would be
|
||||
appreciated but is not required.
|
||||
|
||||
2. Altered source versions must be plainly marked as such, and must not be
|
||||
misrepresented as being the original software.
|
||||
|
||||
3. This notice may not be removed or altered from any source
|
||||
distribution.
|
||||
-32
@@ -1,32 +0,0 @@
|
||||
// Copyright 2012 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Extensions to the standard "os" package.
|
||||
package osext
|
||||
|
||||
import "path/filepath"
|
||||
|
||||
// Executable returns an absolute path that can be used to
|
||||
// re-invoke the current program.
|
||||
// It may not be valid after the current program exits.
|
||||
func Executable() (string, error) {
|
||||
p, err := executable()
|
||||
return filepath.Clean(p), err
|
||||
}
|
||||
|
||||
// Returns same path as Executable, returns just the folder
|
||||
// path. Excludes the executable name.
|
||||
func ExecutableFolder() (string, error) {
|
||||
p, err := Executable()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
folder, _ := filepath.Split(p)
|
||||
return folder, nil
|
||||
}
|
||||
|
||||
// Depricated. Same as Executable().
|
||||
func GetExePath() (exePath string, err error) {
|
||||
return Executable()
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
// Copyright 2012 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package osext
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func executable() (string, error) {
|
||||
f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
return syscall.Fd2path(int(f.Fd()))
|
||||
}
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
// Copyright 2012 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build linux netbsd openbsd
|
||||
|
||||
package osext
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func executable() (string, error) {
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
return os.Readlink("/proc/self/exe")
|
||||
case "netbsd":
|
||||
return os.Readlink("/proc/curproc/exe")
|
||||
case "openbsd":
|
||||
return os.Readlink("/proc/curproc/file")
|
||||
}
|
||||
return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
|
||||
}
|
||||
-79
@@ -1,79 +0,0 @@
|
||||
// Copyright 2012 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build darwin freebsd
|
||||
|
||||
package osext
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var initCwd, initCwdErr = os.Getwd()
|
||||
|
||||
func executable() (string, error) {
|
||||
var mib [4]int32
|
||||
switch runtime.GOOS {
|
||||
case "freebsd":
|
||||
mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
|
||||
case "darwin":
|
||||
mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
|
||||
}
|
||||
|
||||
n := uintptr(0)
|
||||
// Get length.
|
||||
_, _, errNum := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
|
||||
if errNum != 0 {
|
||||
return "", errNum
|
||||
}
|
||||
if n == 0 { // This shouldn't happen.
|
||||
return "", nil
|
||||
}
|
||||
buf := make([]byte, n)
|
||||
_, _, errNum = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
|
||||
if errNum != 0 {
|
||||
return "", errNum
|
||||
}
|
||||
if n == 0 { // This shouldn't happen.
|
||||
return "", nil
|
||||
}
|
||||
for i, v := range buf {
|
||||
if v == 0 {
|
||||
buf = buf[:i]
|
||||
break
|
||||
}
|
||||
}
|
||||
var err error
|
||||
execPath := string(buf)
|
||||
// execPath will not be empty due to above checks.
|
||||
// Try to get the absolute path if the execPath is not rooted.
|
||||
if execPath[0] != '/' {
|
||||
execPath, err = getAbs(execPath)
|
||||
if err != nil {
|
||||
return execPath, err
|
||||
}
|
||||
}
|
||||
// For darwin KERN_PROCARGS may return the path to a symlink rather than the
|
||||
// actual executable.
|
||||
if runtime.GOOS == "darwin" {
|
||||
if execPath, err = filepath.EvalSymlinks(execPath); err != nil {
|
||||
return execPath, err
|
||||
}
|
||||
}
|
||||
return execPath, nil
|
||||
}
|
||||
|
||||
func getAbs(execPath string) (string, error) {
|
||||
if initCwdErr != nil {
|
||||
return execPath, initCwdErr
|
||||
}
|
||||
// The execPath may begin with a "../" or a "./" so clean it first.
|
||||
// Join the two paths, trailing and starting slashes undetermined, so use
|
||||
// the generic Join function.
|
||||
return filepath.Join(initCwd, filepath.Clean(execPath)), nil
|
||||
}
|
||||
-79
@@ -1,79 +0,0 @@
|
||||
// Copyright 2012 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build darwin linux freebsd netbsd windows
|
||||
|
||||
package osext
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
oexec "os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
|
||||
|
||||
func TestExecPath(t *testing.T) {
|
||||
ep, err := Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("ExecPath failed: %v", err)
|
||||
}
|
||||
// we want fn to be of the form "dir/prog"
|
||||
dir := filepath.Dir(filepath.Dir(ep))
|
||||
fn, err := filepath.Rel(dir, ep)
|
||||
if err != nil {
|
||||
t.Fatalf("filepath.Rel: %v", err)
|
||||
}
|
||||
cmd := &oexec.Cmd{}
|
||||
// make child start with a relative program path
|
||||
cmd.Dir = dir
|
||||
cmd.Path = fn
|
||||
// forge argv[0] for child, so that we can verify we could correctly
|
||||
// get real path of the executable without influenced by argv[0].
|
||||
cmd.Args = []string{"-", "-test.run=XXXX"}
|
||||
cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("exec(self) failed: %v", err)
|
||||
}
|
||||
outs := string(out)
|
||||
if !filepath.IsAbs(outs) {
|
||||
t.Fatalf("Child returned %q, want an absolute path", out)
|
||||
}
|
||||
if !sameFile(outs, ep) {
|
||||
t.Fatalf("Child returned %q, not the same file as %q", out, ep)
|
||||
}
|
||||
}
|
||||
|
||||
func sameFile(fn1, fn2 string) bool {
|
||||
fi1, err := os.Stat(fn1)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
fi2, err := os.Stat(fn2)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return os.SameFile(fi1, fi2)
|
||||
}
|
||||
|
||||
func init() {
|
||||
if e := os.Getenv(execPath_EnvVar); e != "" {
|
||||
// first chdir to another path
|
||||
dir := "/"
|
||||
if runtime.GOOS == "windows" {
|
||||
dir = filepath.VolumeName(".")
|
||||
}
|
||||
os.Chdir(dir)
|
||||
if ep, err := Executable(); err != nil {
|
||||
fmt.Fprint(os.Stderr, "ERROR: ", err)
|
||||
} else {
|
||||
fmt.Fprint(os.Stderr, ep)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
-496
@@ -1,496 +0,0 @@
|
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package transform provides reader and writer wrappers that transform the
|
||||
// bytes passing through as well as various transformations. Example
|
||||
// transformations provided by other packages include normalization and
|
||||
// conversion between character sets.
|
||||
package transform
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrShortDst means that the destination buffer was too short to
|
||||
// receive all of the transformed bytes.
|
||||
ErrShortDst = errors.New("transform: short destination buffer")
|
||||
|
||||
// ErrShortSrc means that the source buffer has insufficient data to
|
||||
// complete the transformation.
|
||||
ErrShortSrc = errors.New("transform: short source buffer")
|
||||
|
||||
// errInconsistentByteCount means that Transform returned success (nil
|
||||
// error) but also returned nSrc inconsistent with the src argument.
|
||||
errInconsistentByteCount = errors.New("transform: inconsistent byte count returned")
|
||||
|
||||
// errShortInternal means that an internal buffer is not large enough
|
||||
// to make progress and the Transform operation must be aborted.
|
||||
errShortInternal = errors.New("transform: short internal buffer")
|
||||
)
|
||||
|
||||
// Transformer transforms bytes.
|
||||
type Transformer interface {
|
||||
// Transform writes to dst the transformed bytes read from src, and
|
||||
// returns the number of dst bytes written and src bytes read. The
|
||||
// atEOF argument tells whether src represents the last bytes of the
|
||||
// input.
|
||||
//
|
||||
// Callers should always process the nDst bytes produced and account
|
||||
// for the nSrc bytes consumed before considering the error err.
|
||||
//
|
||||
// A nil error means that all of the transformed bytes (whether freshly
|
||||
// transformed from src or left over from previous Transform calls)
|
||||
// were written to dst. A nil error can be returned regardless of
|
||||
// whether atEOF is true. If err is nil then nSrc must equal len(src);
|
||||
// the converse is not necessarily true.
|
||||
//
|
||||
// ErrShortDst means that dst was too short to receive all of the
|
||||
// transformed bytes. ErrShortSrc means that src had insufficient data
|
||||
// to complete the transformation. If both conditions apply, then
|
||||
// either error may be returned. Other than the error conditions listed
|
||||
// here, implementations are free to report other errors that arise.
|
||||
Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
|
||||
}
|
||||
|
||||
// TODO: Do we require that a Transformer be reusable if it returns a nil error
|
||||
// or do we always require a reset after use? Is Reset mandatory or optional?
|
||||
|
||||
// Reader wraps another io.Reader by transforming the bytes read.
|
||||
type Reader struct {
|
||||
r io.Reader
|
||||
t Transformer
|
||||
err error
|
||||
|
||||
// dst[dst0:dst1] contains bytes that have been transformed by t but
|
||||
// not yet copied out via Read.
|
||||
dst []byte
|
||||
dst0, dst1 int
|
||||
|
||||
// src[src0:src1] contains bytes that have been read from r but not
|
||||
// yet transformed through t.
|
||||
src []byte
|
||||
src0, src1 int
|
||||
|
||||
// transformComplete is whether the transformation is complete,
|
||||
// regardless of whether or not it was successful.
|
||||
transformComplete bool
|
||||
}
|
||||
|
||||
const defaultBufSize = 4096
|
||||
|
||||
// NewReader returns a new Reader that wraps r by transforming the bytes read
|
||||
// via t.
|
||||
func NewReader(r io.Reader, t Transformer) *Reader {
|
||||
return &Reader{
|
||||
r: r,
|
||||
t: t,
|
||||
dst: make([]byte, defaultBufSize),
|
||||
src: make([]byte, defaultBufSize),
|
||||
}
|
||||
}
|
||||
|
||||
// Read implements the io.Reader interface.
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := 0, error(nil)
|
||||
for {
|
||||
// Copy out any transformed bytes and return the final error if we are done.
|
||||
if r.dst0 != r.dst1 {
|
||||
n = copy(p, r.dst[r.dst0:r.dst1])
|
||||
r.dst0 += n
|
||||
if r.dst0 == r.dst1 && r.transformComplete {
|
||||
return n, r.err
|
||||
}
|
||||
return n, nil
|
||||
} else if r.transformComplete {
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
// Try to transform some source bytes, or to flush the transformer if we
|
||||
// are out of source bytes. We do this even if r.r.Read returned an error.
|
||||
// As the io.Reader documentation says, "process the n > 0 bytes returned
|
||||
// before considering the error".
|
||||
if r.src0 != r.src1 || r.err != nil {
|
||||
r.dst0 = 0
|
||||
r.dst1, n, err = r.t.Transform(r.dst, r.src[r.src0:r.src1], r.err == io.EOF)
|
||||
r.src0 += n
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
if r.src0 != r.src1 {
|
||||
r.err = errInconsistentByteCount
|
||||
}
|
||||
// The Transform call was successful; we are complete if we
|
||||
// cannot read more bytes into src.
|
||||
r.transformComplete = r.err != nil
|
||||
continue
|
||||
case err == ErrShortDst && r.dst1 != 0:
|
||||
// Make room in dst by copying out, and try again.
|
||||
continue
|
||||
case err == ErrShortSrc && r.src1-r.src0 != len(r.src) && r.err == nil:
|
||||
// Read more bytes into src via the code below, and try again.
|
||||
default:
|
||||
r.transformComplete = true
|
||||
// The reader error (r.err) takes precedence over the
|
||||
// transformer error (err) unless r.err is nil or io.EOF.
|
||||
if r.err == nil || r.err == io.EOF {
|
||||
r.err = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Move any untransformed source bytes to the start of the buffer
|
||||
// and read more bytes.
|
||||
if r.src0 != 0 {
|
||||
r.src0, r.src1 = 0, copy(r.src, r.src[r.src0:r.src1])
|
||||
}
|
||||
n, r.err = r.r.Read(r.src[r.src1:])
|
||||
r.src1 += n
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: implement ReadByte (and ReadRune??).
|
||||
|
||||
// Writer wraps another io.Writer by transforming the bytes read.
|
||||
// The user needs to call Close to flush unwritten bytes that may
|
||||
// be buffered.
|
||||
type Writer struct {
|
||||
w io.Writer
|
||||
t Transformer
|
||||
dst []byte
|
||||
|
||||
// src[:n] contains bytes that have not yet passed through t.
|
||||
src []byte
|
||||
n int
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer that wraps w by transforming the bytes written
|
||||
// via t.
|
||||
func NewWriter(w io.Writer, t Transformer) *Writer {
|
||||
return &Writer{
|
||||
w: w,
|
||||
t: t,
|
||||
dst: make([]byte, defaultBufSize),
|
||||
src: make([]byte, defaultBufSize),
|
||||
}
|
||||
}
|
||||
|
||||
// Write implements the io.Writer interface. If there are not enough
|
||||
// bytes available to complete a Transform, the bytes will be buffered
|
||||
// for the next write. Call Close to convert the remaining bytes.
|
||||
func (w *Writer) Write(data []byte) (n int, err error) {
|
||||
src := data
|
||||
if w.n > 0 {
|
||||
// Append bytes from data to the last remainder.
|
||||
// TODO: limit the amount copied on first try.
|
||||
n = copy(w.src[w.n:], data)
|
||||
w.n += n
|
||||
src = w.src[:w.n]
|
||||
}
|
||||
for {
|
||||
nDst, nSrc, err := w.t.Transform(w.dst, src, false)
|
||||
if _, werr := w.w.Write(w.dst[:nDst]); werr != nil {
|
||||
return n, werr
|
||||
}
|
||||
src = src[nSrc:]
|
||||
if w.n > 0 && len(src) <= n {
|
||||
// Enough bytes from w.src have been consumed. We make src point
|
||||
// to data instead to reduce the copying.
|
||||
w.n = 0
|
||||
n -= len(src)
|
||||
src = data[n:]
|
||||
if n < len(data) && (err == nil || err == ErrShortSrc) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
n += nSrc
|
||||
}
|
||||
switch {
|
||||
case err == ErrShortDst && nDst > 0:
|
||||
case err == ErrShortSrc && len(src) < len(w.src):
|
||||
m := copy(w.src, src)
|
||||
// If w.n > 0, bytes from data were already copied to w.src and n
|
||||
// was already set to the number of bytes consumed.
|
||||
if w.n == 0 {
|
||||
n += m
|
||||
}
|
||||
w.n = m
|
||||
return n, nil
|
||||
case err == nil && w.n > 0:
|
||||
return n, errInconsistentByteCount
|
||||
default:
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close implements the io.Closer interface.
|
||||
func (w *Writer) Close() error {
|
||||
for src := w.src[:w.n]; len(src) > 0; {
|
||||
nDst, nSrc, err := w.t.Transform(w.dst, src, true)
|
||||
if nDst == 0 {
|
||||
return err
|
||||
}
|
||||
if _, werr := w.w.Write(w.dst[:nDst]); werr != nil {
|
||||
return werr
|
||||
}
|
||||
if err != ErrShortDst {
|
||||
return err
|
||||
}
|
||||
src = src[nSrc:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type nop struct{}
|
||||
|
||||
func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := copy(dst, src)
|
||||
if n < len(src) {
|
||||
err = ErrShortDst
|
||||
}
|
||||
return n, n, err
|
||||
}
|
||||
|
||||
type discard struct{}
|
||||
|
||||
func (discard) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
return 0, len(src), nil
|
||||
}
|
||||
|
||||
var (
|
||||
// Discard is a Transformer for which all Transform calls succeed
|
||||
// by consuming all bytes and writing nothing.
|
||||
Discard Transformer = discard{}
|
||||
|
||||
// Nop is a Transformer that copies src to dst.
|
||||
Nop Transformer = nop{}
|
||||
)
|
||||
|
||||
// chain is a sequence of links. A chain with N Transformers has N+1 links and
|
||||
// N+1 buffers. Of those N+1 buffers, the first and last are the src and dst
|
||||
// buffers given to chain.Transform and the middle N-1 buffers are intermediate
|
||||
// buffers owned by the chain. The i'th link transforms bytes from the i'th
|
||||
// buffer chain.link[i].b at read offset chain.link[i].p to the i+1'th buffer
|
||||
// chain.link[i+1].b at write offset chain.link[i+1].n, for i in [0, N).
|
||||
type chain struct {
|
||||
link []link
|
||||
err error
|
||||
// errStart is the index at which the error occurred plus 1. Processing
|
||||
// errStart at this level at the next call to Transform. As long as
|
||||
// errStart > 0, chain will not consume any more source bytes.
|
||||
errStart int
|
||||
}
|
||||
|
||||
func (c *chain) fatalError(errIndex int, err error) {
|
||||
if i := errIndex + 1; i > c.errStart {
|
||||
c.errStart = i
|
||||
c.err = err
|
||||
}
|
||||
}
|
||||
|
||||
type link struct {
|
||||
t Transformer
|
||||
// b[p:n] holds the bytes to be transformed by t.
|
||||
b []byte
|
||||
p int
|
||||
n int
|
||||
}
|
||||
|
||||
func (l *link) src() []byte {
|
||||
return l.b[l.p:l.n]
|
||||
}
|
||||
|
||||
func (l *link) dst() []byte {
|
||||
return l.b[l.n:]
|
||||
}
|
||||
|
||||
// Chain returns a Transformer that applies t in sequence.
|
||||
func Chain(t ...Transformer) Transformer {
|
||||
if len(t) == 0 {
|
||||
return nop{}
|
||||
}
|
||||
c := &chain{link: make([]link, len(t)+1)}
|
||||
for i, tt := range t {
|
||||
c.link[i].t = tt
|
||||
}
|
||||
// Allocate intermediate buffers.
|
||||
b := make([][defaultBufSize]byte, len(t)-1)
|
||||
for i := range b {
|
||||
c.link[i+1].b = b[i][:]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Transform applies the transformers of c in sequence.
|
||||
func (c *chain) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
// Set up src and dst in the chain.
|
||||
srcL := &c.link[0]
|
||||
dstL := &c.link[len(c.link)-1]
|
||||
srcL.b, srcL.p, srcL.n = src, 0, len(src)
|
||||
dstL.b, dstL.n = dst, 0
|
||||
var lastFull, needProgress bool // for detecting progress
|
||||
|
||||
// i is the index of the next Transformer to apply, for i in [low, high].
|
||||
// low is the lowest index for which c.link[low] may still produce bytes.
|
||||
// high is the highest index for which c.link[high] has a Transformer.
|
||||
// The error returned by Transform determines whether to increase or
|
||||
// decrease i. We try to completely fill a buffer before converting it.
|
||||
for low, i, high := c.errStart, c.errStart, len(c.link)-2; low <= i && i <= high; {
|
||||
in, out := &c.link[i], &c.link[i+1]
|
||||
nDst, nSrc, err0 := in.t.Transform(out.dst(), in.src(), atEOF && low == i)
|
||||
out.n += nDst
|
||||
in.p += nSrc
|
||||
if i > 0 && in.p == in.n {
|
||||
in.p, in.n = 0, 0
|
||||
}
|
||||
needProgress, lastFull = lastFull, false
|
||||
switch err0 {
|
||||
case ErrShortDst:
|
||||
// Process the destination buffer next. Return if we are already
|
||||
// at the high index.
|
||||
if i == high {
|
||||
return dstL.n, srcL.p, ErrShortDst
|
||||
}
|
||||
if out.n != 0 {
|
||||
i++
|
||||
// If the Transformer at the next index is not able to process any
|
||||
// source bytes there is nothing that can be done to make progress
|
||||
// and the bytes will remain unprocessed. lastFull is used to
|
||||
// detect this and break out of the loop with a fatal error.
|
||||
lastFull = true
|
||||
continue
|
||||
}
|
||||
// The destination buffer was too small, but is completely empty.
|
||||
// Return a fatal error as this transformation can never complete.
|
||||
c.fatalError(i, errShortInternal)
|
||||
case ErrShortSrc:
|
||||
if i == 0 {
|
||||
// Save ErrShortSrc in err. All other errors take precedence.
|
||||
err = ErrShortSrc
|
||||
break
|
||||
}
|
||||
// Source bytes were depleted before filling up the destination buffer.
|
||||
// Verify we made some progress, move the remaining bytes to the errStart
|
||||
// and try to get more source bytes.
|
||||
if needProgress && nSrc == 0 || in.n-in.p == len(in.b) {
|
||||
// There were not enough source bytes to proceed while the source
|
||||
// buffer cannot hold any more bytes. Return a fatal error as this
|
||||
// transformation can never complete.
|
||||
c.fatalError(i, errShortInternal)
|
||||
break
|
||||
}
|
||||
// in.b is an internal buffer and we can make progress.
|
||||
in.p, in.n = 0, copy(in.b, in.src())
|
||||
fallthrough
|
||||
case nil:
|
||||
// if i == low, we have depleted the bytes at index i or any lower levels.
|
||||
// In that case we increase low and i. In all other cases we decrease i to
|
||||
// fetch more bytes before proceeding to the next index.
|
||||
if i > low {
|
||||
i--
|
||||
continue
|
||||
}
|
||||
default:
|
||||
c.fatalError(i, err0)
|
||||
}
|
||||
// Exhausted level low or fatal error: increase low and continue
|
||||
// to process the bytes accepted so far.
|
||||
i++
|
||||
low = i
|
||||
}
|
||||
|
||||
// If c.errStart > 0, this means we found a fatal error. We will clear
|
||||
// all upstream buffers. At this point, no more progress can be made
|
||||
// downstream, as Transform would have bailed while handling ErrShortDst.
|
||||
if c.errStart > 0 {
|
||||
for i := 1; i < c.errStart; i++ {
|
||||
c.link[i].p, c.link[i].n = 0, 0
|
||||
}
|
||||
err, c.errStart, c.err = c.err, 0, nil
|
||||
}
|
||||
return dstL.n, srcL.p, err
|
||||
}
|
||||
|
||||
// RemoveFunc returns a Transformer that removes from the input all runes r for
|
||||
// which f(r) is true. Illegal bytes in the input are replaced by RuneError.
|
||||
func RemoveFunc(f func(r rune) bool) Transformer {
|
||||
return removeF(f)
|
||||
}
|
||||
|
||||
type removeF func(r rune) bool
|
||||
|
||||
// Transform implements the Transformer interface.
|
||||
func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
for r, sz := rune(0), 0; len(src) > 0; src = src[sz:] {
|
||||
|
||||
if r = rune(src[0]); r < utf8.RuneSelf {
|
||||
sz = 1
|
||||
} else {
|
||||
r, sz = utf8.DecodeRune(src)
|
||||
|
||||
if sz == 1 {
|
||||
// Invalid rune.
|
||||
if !atEOF && !utf8.FullRune(src[nSrc:]) {
|
||||
err = ErrShortSrc
|
||||
break
|
||||
}
|
||||
// We replace illegal bytes with RuneError. Not doing so might
|
||||
// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
|
||||
// The resulting byte sequence may subsequently contain runes
|
||||
// for which t(r) is true that were passed unnoticed.
|
||||
if !t(r) {
|
||||
if nDst+3 > len(dst) {
|
||||
err = ErrShortDst
|
||||
break
|
||||
}
|
||||
nDst += copy(dst[nDst:], "\uFFFD")
|
||||
}
|
||||
nSrc++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if !t(r) {
|
||||
if nDst+sz > len(dst) {
|
||||
err = ErrShortDst
|
||||
break
|
||||
}
|
||||
nDst += copy(dst[nDst:], src[:sz])
|
||||
}
|
||||
nSrc += sz
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Bytes returns a new byte slice with the result of converting b using t.
|
||||
// If any unrecoverable error occurs it returns nil.
|
||||
func Bytes(t Transformer, b []byte) []byte {
|
||||
out := make([]byte, len(b))
|
||||
n := 0
|
||||
for {
|
||||
nDst, nSrc, err := t.Transform(out[n:], b, true)
|
||||
n += nDst
|
||||
if err == nil {
|
||||
return out[:n]
|
||||
} else if err != ErrShortDst {
|
||||
return nil
|
||||
}
|
||||
b = b[nSrc:]
|
||||
|
||||
// Grow the destination buffer.
|
||||
sz := len(out)
|
||||
if sz <= 256 {
|
||||
sz *= 2
|
||||
} else {
|
||||
sz += sz >> 1
|
||||
}
|
||||
out2 := make([]byte, sz)
|
||||
copy(out2, out[:n])
|
||||
out = out2
|
||||
}
|
||||
}
|
||||
-901
@@ -1,901 +0,0 @@
|
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package transform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type lowerCaseASCII struct{}
|
||||
|
||||
func (lowerCaseASCII) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := len(src)
|
||||
if n > len(dst) {
|
||||
n, err = len(dst), ErrShortDst
|
||||
}
|
||||
for i, c := range src[:n] {
|
||||
if 'A' <= c && c <= 'Z' {
|
||||
c += 'a' - 'A'
|
||||
}
|
||||
dst[i] = c
|
||||
}
|
||||
return n, n, err
|
||||
}
|
||||
|
||||
var errYouMentionedX = errors.New("you mentioned X")
|
||||
|
||||
type dontMentionX struct{}
|
||||
|
||||
func (dontMentionX) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := len(src)
|
||||
if n > len(dst) {
|
||||
n, err = len(dst), ErrShortDst
|
||||
}
|
||||
for i, c := range src[:n] {
|
||||
if c == 'X' {
|
||||
return i, i, errYouMentionedX
|
||||
}
|
||||
dst[i] = c
|
||||
}
|
||||
return n, n, err
|
||||
}
|
||||
|
||||
// doublerAtEOF is a strange Transformer that transforms "this" to "tthhiiss",
|
||||
// but only if atEOF is true.
|
||||
type doublerAtEOF struct{}
|
||||
|
||||
func (doublerAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
if !atEOF {
|
||||
return 0, 0, ErrShortSrc
|
||||
}
|
||||
for i, c := range src {
|
||||
if 2*i+2 >= len(dst) {
|
||||
return 2 * i, i, ErrShortDst
|
||||
}
|
||||
dst[2*i+0] = c
|
||||
dst[2*i+1] = c
|
||||
}
|
||||
return 2 * len(src), len(src), nil
|
||||
}
|
||||
|
||||
// rleDecode and rleEncode implement a toy run-length encoding: "aabbbbbbbbbb"
|
||||
// is encoded as "2a10b". The decoding is assumed to not contain any numbers.
|
||||
|
||||
type rleDecode struct{}
|
||||
|
||||
func (rleDecode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
loop:
|
||||
for len(src) > 0 {
|
||||
n := 0
|
||||
for i, c := range src {
|
||||
if '0' <= c && c <= '9' {
|
||||
n = 10*n + int(c-'0')
|
||||
continue
|
||||
}
|
||||
if i == 0 {
|
||||
return nDst, nSrc, errors.New("rleDecode: bad input")
|
||||
}
|
||||
if n > len(dst) {
|
||||
return nDst, nSrc, ErrShortDst
|
||||
}
|
||||
for j := 0; j < n; j++ {
|
||||
dst[j] = c
|
||||
}
|
||||
dst, src = dst[n:], src[i+1:]
|
||||
nDst, nSrc = nDst+n, nSrc+i+1
|
||||
continue loop
|
||||
}
|
||||
if atEOF {
|
||||
return nDst, nSrc, errors.New("rleDecode: bad input")
|
||||
}
|
||||
return nDst, nSrc, ErrShortSrc
|
||||
}
|
||||
return nDst, nSrc, nil
|
||||
}
|
||||
|
||||
type rleEncode struct {
|
||||
// allowStutter means that "xxxxxxxx" can be encoded as "5x3x"
|
||||
// instead of always as "8x".
|
||||
allowStutter bool
|
||||
}
|
||||
|
||||
func (e rleEncode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
for len(src) > 0 {
|
||||
n, c0 := len(src), src[0]
|
||||
for i, c := range src[1:] {
|
||||
if c != c0 {
|
||||
n = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if n == len(src) && !atEOF && !e.allowStutter {
|
||||
return nDst, nSrc, ErrShortSrc
|
||||
}
|
||||
s := strconv.Itoa(n)
|
||||
if len(s) >= len(dst) {
|
||||
return nDst, nSrc, ErrShortDst
|
||||
}
|
||||
copy(dst, s)
|
||||
dst[len(s)] = c0
|
||||
dst, src = dst[len(s)+1:], src[n:]
|
||||
nDst, nSrc = nDst+len(s)+1, nSrc+n
|
||||
}
|
||||
return nDst, nSrc, nil
|
||||
}
|
||||
|
||||
type testCase struct {
|
||||
desc string
|
||||
t Transformer
|
||||
src string
|
||||
dstSize int
|
||||
srcSize int
|
||||
ioSize int
|
||||
wantStr string
|
||||
wantErr error
|
||||
wantIter int // number of iterations taken; 0 means we don't care.
|
||||
}
|
||||
|
||||
func (t testCase) String() string {
|
||||
return tstr(t.t) + "; " + t.desc
|
||||
}
|
||||
|
||||
func tstr(t Transformer) string {
|
||||
if stringer, ok := t.(fmt.Stringer); ok {
|
||||
return stringer.String()
|
||||
}
|
||||
s := fmt.Sprintf("%T", t)
|
||||
return s[1+strings.Index(s, "."):]
|
||||
}
|
||||
|
||||
func (c chain) String() string {
|
||||
buf := &bytes.Buffer{}
|
||||
buf.WriteString("Chain(")
|
||||
for i, l := range c.link[:len(c.link)-1] {
|
||||
if i != 0 {
|
||||
fmt.Fprint(buf, ", ")
|
||||
}
|
||||
buf.WriteString(tstr(l.t))
|
||||
}
|
||||
buf.WriteString(")")
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
var testCases = []testCase{
|
||||
{
|
||||
desc: "basic",
|
||||
t: lowerCaseASCII{},
|
||||
src: "Hello WORLD.",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "hello world.",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "small dst",
|
||||
t: lowerCaseASCII{},
|
||||
src: "Hello WORLD.",
|
||||
dstSize: 3,
|
||||
srcSize: 100,
|
||||
wantStr: "hello world.",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "small src",
|
||||
t: lowerCaseASCII{},
|
||||
src: "Hello WORLD.",
|
||||
dstSize: 100,
|
||||
srcSize: 4,
|
||||
wantStr: "hello world.",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "small buffers",
|
||||
t: lowerCaseASCII{},
|
||||
src: "Hello WORLD.",
|
||||
dstSize: 3,
|
||||
srcSize: 4,
|
||||
wantStr: "hello world.",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "very small buffers",
|
||||
t: lowerCaseASCII{},
|
||||
src: "Hello WORLD.",
|
||||
dstSize: 1,
|
||||
srcSize: 1,
|
||||
wantStr: "hello world.",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "basic",
|
||||
t: dontMentionX{},
|
||||
src: "The First Rule of Transform Club: don't mention Mister X, ever.",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "The First Rule of Transform Club: don't mention Mister ",
|
||||
wantErr: errYouMentionedX,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "small buffers",
|
||||
t: dontMentionX{},
|
||||
src: "The First Rule of Transform Club: don't mention Mister X, ever.",
|
||||
dstSize: 10,
|
||||
srcSize: 10,
|
||||
wantStr: "The First Rule of Transform Club: don't mention Mister ",
|
||||
wantErr: errYouMentionedX,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "very small buffers",
|
||||
t: dontMentionX{},
|
||||
src: "The First Rule of Transform Club: don't mention Mister X, ever.",
|
||||
dstSize: 1,
|
||||
srcSize: 1,
|
||||
wantStr: "The First Rule of Transform Club: don't mention Mister ",
|
||||
wantErr: errYouMentionedX,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "only transform at EOF",
|
||||
t: doublerAtEOF{},
|
||||
src: "this",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "tthhiiss",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "basic",
|
||||
t: rleDecode{},
|
||||
src: "1a2b3c10d11e0f1g",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "abbcccddddddddddeeeeeeeeeeeg",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "long",
|
||||
t: rleDecode{},
|
||||
src: "12a23b34c45d56e99z",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: strings.Repeat("a", 12) +
|
||||
strings.Repeat("b", 23) +
|
||||
strings.Repeat("c", 34) +
|
||||
strings.Repeat("d", 45) +
|
||||
strings.Repeat("e", 56) +
|
||||
strings.Repeat("z", 99),
|
||||
},
|
||||
|
||||
{
|
||||
desc: "tight buffers",
|
||||
t: rleDecode{},
|
||||
src: "1a2b3c10d11e0f1g",
|
||||
dstSize: 11,
|
||||
srcSize: 3,
|
||||
wantStr: "abbcccddddddddddeeeeeeeeeeeg",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short dst",
|
||||
t: rleDecode{},
|
||||
src: "1a2b3c10d11e0f1g",
|
||||
dstSize: 10,
|
||||
srcSize: 3,
|
||||
wantStr: "abbcccdddddddddd",
|
||||
wantErr: ErrShortDst,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short src",
|
||||
t: rleDecode{},
|
||||
src: "1a2b3c10d11e0f1g",
|
||||
dstSize: 11,
|
||||
srcSize: 2,
|
||||
ioSize: 2,
|
||||
wantStr: "abbccc",
|
||||
wantErr: ErrShortSrc,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "basic",
|
||||
t: rleEncode{},
|
||||
src: "abbcccddddddddddeeeeeeeeeeeg",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "1a2b3c10d11e1g",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "long",
|
||||
t: rleEncode{},
|
||||
src: strings.Repeat("a", 12) +
|
||||
strings.Repeat("b", 23) +
|
||||
strings.Repeat("c", 34) +
|
||||
strings.Repeat("d", 45) +
|
||||
strings.Repeat("e", 56) +
|
||||
strings.Repeat("z", 99),
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "12a23b34c45d56e99z",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "tight buffers",
|
||||
t: rleEncode{},
|
||||
src: "abbcccddddddddddeeeeeeeeeeeg",
|
||||
dstSize: 3,
|
||||
srcSize: 12,
|
||||
wantStr: "1a2b3c10d11e1g",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short dst",
|
||||
t: rleEncode{},
|
||||
src: "abbcccddddddddddeeeeeeeeeeeg",
|
||||
dstSize: 2,
|
||||
srcSize: 12,
|
||||
wantStr: "1a2b3c",
|
||||
wantErr: ErrShortDst,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short src",
|
||||
t: rleEncode{},
|
||||
src: "abbcccddddddddddeeeeeeeeeeeg",
|
||||
dstSize: 3,
|
||||
srcSize: 11,
|
||||
ioSize: 11,
|
||||
wantStr: "1a2b3c10d",
|
||||
wantErr: ErrShortSrc,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "allowStutter = false",
|
||||
t: rleEncode{allowStutter: false},
|
||||
src: "aaaabbbbbbbbccccddddd",
|
||||
dstSize: 10,
|
||||
srcSize: 10,
|
||||
wantStr: "4a8b4c5d",
|
||||
},
|
||||
|
||||
{
|
||||
desc: "allowStutter = true",
|
||||
t: rleEncode{allowStutter: true},
|
||||
src: "aaaabbbbbbbbccccddddd",
|
||||
dstSize: 10,
|
||||
srcSize: 10,
|
||||
ioSize: 10,
|
||||
wantStr: "4a6b2b4c4d1d",
|
||||
},
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
reset(tc.t)
|
||||
r := NewReader(strings.NewReader(tc.src), tc.t)
|
||||
// Differently sized dst and src buffers are not part of the
|
||||
// exported API. We override them manually.
|
||||
r.dst = make([]byte, tc.dstSize)
|
||||
r.src = make([]byte, tc.srcSize)
|
||||
got, err := ioutil.ReadAll(r)
|
||||
str := string(got)
|
||||
if str != tc.wantStr || err != tc.wantErr {
|
||||
t.Errorf("%s:\ngot %q, %v\nwant %q, %v", tc, str, err, tc.wantStr, tc.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func reset(t Transformer) {
|
||||
var dst [128]byte
|
||||
for err := ErrShortDst; err != nil; {
|
||||
_, _, err = t.Transform(dst[:], nil, true)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriter(t *testing.T) {
|
||||
tests := append(testCases, chainTests()...)
|
||||
for _, tc := range tests {
|
||||
sizes := []int{1, 2, 3, 4, 5, 10, 100, 1000}
|
||||
if tc.ioSize > 0 {
|
||||
sizes = []int{tc.ioSize}
|
||||
}
|
||||
for _, sz := range sizes {
|
||||
bb := &bytes.Buffer{}
|
||||
reset(tc.t)
|
||||
w := NewWriter(bb, tc.t)
|
||||
// Differently sized dst and src buffers are not part of the
|
||||
// exported API. We override them manually.
|
||||
w.dst = make([]byte, tc.dstSize)
|
||||
w.src = make([]byte, tc.srcSize)
|
||||
src := make([]byte, sz)
|
||||
var err error
|
||||
for b := tc.src; len(b) > 0 && err == nil; {
|
||||
n := copy(src, b)
|
||||
b = b[n:]
|
||||
m := 0
|
||||
m, err = w.Write(src[:n])
|
||||
if m != n && err == nil {
|
||||
t.Errorf("%s:%d: did not consume all bytes %d < %d", tc, sz, m, n)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = w.Close()
|
||||
}
|
||||
str := bb.String()
|
||||
if str != tc.wantStr || err != tc.wantErr {
|
||||
t.Errorf("%s:%d:\ngot %q, %v\nwant %q, %v", tc, sz, str, err, tc.wantStr, tc.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNop(t *testing.T) {
|
||||
testCases := []struct {
|
||||
str string
|
||||
dstSize int
|
||||
err error
|
||||
}{
|
||||
{"", 0, nil},
|
||||
{"", 10, nil},
|
||||
{"a", 0, ErrShortDst},
|
||||
{"a", 1, nil},
|
||||
{"a", 10, nil},
|
||||
}
|
||||
for i, tc := range testCases {
|
||||
dst := make([]byte, tc.dstSize)
|
||||
nDst, nSrc, err := Nop.Transform(dst, []byte(tc.str), true)
|
||||
want := tc.str
|
||||
if tc.dstSize < len(want) {
|
||||
want = want[:tc.dstSize]
|
||||
}
|
||||
if got := string(dst[:nDst]); got != want || err != tc.err || nSrc != nDst {
|
||||
t.Errorf("%d:\ngot %q, %d, %v\nwant %q, %d, %v", i, got, nSrc, err, want, nDst, tc.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscard(t *testing.T) {
|
||||
testCases := []struct {
|
||||
str string
|
||||
dstSize int
|
||||
}{
|
||||
{"", 0},
|
||||
{"", 10},
|
||||
{"a", 0},
|
||||
{"ab", 10},
|
||||
}
|
||||
for i, tc := range testCases {
|
||||
nDst, nSrc, err := Discard.Transform(make([]byte, tc.dstSize), []byte(tc.str), true)
|
||||
if nDst != 0 || nSrc != len(tc.str) || err != nil {
|
||||
t.Errorf("%d:\ngot %q, %d, %v\nwant 0, %d, nil", i, nDst, nSrc, err, len(tc.str))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mkChain creates a Chain transformer. x must be alternating between transformer
|
||||
// and bufSize, like T, (sz, T)*
|
||||
func mkChain(x ...interface{}) *chain {
|
||||
t := []Transformer{}
|
||||
for i := 0; i < len(x); i += 2 {
|
||||
t = append(t, x[i].(Transformer))
|
||||
}
|
||||
c := Chain(t...).(*chain)
|
||||
for i, j := 1, 1; i < len(x); i, j = i+2, j+1 {
|
||||
c.link[j].b = make([]byte, x[i].(int))
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func chainTests() []testCase {
|
||||
return []testCase{
|
||||
{
|
||||
desc: "nil error",
|
||||
t: mkChain(rleEncode{}, 100, lowerCaseASCII{}),
|
||||
src: "ABB",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "1a2b",
|
||||
wantErr: nil,
|
||||
wantIter: 1,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short dst buffer",
|
||||
t: mkChain(lowerCaseASCII{}, 3, rleDecode{}),
|
||||
src: "1a2b3c10d11e0f1g",
|
||||
dstSize: 10,
|
||||
srcSize: 3,
|
||||
wantStr: "abbcccdddddddddd",
|
||||
wantErr: ErrShortDst,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short internal dst buffer",
|
||||
t: mkChain(lowerCaseASCII{}, 3, rleDecode{}, 10, Nop),
|
||||
src: "1a2b3c10d11e0f1g",
|
||||
dstSize: 100,
|
||||
srcSize: 3,
|
||||
wantStr: "abbcccdddddddddd",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short internal dst buffer from input",
|
||||
t: mkChain(rleDecode{}, 10, Nop),
|
||||
src: "1a2b3c10d11e0f1g",
|
||||
dstSize: 100,
|
||||
srcSize: 3,
|
||||
wantStr: "abbcccdddddddddd",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "empty short internal dst buffer",
|
||||
t: mkChain(lowerCaseASCII{}, 3, rleDecode{}, 10, Nop),
|
||||
src: "4a7b11e0f1g",
|
||||
dstSize: 100,
|
||||
srcSize: 3,
|
||||
wantStr: "aaaabbbbbbb",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "empty short internal dst buffer from input",
|
||||
t: mkChain(rleDecode{}, 10, Nop),
|
||||
src: "4a7b11e0f1g",
|
||||
dstSize: 100,
|
||||
srcSize: 3,
|
||||
wantStr: "aaaabbbbbbb",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short internal src buffer after full dst buffer",
|
||||
t: mkChain(Nop, 5, rleEncode{}, 10, Nop),
|
||||
src: "cccccddddd",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "",
|
||||
wantErr: errShortInternal,
|
||||
wantIter: 1,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short internal src buffer after short dst buffer; test lastFull",
|
||||
t: mkChain(rleDecode{}, 5, rleEncode{}, 4, Nop),
|
||||
src: "2a1b4c6d",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "2a1b",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short internal src buffer after successful complete fill",
|
||||
t: mkChain(Nop, 3, rleDecode{}),
|
||||
src: "123a4b",
|
||||
dstSize: 4,
|
||||
srcSize: 3,
|
||||
wantStr: "",
|
||||
wantErr: errShortInternal,
|
||||
wantIter: 1,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short internal src buffer after short dst buffer; test lastFull",
|
||||
t: mkChain(rleDecode{}, 5, rleEncode{}),
|
||||
src: "2a1b4c6d",
|
||||
dstSize: 4,
|
||||
srcSize: 100,
|
||||
wantStr: "2a1b",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short src buffer",
|
||||
t: mkChain(rleEncode{}, 5, Nop),
|
||||
src: "abbcccddddeeeee",
|
||||
dstSize: 4,
|
||||
srcSize: 4,
|
||||
ioSize: 4,
|
||||
wantStr: "1a2b3c",
|
||||
wantErr: ErrShortSrc,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "process all in one go",
|
||||
t: mkChain(rleEncode{}, 5, Nop),
|
||||
src: "abbcccddddeeeeeffffff",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
wantStr: "1a2b3c4d5e6f",
|
||||
wantErr: nil,
|
||||
wantIter: 1,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "complete processing downstream after error",
|
||||
t: mkChain(dontMentionX{}, 2, rleDecode{}, 5, Nop),
|
||||
src: "3a4b5eX",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
ioSize: 100,
|
||||
wantStr: "aaabbbbeeeee",
|
||||
wantErr: errYouMentionedX,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "return downstream fatal errors first (followed by short dst)",
|
||||
t: mkChain(dontMentionX{}, 8, rleDecode{}, 4, Nop),
|
||||
src: "3a4b5eX",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
ioSize: 100,
|
||||
wantStr: "aaabbbb",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "return downstream fatal errors first (followed by short src)",
|
||||
t: mkChain(dontMentionX{}, 5, Nop, 1, rleDecode{}),
|
||||
src: "1a5bX",
|
||||
dstSize: 100,
|
||||
srcSize: 100,
|
||||
ioSize: 100,
|
||||
wantStr: "",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
|
||||
{
|
||||
desc: "short internal",
|
||||
t: mkChain(Nop, 11, rleEncode{}, 3, Nop),
|
||||
src: "abbcccddddddddddeeeeeeeeeeeg",
|
||||
dstSize: 3,
|
||||
srcSize: 100,
|
||||
wantStr: "1a2b3c10d",
|
||||
wantErr: errShortInternal,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func doTransform(tc testCase) (res string, iter int, err error) {
|
||||
reset(tc.t)
|
||||
dst := make([]byte, tc.dstSize)
|
||||
out, in := make([]byte, 0, 2*len(tc.src)), []byte(tc.src)
|
||||
for {
|
||||
iter++
|
||||
src, atEOF := in, true
|
||||
if len(src) > tc.srcSize {
|
||||
src, atEOF = src[:tc.srcSize], false
|
||||
}
|
||||
nDst, nSrc, err := tc.t.Transform(dst, src, atEOF)
|
||||
out = append(out, dst[:nDst]...)
|
||||
in = in[nSrc:]
|
||||
switch {
|
||||
case err == nil && len(in) != 0:
|
||||
case err == ErrShortSrc && nSrc > 0:
|
||||
case err == ErrShortDst && nDst > 0:
|
||||
default:
|
||||
return string(out), iter, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChain(t *testing.T) {
|
||||
if c, ok := Chain().(nop); !ok {
|
||||
t.Errorf("empty chain: %v; want Nop", c)
|
||||
}
|
||||
|
||||
// Test Chain for a single Transformer.
|
||||
for _, tc := range testCases {
|
||||
tc.t = Chain(tc.t)
|
||||
str, _, err := doTransform(tc)
|
||||
if str != tc.wantStr || err != tc.wantErr {
|
||||
t.Errorf("%s:\ngot %q, %v\nwant %q, %v", tc, str, err, tc.wantStr, tc.wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
tests := chainTests()
|
||||
sizes := []int{1, 2, 3, 4, 5, 7, 10, 100, 1000}
|
||||
addTest := func(tc testCase, t *chain) {
|
||||
if t.link[0].t != tc.t && tc.wantErr == ErrShortSrc {
|
||||
tc.wantErr = errShortInternal
|
||||
}
|
||||
if t.link[len(t.link)-2].t != tc.t && tc.wantErr == ErrShortDst {
|
||||
tc.wantErr = errShortInternal
|
||||
}
|
||||
tc.t = t
|
||||
tests = append(tests, tc)
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
for _, sz := range sizes {
|
||||
tt := tc
|
||||
tt.dstSize = sz
|
||||
addTest(tt, mkChain(tc.t, tc.dstSize, Nop))
|
||||
addTest(tt, mkChain(tc.t, tc.dstSize, Nop, 2, Nop))
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop))
|
||||
if sz >= tc.dstSize && (tc.wantErr != ErrShortDst || sz == tc.dstSize) {
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t))
|
||||
addTest(tt, mkChain(Nop, 100, Nop, tc.srcSize, tc.t))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
tt := tc
|
||||
tt.dstSize = 1
|
||||
tt.wantStr = ""
|
||||
addTest(tt, mkChain(tc.t, tc.dstSize, Discard))
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Discard))
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, tc.dstSize, Discard))
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
tt := tc
|
||||
tt.dstSize = 100
|
||||
tt.wantStr = strings.Replace(tc.src, "0f", "", -1)
|
||||
// Chain encoders and decoders.
|
||||
if _, ok := tc.t.(rleEncode); ok && tc.wantErr == nil {
|
||||
addTest(tt, mkChain(tc.t, tc.dstSize, Nop, 1000, rleDecode{}))
|
||||
addTest(tt, mkChain(tc.t, tc.dstSize, Nop, tc.dstSize, rleDecode{}))
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 100, rleDecode{}))
|
||||
// decoding needs larger destinations
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, rleDecode{}, 100, Nop))
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 100, rleDecode{}, 100, Nop))
|
||||
} else if _, ok := tc.t.(rleDecode); ok && tc.wantErr == nil {
|
||||
// The internal buffer size may need to be the sum of the maximum segment
|
||||
// size of the two encoders!
|
||||
addTest(tt, mkChain(tc.t, 2*tc.dstSize, rleEncode{}))
|
||||
addTest(tt, mkChain(tc.t, tc.dstSize, Nop, 101, rleEncode{}))
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 100, rleEncode{}))
|
||||
addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 200, rleEncode{}, 100, Nop))
|
||||
}
|
||||
}
|
||||
for _, tc := range tests {
|
||||
str, iter, err := doTransform(tc)
|
||||
mi := tc.wantIter != 0 && tc.wantIter != iter
|
||||
if str != tc.wantStr || err != tc.wantErr || mi {
|
||||
t.Errorf("%s:\ngot iter:%d, %q, %v\nwant iter:%d, %q, %v", tc, iter, str, err, tc.wantIter, tc.wantStr, tc.wantErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveFunc(t *testing.T) {
|
||||
filter := RemoveFunc(func(r rune) bool {
|
||||
return strings.IndexRune("ab\u0300\u1234,", r) != -1
|
||||
})
|
||||
tests := []testCase{
|
||||
{
|
||||
src: ",",
|
||||
wantStr: "",
|
||||
},
|
||||
|
||||
{
|
||||
src: "c",
|
||||
wantStr: "c",
|
||||
},
|
||||
|
||||
{
|
||||
src: "\u2345",
|
||||
wantStr: "\u2345",
|
||||
},
|
||||
|
||||
{
|
||||
src: "tschüß",
|
||||
wantStr: "tschüß",
|
||||
},
|
||||
|
||||
{
|
||||
src: ",до,свидания,",
|
||||
wantStr: "досвидания",
|
||||
},
|
||||
|
||||
{
|
||||
src: "a\xbd\xb2=\xbc ⌘",
|
||||
wantStr: "\uFFFD\uFFFD=\uFFFD ⌘",
|
||||
},
|
||||
|
||||
{
|
||||
// If we didn't replace illegal bytes with RuneError, the result
|
||||
// would be \u0300 or the code would need to be more complex.
|
||||
src: "\xcc\u0300\x80",
|
||||
wantStr: "\uFFFD\uFFFD",
|
||||
},
|
||||
|
||||
{
|
||||
src: "\xcc\u0300\x80",
|
||||
dstSize: 3,
|
||||
wantStr: "\uFFFD\uFFFD",
|
||||
wantIter: 2,
|
||||
},
|
||||
|
||||
{
|
||||
src: "\u2345",
|
||||
dstSize: 2,
|
||||
wantStr: "",
|
||||
wantErr: ErrShortDst,
|
||||
},
|
||||
|
||||
{
|
||||
src: "\xcc",
|
||||
dstSize: 2,
|
||||
wantStr: "",
|
||||
wantErr: ErrShortDst,
|
||||
},
|
||||
|
||||
{
|
||||
src: "\u0300",
|
||||
dstSize: 2,
|
||||
srcSize: 1,
|
||||
wantStr: "",
|
||||
wantErr: ErrShortSrc,
|
||||
},
|
||||
|
||||
{
|
||||
t: RemoveFunc(func(r rune) bool {
|
||||
return r == utf8.RuneError
|
||||
}),
|
||||
src: "\xcc\u0300\x80",
|
||||
wantStr: "\u0300",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
tc.desc = tc.src
|
||||
if tc.t == nil {
|
||||
tc.t = filter
|
||||
}
|
||||
if tc.dstSize == 0 {
|
||||
tc.dstSize = 100
|
||||
}
|
||||
if tc.srcSize == 0 {
|
||||
tc.srcSize = 100
|
||||
}
|
||||
str, iter, err := doTransform(tc)
|
||||
mi := tc.wantIter != 0 && tc.wantIter != iter
|
||||
if str != tc.wantStr || err != tc.wantErr || mi {
|
||||
t.Errorf("%+q:\ngot iter:%d, %+q, %v\nwant iter:%d, %+q, %v", tc.src, iter, str, err, tc.wantIter, tc.wantStr, tc.wantErr)
|
||||
}
|
||||
|
||||
tc.src = str
|
||||
idem, _, _ := doTransform(tc)
|
||||
if str != idem {
|
||||
t.Errorf("%+q: found %+q; want %+q", tc.src, idem, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytes(t *testing.T) {
|
||||
for _, tt := range append(testCases, chainTests()...) {
|
||||
if tt.desc == "allowStutter = true" {
|
||||
// We don't have control over the buffer size, so we eliminate tests
|
||||
// that depend on a specific buffer size being set.
|
||||
continue
|
||||
}
|
||||
got := Bytes(tt.t, []byte(tt.src))
|
||||
if tt.wantErr != nil {
|
||||
if tt.wantErr != ErrShortDst && tt.wantErr != ErrShortSrc {
|
||||
// Bytes should return nil for non-recoverable errors.
|
||||
if g, w := (got == nil), (tt.wantErr != nil); g != w {
|
||||
t.Errorf("%s:error: got %v; want %v", tt.desc, g, w)
|
||||
}
|
||||
}
|
||||
// The output strings in the tests that expect an error will
|
||||
// almost certainly not be the same as the result of Bytes.
|
||||
continue
|
||||
}
|
||||
if string(got) != tt.wantStr {
|
||||
t.Errorf("%s:string: got %q; want %q", tt.desc, got, tt.wantStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
-30
@@ -1,30 +0,0 @@
|
||||
# Copyright 2011 The Go Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style
|
||||
# license that can be found in the LICENSE file.
|
||||
|
||||
maketables: maketables.go triegen.go
|
||||
go build $^
|
||||
|
||||
maketesttables: maketesttables.go triegen.go
|
||||
go build $^
|
||||
|
||||
normregtest: normregtest.go
|
||||
go build $^
|
||||
|
||||
tables: maketables
|
||||
./maketables > tables.go
|
||||
gofmt -w tables.go
|
||||
|
||||
trietesttables: maketesttables
|
||||
./maketesttables > triedata_test.go
|
||||
gofmt -w triedata_test.go
|
||||
|
||||
# Downloads from www.unicode.org, so not part
|
||||
# of standard test scripts.
|
||||
test: testtables regtest
|
||||
|
||||
testtables: maketables
|
||||
./maketables -test > data_test.go && go test -tags=test
|
||||
|
||||
regtest: normregtest
|
||||
./normregtest
|
||||
-448
@@ -1,448 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const MaxSegmentSize = maxByteBufferSize
|
||||
|
||||
// An Iter iterates over a string or byte slice, while normalizing it
|
||||
// to a given Form.
|
||||
type Iter struct {
|
||||
rb reorderBuffer
|
||||
buf [maxByteBufferSize]byte
|
||||
info Properties // first character saved from previous iteration
|
||||
next iterFunc // implementation of next depends on form
|
||||
asciiF iterFunc
|
||||
|
||||
p int // current position in input source
|
||||
multiSeg []byte // remainder of multi-segment decomposition
|
||||
}
|
||||
|
||||
type iterFunc func(*Iter) []byte
|
||||
|
||||
// Init initializes i to iterate over src after normalizing it to Form f.
|
||||
func (i *Iter) Init(f Form, src []byte) {
|
||||
i.p = 0
|
||||
if len(src) == 0 {
|
||||
i.setDone()
|
||||
i.rb.nsrc = 0
|
||||
return
|
||||
}
|
||||
i.multiSeg = nil
|
||||
i.rb.init(f, src)
|
||||
i.next = i.rb.f.nextMain
|
||||
i.asciiF = nextASCIIBytes
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
}
|
||||
|
||||
// InitString initializes i to iterate over src after normalizing it to Form f.
|
||||
func (i *Iter) InitString(f Form, src string) {
|
||||
i.p = 0
|
||||
if len(src) == 0 {
|
||||
i.setDone()
|
||||
i.rb.nsrc = 0
|
||||
return
|
||||
}
|
||||
i.multiSeg = nil
|
||||
i.rb.initString(f, src)
|
||||
i.next = i.rb.f.nextMain
|
||||
i.asciiF = nextASCIIString
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
}
|
||||
|
||||
// Seek sets the segment to be returned by the next call to Next to start
|
||||
// at position p. It is the responsibility of the caller to set p to the
|
||||
// start of a UTF8 rune.
|
||||
func (i *Iter) Seek(offset int64, whence int) (int64, error) {
|
||||
var abs int64
|
||||
switch whence {
|
||||
case 0:
|
||||
abs = offset
|
||||
case 1:
|
||||
abs = int64(i.p) + offset
|
||||
case 2:
|
||||
abs = int64(i.rb.nsrc) + offset
|
||||
default:
|
||||
return 0, fmt.Errorf("norm: invalid whence")
|
||||
}
|
||||
if abs < 0 {
|
||||
return 0, fmt.Errorf("norm: negative position")
|
||||
}
|
||||
if int(abs) >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
return int64(i.p), nil
|
||||
}
|
||||
i.p = int(abs)
|
||||
i.multiSeg = nil
|
||||
i.next = i.rb.f.nextMain
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
// returnSlice returns a slice of the underlying input type as a byte slice.
|
||||
// If the underlying is of type []byte, it will simply return a slice.
|
||||
// If the underlying is of type string, it will copy the slice to the buffer
|
||||
// and return that.
|
||||
func (i *Iter) returnSlice(a, b int) []byte {
|
||||
if i.rb.src.bytes == nil {
|
||||
return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
|
||||
}
|
||||
return i.rb.src.bytes[a:b]
|
||||
}
|
||||
|
||||
// Pos returns the byte position at which the next call to Next will commence processing.
|
||||
func (i *Iter) Pos() int {
|
||||
return i.p
|
||||
}
|
||||
|
||||
func (i *Iter) setDone() {
|
||||
i.next = nextDone
|
||||
i.p = i.rb.nsrc
|
||||
}
|
||||
|
||||
// Done returns true if there is no more input to process.
|
||||
func (i *Iter) Done() bool {
|
||||
return i.p >= i.rb.nsrc
|
||||
}
|
||||
|
||||
// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
|
||||
// For any input a and b for which f(a) == f(b), subsequent calls
|
||||
// to Next will return the same segments.
|
||||
// Modifying runes are grouped together with the preceding starter, if such a starter exists.
|
||||
// Although not guaranteed, n will typically be the smallest possible n.
|
||||
func (i *Iter) Next() []byte {
|
||||
return i.next(i)
|
||||
}
|
||||
|
||||
func nextASCIIBytes(i *Iter) []byte {
|
||||
p := i.p + 1
|
||||
if p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
return i.rb.src.bytes[i.p:p]
|
||||
}
|
||||
if i.rb.src.bytes[p] < utf8.RuneSelf {
|
||||
p0 := i.p
|
||||
i.p = p
|
||||
return i.rb.src.bytes[p0:p]
|
||||
}
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
i.next = i.rb.f.nextMain
|
||||
return i.next(i)
|
||||
}
|
||||
|
||||
func nextASCIIString(i *Iter) []byte {
|
||||
p := i.p + 1
|
||||
if p >= i.rb.nsrc {
|
||||
i.buf[0] = i.rb.src.str[i.p]
|
||||
i.setDone()
|
||||
return i.buf[:1]
|
||||
}
|
||||
if i.rb.src.str[p] < utf8.RuneSelf {
|
||||
i.buf[0] = i.rb.src.str[i.p]
|
||||
i.p = p
|
||||
return i.buf[:1]
|
||||
}
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
i.next = i.rb.f.nextMain
|
||||
return i.next(i)
|
||||
}
|
||||
|
||||
func nextHangul(i *Iter) []byte {
|
||||
p := i.p
|
||||
next := p + hangulUTF8Size
|
||||
if next >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
} else if i.rb.src.hangul(next) == 0 {
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
i.next = i.rb.f.nextMain
|
||||
return i.next(i)
|
||||
}
|
||||
i.p = next
|
||||
return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
|
||||
}
|
||||
|
||||
func nextDone(i *Iter) []byte {
|
||||
return nil
|
||||
}
|
||||
|
||||
// nextMulti is used for iterating over multi-segment decompositions
|
||||
// for decomposing normal forms.
|
||||
func nextMulti(i *Iter) []byte {
|
||||
j := 0
|
||||
d := i.multiSeg
|
||||
// skip first rune
|
||||
for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
|
||||
}
|
||||
for j < len(d) {
|
||||
info := i.rb.f.info(input{bytes: d}, j)
|
||||
if info.BoundaryBefore() {
|
||||
i.multiSeg = d[j:]
|
||||
return d[:j]
|
||||
}
|
||||
j += int(info.size)
|
||||
}
|
||||
// treat last segment as normal decomposition
|
||||
i.next = i.rb.f.nextMain
|
||||
return i.next(i)
|
||||
}
|
||||
|
||||
// nextMultiNorm is used for iterating over multi-segment decompositions
|
||||
// for composing normal forms.
|
||||
func nextMultiNorm(i *Iter) []byte {
|
||||
j := 0
|
||||
d := i.multiSeg
|
||||
for j < len(d) {
|
||||
info := i.rb.f.info(input{bytes: d}, j)
|
||||
if info.BoundaryBefore() {
|
||||
i.rb.compose()
|
||||
seg := i.buf[:i.rb.flushCopy(i.buf[:])]
|
||||
i.rb.ss.first(info)
|
||||
i.rb.insertUnsafe(input{bytes: d}, j, info)
|
||||
i.multiSeg = d[j+int(info.size):]
|
||||
return seg
|
||||
}
|
||||
i.rb.ss.next(info)
|
||||
i.rb.insertUnsafe(input{bytes: d}, j, info)
|
||||
j += int(info.size)
|
||||
}
|
||||
i.multiSeg = nil
|
||||
i.next = nextComposed
|
||||
return doNormComposed(i)
|
||||
}
|
||||
|
||||
// nextDecomposed is the implementation of Next for forms NFD and NFKD.
|
||||
func nextDecomposed(i *Iter) (next []byte) {
|
||||
outp := 0
|
||||
inCopyStart, outCopyStart := i.p, 0
|
||||
ss := mkStreamSafe(i.info)
|
||||
for {
|
||||
if sz := int(i.info.size); sz <= 1 {
|
||||
p := i.p
|
||||
i.p++ // ASCII or illegal byte. Either way, advance by 1.
|
||||
if i.p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
return i.returnSlice(p, i.p)
|
||||
} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
|
||||
i.next = i.asciiF
|
||||
return i.returnSlice(p, i.p)
|
||||
}
|
||||
outp++
|
||||
} else if d := i.info.Decomposition(); d != nil {
|
||||
// Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
|
||||
// Case 1: there is a leftover to copy. In this case the decomposition
|
||||
// must begin with a modifier and should always be appended.
|
||||
// Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
|
||||
p := outp + len(d)
|
||||
if outp > 0 {
|
||||
i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
|
||||
if p > len(i.buf) {
|
||||
return i.buf[:outp]
|
||||
}
|
||||
} else if i.info.multiSegment() {
|
||||
// outp must be 0 as multi-segment decompositions always
|
||||
// start a new segment.
|
||||
if i.multiSeg == nil {
|
||||
i.multiSeg = d
|
||||
i.next = nextMulti
|
||||
return nextMulti(i)
|
||||
}
|
||||
// We are in the last segment. Treat as normal decomposition.
|
||||
d = i.multiSeg
|
||||
i.multiSeg = nil
|
||||
p = len(d)
|
||||
}
|
||||
prevCC := i.info.tccc
|
||||
if i.p += sz; i.p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
i.info = Properties{} // Force BoundaryBefore to succeed.
|
||||
} else {
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
}
|
||||
switch ss.next(i.info) {
|
||||
case ssOverflow:
|
||||
i.next = nextCGJDecompose
|
||||
fallthrough
|
||||
case ssStarter:
|
||||
if outp > 0 {
|
||||
copy(i.buf[outp:], d)
|
||||
return i.buf[:p]
|
||||
}
|
||||
return d
|
||||
}
|
||||
copy(i.buf[outp:], d)
|
||||
outp = p
|
||||
inCopyStart, outCopyStart = i.p, outp
|
||||
if i.info.ccc < prevCC {
|
||||
goto doNorm
|
||||
}
|
||||
continue
|
||||
} else if r := i.rb.src.hangul(i.p); r != 0 {
|
||||
outp = decomposeHangul(i.buf[:], r)
|
||||
i.p += hangulUTF8Size
|
||||
inCopyStart, outCopyStart = i.p, outp
|
||||
if i.p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
break
|
||||
} else if i.rb.src.hangul(i.p) != 0 {
|
||||
i.next = nextHangul
|
||||
return i.buf[:outp]
|
||||
}
|
||||
} else {
|
||||
p := outp + sz
|
||||
if p > len(i.buf) {
|
||||
break
|
||||
}
|
||||
outp = p
|
||||
i.p += sz
|
||||
}
|
||||
if i.p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
break
|
||||
}
|
||||
prevCC := i.info.tccc
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
if v := ss.next(i.info); v == ssStarter {
|
||||
break
|
||||
} else if v == ssOverflow {
|
||||
i.next = nextCGJDecompose
|
||||
break
|
||||
}
|
||||
if i.info.ccc < prevCC {
|
||||
goto doNorm
|
||||
}
|
||||
}
|
||||
if outCopyStart == 0 {
|
||||
return i.returnSlice(inCopyStart, i.p)
|
||||
} else if inCopyStart < i.p {
|
||||
i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
|
||||
}
|
||||
return i.buf[:outp]
|
||||
doNorm:
|
||||
// Insert what we have decomposed so far in the reorderBuffer.
|
||||
// As we will only reorder, there will always be enough room.
|
||||
i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
|
||||
i.rb.insertDecomposed(i.buf[0:outp])
|
||||
return doNormDecomposed(i)
|
||||
}
|
||||
|
||||
func doNormDecomposed(i *Iter) []byte {
|
||||
for {
|
||||
if s := i.rb.ss.next(i.info); s == ssOverflow {
|
||||
i.next = nextCGJDecompose
|
||||
break
|
||||
}
|
||||
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
|
||||
if i.p += int(i.info.size); i.p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
break
|
||||
}
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
if i.info.ccc == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
// new segment or too many combining characters: exit normalization
|
||||
return i.buf[:i.rb.flushCopy(i.buf[:])]
|
||||
}
|
||||
|
||||
func nextCGJDecompose(i *Iter) []byte {
|
||||
i.rb.ss = 0
|
||||
i.rb.insertCGJ()
|
||||
i.next = nextDecomposed
|
||||
buf := doNormDecomposed(i)
|
||||
return buf
|
||||
}
|
||||
|
||||
// nextComposed is the implementation of Next for forms NFC and NFKC.
|
||||
func nextComposed(i *Iter) []byte {
|
||||
outp, startp := 0, i.p
|
||||
var prevCC uint8
|
||||
ss := mkStreamSafe(i.info)
|
||||
for {
|
||||
if !i.info.isYesC() {
|
||||
goto doNorm
|
||||
}
|
||||
prevCC = i.info.tccc
|
||||
sz := int(i.info.size)
|
||||
if sz == 0 {
|
||||
sz = 1 // illegal rune: copy byte-by-byte
|
||||
}
|
||||
p := outp + sz
|
||||
if p > len(i.buf) {
|
||||
break
|
||||
}
|
||||
outp = p
|
||||
i.p += sz
|
||||
if i.p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
break
|
||||
} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
|
||||
i.next = i.asciiF
|
||||
break
|
||||
}
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
if v := ss.next(i.info); v == ssStarter {
|
||||
break
|
||||
} else if v == ssOverflow {
|
||||
i.next = nextCGJCompose
|
||||
break
|
||||
}
|
||||
if i.info.ccc < prevCC {
|
||||
goto doNorm
|
||||
}
|
||||
}
|
||||
return i.returnSlice(startp, i.p)
|
||||
doNorm:
|
||||
i.p = startp
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
if i.info.multiSegment() {
|
||||
d := i.info.Decomposition()
|
||||
info := i.rb.f.info(input{bytes: d}, 0)
|
||||
i.rb.insertUnsafe(input{bytes: d}, 0, info)
|
||||
i.multiSeg = d[int(info.size):]
|
||||
i.next = nextMultiNorm
|
||||
return nextMultiNorm(i)
|
||||
}
|
||||
i.rb.ss.first(i.info)
|
||||
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
|
||||
return doNormComposed(i)
|
||||
}
|
||||
|
||||
func doNormComposed(i *Iter) []byte {
|
||||
// First rune should already be inserted.
|
||||
for {
|
||||
if i.p += int(i.info.size); i.p >= i.rb.nsrc {
|
||||
i.setDone()
|
||||
break
|
||||
}
|
||||
i.info = i.rb.f.info(i.rb.src, i.p)
|
||||
if s := i.rb.ss.next(i.info); s == ssStarter {
|
||||
break
|
||||
} else if s == ssOverflow {
|
||||
i.next = nextCGJCompose
|
||||
break
|
||||
}
|
||||
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
|
||||
}
|
||||
i.rb.compose()
|
||||
seg := i.buf[:i.rb.flushCopy(i.buf[:])]
|
||||
return seg
|
||||
}
|
||||
|
||||
func nextCGJCompose(i *Iter) []byte {
|
||||
i.rb.ss = 0 // instead of first
|
||||
i.rb.insertCGJ()
|
||||
i.next = nextComposed
|
||||
// Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
|
||||
// even if they are not. This is particularly dubious for U+FF9E and UFF9A.
|
||||
// If we ever change that, insert a check here.
|
||||
i.rb.ss.first(i.info)
|
||||
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
|
||||
return doNormComposed(i)
|
||||
}
|
||||
-1143
File diff suppressed because it is too large
Load Diff
-45
@@ -1,45 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// Generate test data for trie code.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
printTestTables()
|
||||
}
|
||||
|
||||
// We take the smallest, largest and an arbitrary value for each
|
||||
// of the UTF-8 sequence lengths.
|
||||
var testRunes = []rune{
|
||||
0x01, 0x0C, 0x7F, // 1-byte sequences
|
||||
0x80, 0x100, 0x7FF, // 2-byte sequences
|
||||
0x800, 0x999, 0xFFFF, // 3-byte sequences
|
||||
0x10000, 0x10101, 0x10FFFF, // 4-byte sequences
|
||||
0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block
|
||||
}
|
||||
|
||||
const fileHeader = `// Generated by running
|
||||
// maketesttables
|
||||
// DO NOT EDIT
|
||||
|
||||
package norm
|
||||
|
||||
`
|
||||
|
||||
func printTestTables() {
|
||||
fmt.Print(fileHeader)
|
||||
fmt.Printf("var testRunes = %#v\n\n", testRunes)
|
||||
t := newNode()
|
||||
for i, r := range testRunes {
|
||||
t.insert(r, uint16(i))
|
||||
}
|
||||
t.printTables("testdata")
|
||||
}
|
||||
-318
@@ -1,318 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"code.google.com/p/go.text/unicode/norm"
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
loadTestData()
|
||||
CharacterByCharacterTests()
|
||||
StandardTests()
|
||||
PerformanceTest()
|
||||
if errorCount == 0 {
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
}
|
||||
|
||||
const file = "NormalizationTest.txt"
|
||||
|
||||
var url = flag.String("url",
|
||||
"http://www.unicode.org/Public/"+unicode.Version+"/ucd/"+file,
|
||||
"URL of Unicode database directory")
|
||||
var localFiles = flag.Bool("local",
|
||||
false,
|
||||
"data files have been copied to the current directory; for debugging only")
|
||||
|
||||
var logger = log.New(os.Stderr, "", log.Lshortfile)
|
||||
|
||||
// This regression test runs the test set in NormalizationTest.txt
|
||||
// (taken from http://www.unicode.org/Public/<unicode.Version>/ucd/).
|
||||
//
|
||||
// NormalizationTest.txt has form:
|
||||
// @Part0 # Specific cases
|
||||
// #
|
||||
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
|
||||
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
|
||||
//
|
||||
// Each test has 5 columns (c1, c2, c3, c4, c5), where
|
||||
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
|
||||
//
|
||||
// CONFORMANCE:
|
||||
// 1. The following invariants must be true for all conformant implementations
|
||||
//
|
||||
// NFC
|
||||
// c2 == NFC(c1) == NFC(c2) == NFC(c3)
|
||||
// c4 == NFC(c4) == NFC(c5)
|
||||
//
|
||||
// NFD
|
||||
// c3 == NFD(c1) == NFD(c2) == NFD(c3)
|
||||
// c5 == NFD(c4) == NFD(c5)
|
||||
//
|
||||
// NFKC
|
||||
// c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
|
||||
//
|
||||
// NFKD
|
||||
// c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
|
||||
//
|
||||
// 2. For every code point X assigned in this version of Unicode that is not
|
||||
// specifically listed in Part 1, the following invariants must be true
|
||||
// for all conformant implementations:
|
||||
//
|
||||
// X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
|
||||
//
|
||||
|
||||
// Column types.
|
||||
const (
|
||||
cRaw = iota
|
||||
cNFC
|
||||
cNFD
|
||||
cNFKC
|
||||
cNFKD
|
||||
cMaxColumns
|
||||
)
|
||||
|
||||
// Holds data from NormalizationTest.txt
|
||||
var part []Part
|
||||
|
||||
type Part struct {
|
||||
name string
|
||||
number int
|
||||
tests []Test
|
||||
}
|
||||
|
||||
type Test struct {
|
||||
name string
|
||||
partnr int
|
||||
number int
|
||||
r rune // used for character by character test
|
||||
cols [cMaxColumns]string // Each has 5 entries, see below.
|
||||
}
|
||||
|
||||
func (t Test) Name() string {
|
||||
if t.number < 0 {
|
||||
return part[t.partnr].name
|
||||
}
|
||||
return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
|
||||
}
|
||||
|
||||
var partRe = regexp.MustCompile(`@Part(\d) # (.*)$`)
|
||||
var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)$`)
|
||||
|
||||
var counter int
|
||||
|
||||
// Load the data form NormalizationTest.txt
|
||||
func loadTestData() {
|
||||
if *localFiles {
|
||||
pwd, _ := os.Getwd()
|
||||
*url = "file://" + path.Join(pwd, file)
|
||||
}
|
||||
t := &http.Transport{}
|
||||
t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
|
||||
c := &http.Client{Transport: t}
|
||||
resp, err := c.Get(*url)
|
||||
if err != nil {
|
||||
logger.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
logger.Fatal("bad GET status for "+file, resp.Status)
|
||||
}
|
||||
f := resp.Body
|
||||
defer f.Close()
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if len(line) == 0 || line[0] == '#' {
|
||||
continue
|
||||
}
|
||||
m := partRe.FindStringSubmatch(line)
|
||||
if m != nil {
|
||||
if len(m) < 3 {
|
||||
logger.Fatal("Failed to parse Part: ", line)
|
||||
}
|
||||
i, err := strconv.Atoi(m[1])
|
||||
if err != nil {
|
||||
logger.Fatal(err)
|
||||
}
|
||||
name := m[2]
|
||||
part = append(part, Part{name: name[:len(name)-1], number: i})
|
||||
continue
|
||||
}
|
||||
m = testRe.FindStringSubmatch(line)
|
||||
if m == nil || len(m) < 7 {
|
||||
logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
|
||||
}
|
||||
test := Test{name: m[6], partnr: len(part) - 1, number: counter}
|
||||
counter++
|
||||
for j := 1; j < len(m)-1; j++ {
|
||||
for _, split := range strings.Split(m[j], " ") {
|
||||
r, err := strconv.ParseUint(split, 16, 64)
|
||||
if err != nil {
|
||||
logger.Fatal(err)
|
||||
}
|
||||
if test.r == 0 {
|
||||
// save for CharacterByCharacterTests
|
||||
test.r = rune(r)
|
||||
}
|
||||
var buf [utf8.UTFMax]byte
|
||||
sz := utf8.EncodeRune(buf[:], rune(r))
|
||||
test.cols[j-1] += string(buf[:sz])
|
||||
}
|
||||
}
|
||||
part := &part[len(part)-1]
|
||||
part.tests = append(part.tests, test)
|
||||
}
|
||||
if scanner.Err() != nil {
|
||||
logger.Fatal(scanner.Err())
|
||||
}
|
||||
}
|
||||
|
||||
var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}
|
||||
|
||||
var errorCount int
|
||||
|
||||
func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
|
||||
if gold != result {
|
||||
errorCount++
|
||||
if errorCount > 20 {
|
||||
return
|
||||
}
|
||||
logger.Printf("%s:%s: %s(%+q)=%+q; want %+q: %s",
|
||||
t.Name(), name, fstr[f], test, result, gold, t.name)
|
||||
}
|
||||
}
|
||||
|
||||
func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
|
||||
if result != want {
|
||||
errorCount++
|
||||
if errorCount > 20 {
|
||||
return
|
||||
}
|
||||
logger.Printf("%s:%s: %s(%+q)=%v; want %v", t.Name(), name, fstr[f], test, result, want)
|
||||
}
|
||||
}
|
||||
|
||||
func doTest(t *Test, f norm.Form, gold, test string) {
|
||||
testb := []byte(test)
|
||||
result := f.Bytes(testb)
|
||||
cmpResult(t, "Bytes", f, gold, test, string(result))
|
||||
|
||||
sresult := f.String(test)
|
||||
cmpResult(t, "String", f, gold, test, sresult)
|
||||
|
||||
acc := []byte{}
|
||||
i := norm.Iter{}
|
||||
i.InitString(f, test)
|
||||
for !i.Done() {
|
||||
acc = append(acc, i.Next()...)
|
||||
}
|
||||
cmpResult(t, "Iter.Next", f, gold, test, string(acc))
|
||||
|
||||
buf := make([]byte, 128)
|
||||
acc = nil
|
||||
for p := 0; p < len(testb); {
|
||||
nDst, nSrc, _ := f.Transform(buf, testb[p:], true)
|
||||
acc = append(acc, buf[:nDst]...)
|
||||
p += nSrc
|
||||
}
|
||||
cmpResult(t, "Transform", f, gold, test, string(acc))
|
||||
|
||||
for i := range test {
|
||||
out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
|
||||
cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
|
||||
}
|
||||
cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
|
||||
cmpIsNormal(t, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
|
||||
}
|
||||
|
||||
func doConformanceTests(t *Test, partn int) {
|
||||
for i := 0; i <= 2; i++ {
|
||||
doTest(t, norm.NFC, t.cols[1], t.cols[i])
|
||||
doTest(t, norm.NFD, t.cols[2], t.cols[i])
|
||||
doTest(t, norm.NFKC, t.cols[3], t.cols[i])
|
||||
doTest(t, norm.NFKD, t.cols[4], t.cols[i])
|
||||
}
|
||||
for i := 3; i <= 4; i++ {
|
||||
doTest(t, norm.NFC, t.cols[3], t.cols[i])
|
||||
doTest(t, norm.NFD, t.cols[4], t.cols[i])
|
||||
doTest(t, norm.NFKC, t.cols[3], t.cols[i])
|
||||
doTest(t, norm.NFKD, t.cols[4], t.cols[i])
|
||||
}
|
||||
}
|
||||
|
||||
func CharacterByCharacterTests() {
|
||||
tests := part[1].tests
|
||||
var last rune = 0
|
||||
for i := 0; i <= len(tests); i++ { // last one is special case
|
||||
var r rune
|
||||
if i == len(tests) {
|
||||
r = 0x2FA1E // Don't have to go to 0x10FFFF
|
||||
} else {
|
||||
r = tests[i].r
|
||||
}
|
||||
for last++; last < r; last++ {
|
||||
// Check all characters that were not explicitly listed in the test.
|
||||
t := &Test{partnr: 1, number: -1}
|
||||
char := string(last)
|
||||
doTest(t, norm.NFC, char, char)
|
||||
doTest(t, norm.NFD, char, char)
|
||||
doTest(t, norm.NFKC, char, char)
|
||||
doTest(t, norm.NFKD, char, char)
|
||||
}
|
||||
if i < len(tests) {
|
||||
doConformanceTests(&tests[i], 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func StandardTests() {
|
||||
for _, j := range []int{0, 2, 3} {
|
||||
for _, test := range part[j].tests {
|
||||
doConformanceTests(&test, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PerformanceTest verifies that normalization is O(n). If any of the
|
||||
// code does not properly check for maxCombiningChars, normalization
|
||||
// may exhibit O(n**2) behavior.
|
||||
func PerformanceTest() {
|
||||
runtime.GOMAXPROCS(2)
|
||||
success := make(chan bool, 1)
|
||||
go func() {
|
||||
buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
|
||||
buf = append(buf, "\u035B"...)
|
||||
norm.NFC.Append(nil, buf...)
|
||||
success <- true
|
||||
}()
|
||||
timeout := time.After(1 * time.Second)
|
||||
select {
|
||||
case <-success:
|
||||
// test completed before the timeout
|
||||
case <-timeout:
|
||||
errorCount++
|
||||
logger.Printf(`unexpectedly long time to complete PerformanceTest`)
|
||||
}
|
||||
}
|
||||
-6989
File diff suppressed because it is too large
Load Diff
-85
@@ -1,85 +0,0 @@
|
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
|
||||
"code.google.com/p/go.text/transform"
|
||||
)
|
||||
|
||||
// Transform implements the transform.Transformer interface. It may need to
|
||||
// write segments of up to MaxSegmentSize at once. Users should either catch
|
||||
// ErrShortDst and allow dst to grow or have dst be at least of size
|
||||
// MaxTransformChunkSize to be guaranteed of progress.
|
||||
func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := 0
|
||||
// Cap the maximum number of src bytes to check.
|
||||
b := src
|
||||
eof := atEOF
|
||||
if ns := len(dst); ns < len(b) {
|
||||
err = transform.ErrShortDst
|
||||
eof = false
|
||||
b = b[:ns]
|
||||
}
|
||||
i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof)
|
||||
n += copy(dst[n:], b[n:i])
|
||||
if !ok {
|
||||
nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF)
|
||||
return nDst + n, nSrc + n, err
|
||||
}
|
||||
if n < len(src) && !atEOF {
|
||||
err = transform.ErrShortSrc
|
||||
}
|
||||
return n, n, err
|
||||
}
|
||||
|
||||
func flushTransform(rb *reorderBuffer) bool {
|
||||
// Write out (must fully fit in dst, or else it is a ErrShortDst).
|
||||
if len(rb.out) < rb.nrune*utf8.UTFMax {
|
||||
return false
|
||||
}
|
||||
rb.out = rb.out[rb.flushCopy(rb.out):]
|
||||
return true
|
||||
}
|
||||
|
||||
var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc}
|
||||
|
||||
// transform implements the transform.Transformer interface. It is only called
|
||||
// when quickSpan does not pass for a given string.
|
||||
func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
// TODO: get rid of reorderBuffer. See CL 23460044.
|
||||
rb := reorderBuffer{}
|
||||
rb.init(f, src)
|
||||
for {
|
||||
// Load segment into reorder buffer.
|
||||
rb.setFlusher(dst[nDst:], flushTransform)
|
||||
end := decomposeSegment(&rb, nSrc, atEOF)
|
||||
if end < 0 {
|
||||
return nDst, nSrc, errs[-end]
|
||||
}
|
||||
nDst = len(dst) - len(rb.out)
|
||||
nSrc = end
|
||||
|
||||
// Next quickSpan.
|
||||
end = rb.nsrc
|
||||
eof := atEOF
|
||||
if n := nSrc + len(dst) - nDst; n < end {
|
||||
err = transform.ErrShortDst
|
||||
end = n
|
||||
eof = false
|
||||
}
|
||||
end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof)
|
||||
n := copy(dst[nDst:], rb.src.bytes[nSrc:end])
|
||||
nSrc += n
|
||||
nDst += n
|
||||
if ok {
|
||||
if n < rb.nsrc && !atEOF {
|
||||
err = transform.ErrShortSrc
|
||||
}
|
||||
return nDst, nSrc, err
|
||||
}
|
||||
}
|
||||
}
|
||||
-101
@@ -1,101 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"code.google.com/p/go.text/transform"
|
||||
)
|
||||
|
||||
func TestTransform(t *testing.T) {
|
||||
tests := []struct {
|
||||
f Form
|
||||
in, out string
|
||||
eof bool
|
||||
dstSize int
|
||||
err error
|
||||
}{
|
||||
{NFC, "ab", "ab", true, 2, nil},
|
||||
{NFC, "qx", "qx", true, 2, nil},
|
||||
{NFD, "qx", "qx", true, 2, nil},
|
||||
{NFC, "", "", true, 1, nil},
|
||||
{NFD, "", "", true, 1, nil},
|
||||
{NFC, "", "", false, 1, nil},
|
||||
{NFD, "", "", false, 1, nil},
|
||||
|
||||
// Normalized segment does not fit in destination.
|
||||
{NFD, "ö", "", true, 1, transform.ErrShortDst},
|
||||
{NFD, "ö", "", true, 2, transform.ErrShortDst},
|
||||
|
||||
// As an artifact of the algorithm, only full segments are written.
|
||||
// This is not strictly required, and some bytes could be written.
|
||||
// In practice, for Transform to not block, the destination buffer
|
||||
// should be at least MaxSegmentSize to work anyway and these edge
|
||||
// conditions will be relatively rare.
|
||||
{NFC, "ab", "", true, 1, transform.ErrShortDst},
|
||||
// This is even true for inert runes.
|
||||
{NFC, "qx", "", true, 1, transform.ErrShortDst},
|
||||
{NFC, "a\u0300abc", "\u00e0a", true, 4, transform.ErrShortDst},
|
||||
|
||||
// We cannot write a segment if succesive runes could still change the result.
|
||||
{NFD, "ö", "", false, 3, transform.ErrShortSrc},
|
||||
{NFC, "a\u0300", "", false, 4, transform.ErrShortSrc},
|
||||
{NFD, "a\u0300", "", false, 4, transform.ErrShortSrc},
|
||||
{NFC, "ö", "", false, 3, transform.ErrShortSrc},
|
||||
|
||||
{NFC, "a\u0300", "", true, 1, transform.ErrShortDst},
|
||||
// Theoretically could fit, but won't due to simplified checks.
|
||||
{NFC, "a\u0300", "", true, 2, transform.ErrShortDst},
|
||||
{NFC, "a\u0300", "", true, 3, transform.ErrShortDst},
|
||||
{NFC, "a\u0300", "\u00e0", true, 4, nil},
|
||||
|
||||
{NFD, "öa\u0300", "o\u0308", false, 8, transform.ErrShortSrc},
|
||||
{NFD, "öa\u0300ö", "o\u0308a\u0300", true, 8, transform.ErrShortDst},
|
||||
{NFD, "öa\u0300ö", "o\u0308a\u0300", false, 12, transform.ErrShortSrc},
|
||||
|
||||
// Illegal input is copied verbatim.
|
||||
{NFD, "\xbd\xb2=\xbc ", "\xbd\xb2=\xbc ", true, 8, nil},
|
||||
}
|
||||
b := make([]byte, 100)
|
||||
for i, tt := range tests {
|
||||
nDst, _, err := tt.f.Transform(b[:tt.dstSize], []byte(tt.in), tt.eof)
|
||||
out := string(b[:nDst])
|
||||
if out != tt.out || err != tt.err {
|
||||
t.Errorf("%d: was %+q (%v); want %+q (%v)", i, out, err, tt.out, tt.err)
|
||||
}
|
||||
if want := tt.f.String(tt.in)[:nDst]; want != out {
|
||||
t.Errorf("%d: incorect normalization: was %+q; want %+q", i, out, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var transBufSizes = []int{
|
||||
MaxTransformChunkSize,
|
||||
3 * MaxTransformChunkSize / 2,
|
||||
2 * MaxTransformChunkSize,
|
||||
3 * MaxTransformChunkSize,
|
||||
100 * MaxTransformChunkSize,
|
||||
}
|
||||
|
||||
func doTransNorm(f Form, buf []byte, b []byte) []byte {
|
||||
acc := []byte{}
|
||||
for p := 0; p < len(b); {
|
||||
nd, ns, _ := f.Transform(buf[:], b[p:], true)
|
||||
p += ns
|
||||
acc = append(acc, buf[:nd]...)
|
||||
}
|
||||
return acc
|
||||
}
|
||||
|
||||
func TestTransformNorm(t *testing.T) {
|
||||
for _, sz := range transBufSizes {
|
||||
buf := make([]byte, sz)
|
||||
runNormTests(t, fmt.Sprintf("Transform:%d", sz), func(f Form, out []byte, s string) []byte {
|
||||
return doTransNorm(f, buf, append(out, s...))
|
||||
})
|
||||
}
|
||||
}
|
||||
-232
@@ -1,232 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
type valueRange struct {
|
||||
value uint16 // header: value:stride
|
||||
lo, hi byte // header: lo:n
|
||||
}
|
||||
|
||||
type trie struct {
|
||||
index []uint8
|
||||
values []uint16
|
||||
sparse []valueRange
|
||||
sparseOffset []uint16
|
||||
cutoff uint8 // indices >= cutoff are sparse
|
||||
}
|
||||
|
||||
// lookupValue determines the type of block n and looks up the value for b.
|
||||
// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
|
||||
// is a list of ranges with an accompanying value. Given a matching range r,
|
||||
// the value for b is by r.value + (b - r.lo) * stride.
|
||||
func (t *trie) lookupValue(n uint8, b byte) uint16 {
|
||||
if n < t.cutoff {
|
||||
return t.values[uint16(n)<<6+uint16(b)]
|
||||
}
|
||||
offset := t.sparseOffset[n-t.cutoff]
|
||||
header := t.sparse[offset]
|
||||
lo := offset + 1
|
||||
hi := lo + uint16(header.lo)
|
||||
for lo < hi {
|
||||
m := lo + (hi-lo)/2
|
||||
r := t.sparse[m]
|
||||
if r.lo <= b && b <= r.hi {
|
||||
return r.value + uint16(b-r.lo)*header.value
|
||||
}
|
||||
if b < r.lo {
|
||||
hi = m
|
||||
} else {
|
||||
lo = m + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
const (
|
||||
t1 = 0x00 // 0000 0000
|
||||
tx = 0x80 // 1000 0000
|
||||
t2 = 0xC0 // 1100 0000
|
||||
t3 = 0xE0 // 1110 0000
|
||||
t4 = 0xF0 // 1111 0000
|
||||
t5 = 0xF8 // 1111 1000
|
||||
t6 = 0xFC // 1111 1100
|
||||
te = 0xFE // 1111 1110
|
||||
)
|
||||
|
||||
// lookup returns the trie value for the first UTF-8 encoding in s and
|
||||
// the width in bytes of this encoding. The size will be 0 if s does not
|
||||
// hold enough bytes to complete the encoding. len(s) must be greater than 0.
|
||||
func (t *trie) lookup(s []byte) (v uint16, sz int) {
|
||||
c0 := s[0]
|
||||
switch {
|
||||
case c0 < tx:
|
||||
return t.values[c0], 1
|
||||
case c0 < t2:
|
||||
return 0, 1
|
||||
case c0 < t3:
|
||||
if len(s) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
return t.lookupValue(i, c1), 2
|
||||
case c0 < t4:
|
||||
if len(s) < 3 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
return t.lookupValue(i, c2), 3
|
||||
case c0 < t5:
|
||||
if len(s) < 4 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
o = uint16(i)<<6 + uint16(c2)
|
||||
i = t.index[o]
|
||||
c3 := s[3]
|
||||
if c3 < tx || t2 <= c3 {
|
||||
return 0, 3
|
||||
}
|
||||
return t.lookupValue(i, c3), 4
|
||||
}
|
||||
// Illegal rune
|
||||
return 0, 1
|
||||
}
|
||||
|
||||
// lookupString returns the trie value for the first UTF-8 encoding in s and
|
||||
// the width in bytes of this encoding. The size will be 0 if s does not
|
||||
// hold enough bytes to complete the encoding. len(s) must be greater than 0.
|
||||
func (t *trie) lookupString(s string) (v uint16, sz int) {
|
||||
c0 := s[0]
|
||||
switch {
|
||||
case c0 < tx:
|
||||
return t.values[c0], 1
|
||||
case c0 < t2:
|
||||
return 0, 1
|
||||
case c0 < t3:
|
||||
if len(s) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
return t.lookupValue(i, c1), 2
|
||||
case c0 < t4:
|
||||
if len(s) < 3 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
return t.lookupValue(i, c2), 3
|
||||
case c0 < t5:
|
||||
if len(s) < 4 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
o = uint16(i)<<6 + uint16(c2)
|
||||
i = t.index[o]
|
||||
c3 := s[3]
|
||||
if c3 < tx || t2 <= c3 {
|
||||
return 0, 3
|
||||
}
|
||||
return t.lookupValue(i, c3), 4
|
||||
}
|
||||
// Illegal rune
|
||||
return 0, 1
|
||||
}
|
||||
|
||||
// lookupUnsafe returns the trie value for the first UTF-8 encoding in s.
|
||||
// s must hold a full encoding.
|
||||
func (t *trie) lookupUnsafe(s []byte) uint16 {
|
||||
c0 := s[0]
|
||||
if c0 < tx {
|
||||
return t.values[c0]
|
||||
}
|
||||
if c0 < t2 {
|
||||
return 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
if c0 < t3 {
|
||||
return t.lookupValue(i, s[1])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[1])]
|
||||
if c0 < t4 {
|
||||
return t.lookupValue(i, s[2])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[2])]
|
||||
if c0 < t5 {
|
||||
return t.lookupValue(i, s[3])
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s.
|
||||
// s must hold a full encoding.
|
||||
func (t *trie) lookupStringUnsafe(s string) uint16 {
|
||||
c0 := s[0]
|
||||
if c0 < tx {
|
||||
return t.values[c0]
|
||||
}
|
||||
if c0 < t2 {
|
||||
return 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
if c0 < t3 {
|
||||
return t.lookupValue(i, s[1])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[1])]
|
||||
if c0 < t4 {
|
||||
return t.lookupValue(i, s[2])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[2])]
|
||||
if c0 < t5 {
|
||||
return t.lookupValue(i, s[3])
|
||||
}
|
||||
return 0
|
||||
}
|
||||
-152
@@ -1,152 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Test data is located in triedata_test.go; generated by maketesttables.
|
||||
var testdata = testdataTrie
|
||||
|
||||
type rangeTest struct {
|
||||
block uint8
|
||||
lookup byte
|
||||
result uint16
|
||||
table []valueRange
|
||||
offsets []uint16
|
||||
}
|
||||
|
||||
var range1Off = []uint16{0, 2}
|
||||
var range1 = []valueRange{
|
||||
{0, 1, 0},
|
||||
{1, 0x80, 0x80},
|
||||
{0, 2, 0},
|
||||
{1, 0x80, 0x80},
|
||||
{9, 0xff, 0xff},
|
||||
}
|
||||
|
||||
var rangeTests = []rangeTest{
|
||||
{10, 0x80, 1, range1, range1Off},
|
||||
{10, 0x00, 0, range1, range1Off},
|
||||
{11, 0x80, 1, range1, range1Off},
|
||||
{11, 0xff, 9, range1, range1Off},
|
||||
{11, 0x00, 0, range1, range1Off},
|
||||
}
|
||||
|
||||
func TestLookupSparse(t *testing.T) {
|
||||
for i, test := range rangeTests {
|
||||
n := trie{sparse: test.table, sparseOffset: test.offsets, cutoff: 10}
|
||||
v := n.lookupValue(test.block, test.lookup)
|
||||
if v != test.result {
|
||||
t.Errorf("LookupSparse:%d: found %X; want %X", i, v, test.result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test cases for illegal runes.
|
||||
type trietest struct {
|
||||
size int
|
||||
bytes []byte
|
||||
}
|
||||
|
||||
var tests = []trietest{
|
||||
// illegal runes
|
||||
{1, []byte{0x80}},
|
||||
{1, []byte{0xFF}},
|
||||
{1, []byte{t2, tx - 1}},
|
||||
{1, []byte{t2, t2}},
|
||||
{2, []byte{t3, tx, tx - 1}},
|
||||
{2, []byte{t3, tx, t2}},
|
||||
{1, []byte{t3, tx - 1, tx}},
|
||||
{3, []byte{t4, tx, tx, tx - 1}},
|
||||
{3, []byte{t4, tx, tx, t2}},
|
||||
{1, []byte{t4, t2, tx, tx - 1}},
|
||||
{2, []byte{t4, tx, t2, tx - 1}},
|
||||
|
||||
// short runes
|
||||
{0, []byte{t2}},
|
||||
{0, []byte{t3, tx}},
|
||||
{0, []byte{t4, tx, tx}},
|
||||
|
||||
// we only support UTF-8 up to utf8.UTFMax bytes (4 bytes)
|
||||
{1, []byte{t5, tx, tx, tx, tx}},
|
||||
{1, []byte{t6, tx, tx, tx, tx, tx}},
|
||||
}
|
||||
|
||||
func mkUTF8(r rune) ([]byte, int) {
|
||||
var b [utf8.UTFMax]byte
|
||||
sz := utf8.EncodeRune(b[:], r)
|
||||
return b[:sz], sz
|
||||
}
|
||||
|
||||
func TestLookup(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, szg := mkUTF8(tt)
|
||||
v, szt := testdata.lookup(b)
|
||||
if int(v) != i {
|
||||
t.Errorf("lookup(%U): found value %#x, expected %#x", tt, v, i)
|
||||
}
|
||||
if szt != szg {
|
||||
t.Errorf("lookup(%U): found size %d, expected %d", tt, szt, szg)
|
||||
}
|
||||
}
|
||||
for i, tt := range tests {
|
||||
v, sz := testdata.lookup(tt.bytes)
|
||||
if v != 0 {
|
||||
t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
|
||||
}
|
||||
if sz != tt.size {
|
||||
t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
|
||||
}
|
||||
}
|
||||
// Verify defaults.
|
||||
if v, _ := testdata.lookup([]byte{0xC1, 0x8C}); v != 0 {
|
||||
t.Errorf("lookup of non-existing rune should be 0; found %X", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupUnsafe(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, _ := mkUTF8(tt)
|
||||
v := testdata.lookupUnsafe(b)
|
||||
if int(v) != i {
|
||||
t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupString(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, szg := mkUTF8(tt)
|
||||
v, szt := testdata.lookupString(string(b))
|
||||
if int(v) != i {
|
||||
t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
|
||||
}
|
||||
if szt != szg {
|
||||
t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
|
||||
}
|
||||
}
|
||||
for i, tt := range tests {
|
||||
v, sz := testdata.lookupString(string(tt.bytes))
|
||||
if int(v) != 0 {
|
||||
t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
|
||||
}
|
||||
if sz != tt.size {
|
||||
t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupStringUnsafe(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, _ := mkUTF8(tt)
|
||||
v := testdata.lookupStringUnsafe(string(b))
|
||||
if int(v) != i {
|
||||
t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
-85
@@ -1,85 +0,0 @@
|
||||
// Generated by running
|
||||
// maketesttables
|
||||
// DO NOT EDIT
|
||||
|
||||
package norm
|
||||
|
||||
var testRunes = []int32{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111, 512, 513, 514, 528, 533}
|
||||
|
||||
// testdataValues: 192 entries, 384 bytes
|
||||
// Block 2 is the null block.
|
||||
var testdataValues = [192]uint16{
|
||||
// Block 0x0, offset 0x0
|
||||
0x000c: 0x0001,
|
||||
// Block 0x1, offset 0x40
|
||||
0x007f: 0x0002,
|
||||
// Block 0x2, offset 0x80
|
||||
}
|
||||
|
||||
// testdataSparseOffset: 10 entries, 20 bytes
|
||||
var testdataSparseOffset = []uint16{0x0, 0x2, 0x4, 0x8, 0xa, 0xc, 0xe, 0x10, 0x12, 0x14}
|
||||
|
||||
// testdataSparseValues: 22 entries, 88 bytes
|
||||
var testdataSparseValues = [22]valueRange{
|
||||
// Block 0x0, offset 0x1
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0003, lo: 0x80, hi: 0x80},
|
||||
// Block 0x1, offset 0x2
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0004, lo: 0x80, hi: 0x80},
|
||||
// Block 0x2, offset 0x3
|
||||
{value: 0x0001, lo: 0x03},
|
||||
{value: 0x000c, lo: 0x80, hi: 0x82},
|
||||
{value: 0x000f, lo: 0x90, hi: 0x90},
|
||||
{value: 0x0010, lo: 0x95, hi: 0x95},
|
||||
// Block 0x3, offset 0x4
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0005, lo: 0xbf, hi: 0xbf},
|
||||
// Block 0x4, offset 0x5
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0006, lo: 0x80, hi: 0x80},
|
||||
// Block 0x5, offset 0x6
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0007, lo: 0x99, hi: 0x99},
|
||||
// Block 0x6, offset 0x7
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0008, lo: 0xbf, hi: 0xbf},
|
||||
// Block 0x7, offset 0x8
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0009, lo: 0x80, hi: 0x80},
|
||||
// Block 0x8, offset 0x9
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x000a, lo: 0x81, hi: 0x81},
|
||||
// Block 0x9, offset 0xa
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x000b, lo: 0xbf, hi: 0xbf},
|
||||
}
|
||||
|
||||
// testdataLookup: 640 bytes
|
||||
// Block 0 is the null block.
|
||||
var testdataLookup = [640]uint8{
|
||||
// Block 0x0, offset 0x0
|
||||
// Block 0x1, offset 0x40
|
||||
// Block 0x2, offset 0x80
|
||||
// Block 0x3, offset 0xc0
|
||||
0x0c2: 0x01, 0x0c4: 0x02,
|
||||
0x0c8: 0x03,
|
||||
0x0df: 0x04,
|
||||
0x0e0: 0x02,
|
||||
0x0ef: 0x03,
|
||||
0x0f0: 0x05, 0x0f4: 0x07,
|
||||
// Block 0x4, offset 0x100
|
||||
0x120: 0x05, 0x126: 0x06,
|
||||
// Block 0x5, offset 0x140
|
||||
0x17f: 0x07,
|
||||
// Block 0x6, offset 0x180
|
||||
0x180: 0x08, 0x184: 0x09,
|
||||
// Block 0x7, offset 0x1c0
|
||||
0x1d0: 0x04,
|
||||
// Block 0x8, offset 0x200
|
||||
0x23f: 0x0a,
|
||||
// Block 0x9, offset 0x240
|
||||
0x24f: 0x06,
|
||||
}
|
||||
|
||||
var testdataTrie = trie{testdataLookup[:], testdataValues[:], testdataSparseValues[:], testdataSparseOffset[:], 1}
|
||||
-317
@@ -1,317 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// Trie table generator.
|
||||
// Used by make*tables tools to generate a go file with trie data structures
|
||||
// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
|
||||
// sequence are used to lookup offsets in the index table to be used for the
|
||||
// next byte. The last byte is used to index into a table with 16-bit values.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"log"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
blockSize = 64
|
||||
blockOffset = 2 // Subtract two blocks to compensate for the 0x80 added to continuation bytes.
|
||||
maxSparseEntries = 16
|
||||
)
|
||||
|
||||
// Intermediate trie structure
|
||||
type trieNode struct {
|
||||
table [256]*trieNode
|
||||
value int
|
||||
b byte
|
||||
leaf bool
|
||||
}
|
||||
|
||||
func newNode() *trieNode {
|
||||
return new(trieNode)
|
||||
}
|
||||
|
||||
func (n trieNode) String() string {
|
||||
s := fmt.Sprint("trieNode{table: { non-nil at index: ")
|
||||
for i, v := range n.table {
|
||||
if v != nil {
|
||||
s += fmt.Sprintf("%d, ", i)
|
||||
}
|
||||
}
|
||||
s += fmt.Sprintf("}, value:%#x, b:%#x leaf:%v}", n.value, n.b, n.leaf)
|
||||
return s
|
||||
}
|
||||
|
||||
func (n trieNode) isInternal() bool {
|
||||
internal := true
|
||||
for i := 0; i < 256; i++ {
|
||||
if nn := n.table[i]; nn != nil {
|
||||
if !internal && !nn.leaf {
|
||||
log.Fatalf("triegen: isInternal: node contains both leaf and non-leaf children (%v)", n)
|
||||
}
|
||||
internal = internal && !nn.leaf
|
||||
}
|
||||
}
|
||||
return internal
|
||||
}
|
||||
|
||||
func (n trieNode) mostFrequentStride() int {
|
||||
counts := make(map[int]int)
|
||||
v := 0
|
||||
for _, t := range n.table[0x80 : 0x80+blockSize] {
|
||||
if t != nil {
|
||||
if stride := t.value - v; v != 0 && stride >= 0 {
|
||||
counts[stride]++
|
||||
}
|
||||
v = t.value
|
||||
} else {
|
||||
v = 0
|
||||
}
|
||||
}
|
||||
var maxs, maxc int
|
||||
for stride, cnt := range counts {
|
||||
if cnt > maxc || (cnt == maxc && stride < maxs) {
|
||||
maxs, maxc = stride, cnt
|
||||
}
|
||||
}
|
||||
return maxs
|
||||
}
|
||||
|
||||
func (n trieNode) countSparseEntries() int {
|
||||
stride := n.mostFrequentStride()
|
||||
var count, v int
|
||||
for _, t := range n.table[0x80 : 0x80+blockSize] {
|
||||
tv := 0
|
||||
if t != nil {
|
||||
tv = t.value
|
||||
}
|
||||
if tv-v != stride {
|
||||
if tv != 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
v = tv
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (n *trieNode) insert(r rune, value uint16) {
|
||||
var p [utf8.UTFMax]byte
|
||||
sz := utf8.EncodeRune(p[:], r)
|
||||
|
||||
for i := 0; i < sz; i++ {
|
||||
if n.leaf {
|
||||
log.Fatalf("triegen: insert: node (%#v) should not be a leaf", n)
|
||||
}
|
||||
nn := n.table[p[i]]
|
||||
if nn == nil {
|
||||
nn = newNode()
|
||||
nn.b = p[i]
|
||||
n.table[p[i]] = nn
|
||||
}
|
||||
n = nn
|
||||
}
|
||||
n.value = int(value)
|
||||
n.leaf = true
|
||||
}
|
||||
|
||||
type nodeIndex struct {
|
||||
lookupBlocks []*trieNode
|
||||
valueBlocks []*trieNode
|
||||
sparseBlocks []*trieNode
|
||||
sparseOffset []uint16
|
||||
sparseCount int
|
||||
|
||||
lookupBlockIdx map[uint32]int
|
||||
valueBlockIdx map[uint32]int
|
||||
}
|
||||
|
||||
func newIndex() *nodeIndex {
|
||||
index := &nodeIndex{}
|
||||
index.lookupBlocks = make([]*trieNode, 0)
|
||||
index.valueBlocks = make([]*trieNode, 0)
|
||||
index.sparseBlocks = make([]*trieNode, 0)
|
||||
index.sparseOffset = make([]uint16, 1)
|
||||
index.lookupBlockIdx = make(map[uint32]int)
|
||||
index.valueBlockIdx = make(map[uint32]int)
|
||||
return index
|
||||
}
|
||||
|
||||
func computeOffsets(index *nodeIndex, n *trieNode) int {
|
||||
if n.leaf {
|
||||
return n.value
|
||||
}
|
||||
hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
|
||||
// We only index continuation bytes.
|
||||
for i := 0; i < blockSize; i++ {
|
||||
v := 0
|
||||
if nn := n.table[0x80+i]; nn != nil {
|
||||
v = computeOffsets(index, nn)
|
||||
}
|
||||
hasher.Write([]byte{uint8(v >> 8), uint8(v)})
|
||||
}
|
||||
h := hasher.Sum32()
|
||||
if n.isInternal() {
|
||||
v, ok := index.lookupBlockIdx[h]
|
||||
if !ok {
|
||||
v = len(index.lookupBlocks) - blockOffset
|
||||
index.lookupBlocks = append(index.lookupBlocks, n)
|
||||
index.lookupBlockIdx[h] = v
|
||||
}
|
||||
n.value = v
|
||||
} else {
|
||||
v, ok := index.valueBlockIdx[h]
|
||||
if !ok {
|
||||
if c := n.countSparseEntries(); c > maxSparseEntries {
|
||||
v = len(index.valueBlocks) - blockOffset
|
||||
index.valueBlocks = append(index.valueBlocks, n)
|
||||
index.valueBlockIdx[h] = v
|
||||
} else {
|
||||
v = -len(index.sparseOffset)
|
||||
index.sparseBlocks = append(index.sparseBlocks, n)
|
||||
index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount))
|
||||
index.sparseCount += c + 1
|
||||
index.valueBlockIdx[h] = v
|
||||
}
|
||||
}
|
||||
n.value = v
|
||||
}
|
||||
return n.value
|
||||
}
|
||||
|
||||
func printValueBlock(nr int, n *trieNode, offset int) {
|
||||
boff := nr * blockSize
|
||||
fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
|
||||
var printnewline bool
|
||||
for i := 0; i < blockSize; i++ {
|
||||
if i%6 == 0 {
|
||||
printnewline = true
|
||||
}
|
||||
v := 0
|
||||
if nn := n.table[i+offset]; nn != nil {
|
||||
v = nn.value
|
||||
}
|
||||
if v != 0 {
|
||||
if printnewline {
|
||||
fmt.Printf("\n")
|
||||
printnewline = false
|
||||
}
|
||||
fmt.Printf("%#04x:%#04x, ", boff+i, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func printSparseBlock(nr int, n *trieNode) {
|
||||
boff := -n.value
|
||||
fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
|
||||
v := 0
|
||||
//stride := f(n)
|
||||
stride := n.mostFrequentStride()
|
||||
c := n.countSparseEntries()
|
||||
fmt.Printf("\n{value:%#04x,lo:%#02x},", stride, uint8(c))
|
||||
for i, nn := range n.table[0x80 : 0x80+blockSize] {
|
||||
nv := 0
|
||||
if nn != nil {
|
||||
nv = nn.value
|
||||
}
|
||||
if nv-v != stride {
|
||||
if v != 0 {
|
||||
fmt.Printf(",hi:%#02x},", 0x80+i-1)
|
||||
}
|
||||
if nv != 0 {
|
||||
fmt.Printf("\n{value:%#04x,lo:%#02x", nv, nn.b)
|
||||
}
|
||||
}
|
||||
v = nv
|
||||
}
|
||||
if v != 0 {
|
||||
fmt.Printf(",hi:%#02x},", 0x80+blockSize-1)
|
||||
}
|
||||
}
|
||||
|
||||
func printLookupBlock(nr int, n *trieNode, offset, cutoff int) {
|
||||
boff := nr * blockSize
|
||||
fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
|
||||
var printnewline bool
|
||||
for i := 0; i < blockSize; i++ {
|
||||
if i%8 == 0 {
|
||||
printnewline = true
|
||||
}
|
||||
v := 0
|
||||
if nn := n.table[i+offset]; nn != nil {
|
||||
v = nn.value
|
||||
}
|
||||
if v != 0 {
|
||||
if v < 0 {
|
||||
v = -v - 1 + cutoff
|
||||
}
|
||||
if printnewline {
|
||||
fmt.Printf("\n")
|
||||
printnewline = false
|
||||
}
|
||||
fmt.Printf("%#03x:%#02x, ", boff+i, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// printTables returns the size in bytes of the generated tables.
|
||||
func (t *trieNode) printTables(name string) int {
|
||||
index := newIndex()
|
||||
// Values for 7-bit ASCII are stored in first two block, followed by nil block.
|
||||
index.valueBlocks = append(index.valueBlocks, nil, nil, nil)
|
||||
// First byte of multi-byte UTF-8 codepoints are indexed in 4th block.
|
||||
index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil, nil)
|
||||
// Index starter bytes of multi-byte UTF-8.
|
||||
for i := 0xC0; i < 0x100; i++ {
|
||||
if t.table[i] != nil {
|
||||
computeOffsets(index, t.table[i])
|
||||
}
|
||||
}
|
||||
|
||||
nv := len(index.valueBlocks) * blockSize
|
||||
fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2)
|
||||
fmt.Printf("// Block 2 is the null block.\n")
|
||||
fmt.Printf("var %sValues = [%d]uint16 {", name, nv)
|
||||
printValueBlock(0, t, 0)
|
||||
printValueBlock(1, t, 64)
|
||||
printValueBlock(2, newNode(), 0)
|
||||
for i := 3; i < len(index.valueBlocks); i++ {
|
||||
printValueBlock(i, index.valueBlocks[i], 0x80)
|
||||
}
|
||||
fmt.Print("\n}\n\n")
|
||||
|
||||
ls := len(index.sparseBlocks)
|
||||
fmt.Printf("// %sSparseOffset: %d entries, %d bytes\n", name, ls, ls*2)
|
||||
fmt.Printf("var %sSparseOffset = %#v\n\n", name, index.sparseOffset[1:])
|
||||
|
||||
ns := index.sparseCount
|
||||
fmt.Printf("// %sSparseValues: %d entries, %d bytes\n", name, ns, ns*4)
|
||||
fmt.Printf("var %sSparseValues = [%d]valueRange {", name, ns)
|
||||
for i, n := range index.sparseBlocks {
|
||||
printSparseBlock(i, n)
|
||||
}
|
||||
fmt.Print("\n}\n\n")
|
||||
|
||||
cutoff := len(index.valueBlocks) - blockOffset
|
||||
ni := len(index.lookupBlocks) * blockSize
|
||||
fmt.Printf("// %sLookup: %d bytes\n", name, ni)
|
||||
fmt.Printf("// Block 0 is the null block.\n")
|
||||
fmt.Printf("var %sLookup = [%d]uint8 {", name, ni)
|
||||
printLookupBlock(0, newNode(), 0, cutoff)
|
||||
printLookupBlock(1, newNode(), 0, cutoff)
|
||||
printLookupBlock(2, newNode(), 0, cutoff)
|
||||
printLookupBlock(3, t, 0xC0, cutoff)
|
||||
for i := 4; i < len(index.lookupBlocks); i++ {
|
||||
printLookupBlock(i, index.lookupBlocks[i], 0x80, cutoff)
|
||||
}
|
||||
fmt.Print("\n}\n\n")
|
||||
fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:], %sSparseValues[:], %sSparseOffset[:], %d}\n\n",
|
||||
name, name, name, name, name, cutoff)
|
||||
return nv*2 + ns*4 + ni + ls*2
|
||||
}
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package snappy
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// ErrCorrupt reports that the input is invalid.
|
||||
var ErrCorrupt = errors.New("snappy: corrupt input")
|
||||
|
||||
// DecodedLen returns the length of the decoded block.
|
||||
func DecodedLen(src []byte) (int, error) {
|
||||
v, _, err := decodedLen(src)
|
||||
return v, err
|
||||
}
|
||||
|
||||
// decodedLen returns the length of the decoded block and the number of bytes
|
||||
// that the length header occupied.
|
||||
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
|
||||
v, n := binary.Uvarint(src)
|
||||
if n == 0 {
|
||||
return 0, 0, ErrCorrupt
|
||||
}
|
||||
if uint64(int(v)) != v {
|
||||
return 0, 0, errors.New("snappy: decoded block is too large")
|
||||
}
|
||||
return int(v), n, nil
|
||||
}
|
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
// It is valid to pass a nil dst.
|
||||
func Decode(dst, src []byte) ([]byte, error) {
|
||||
dLen, s, err := decodedLen(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(dst) < dLen {
|
||||
dst = make([]byte, dLen)
|
||||
}
|
||||
|
||||
var d, offset, length int
|
||||
for s < len(src) {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s += 1
|
||||
case x == 60:
|
||||
s += 2
|
||||
if s > len(src) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
x = uint(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
if s > len(src) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
x = uint(src[s-2]) | uint(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
if s > len(src) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
if s > len(src) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
|
||||
}
|
||||
length = int(x + 1)
|
||||
if length <= 0 {
|
||||
return nil, errors.New("snappy: unsupported literal length")
|
||||
}
|
||||
if length > len(dst)-d || length > len(src)-s {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
if s > len(src) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
length = 4 + int(src[s-2])>>2&0x7
|
||||
offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
|
||||
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
if s > len(src) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(src[s-2]) | int(src[s-1])<<8
|
||||
|
||||
case tagCopy4:
|
||||
return nil, errors.New("snappy: unsupported COPY_4 tag")
|
||||
}
|
||||
|
||||
end := d + length
|
||||
if offset > d || end > len(dst) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
for ; d < end; d++ {
|
||||
dst[d] = dst[d-offset]
|
||||
}
|
||||
}
|
||||
if d != dLen {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return dst[:d], nil
|
||||
}
|
||||
-174
@@ -1,174 +0,0 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package snappy
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// We limit how far copy back-references can go, the same as the C++ code.
|
||||
const maxOffset = 1 << 15
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
func emitLiteral(dst, lit []byte) int {
|
||||
i, n := 0, uint(len(lit)-1)
|
||||
switch {
|
||||
case n < 60:
|
||||
dst[0] = uint8(n)<<2 | tagLiteral
|
||||
i = 1
|
||||
case n < 1<<8:
|
||||
dst[0] = 60<<2 | tagLiteral
|
||||
dst[1] = uint8(n)
|
||||
i = 2
|
||||
case n < 1<<16:
|
||||
dst[0] = 61<<2 | tagLiteral
|
||||
dst[1] = uint8(n)
|
||||
dst[2] = uint8(n >> 8)
|
||||
i = 3
|
||||
case n < 1<<24:
|
||||
dst[0] = 62<<2 | tagLiteral
|
||||
dst[1] = uint8(n)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[3] = uint8(n >> 16)
|
||||
i = 4
|
||||
case int64(n) < 1<<32:
|
||||
dst[0] = 63<<2 | tagLiteral
|
||||
dst[1] = uint8(n)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[4] = uint8(n >> 24)
|
||||
i = 5
|
||||
default:
|
||||
panic("snappy: source buffer is too long")
|
||||
}
|
||||
if copy(dst[i:], lit) != len(lit) {
|
||||
panic("snappy: destination buffer is too short")
|
||||
}
|
||||
return i + len(lit)
|
||||
}
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
func emitCopy(dst []byte, offset, length int) int {
|
||||
i := 0
|
||||
for length > 0 {
|
||||
x := length - 4
|
||||
if 0 <= x && x < 1<<3 && offset < 1<<11 {
|
||||
dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
|
||||
dst[i+1] = uint8(offset)
|
||||
i += 2
|
||||
break
|
||||
}
|
||||
|
||||
x = length
|
||||
if x > 1<<6 {
|
||||
x = 1 << 6
|
||||
}
|
||||
dst[i+0] = uint8(x-1)<<2 | tagCopy2
|
||||
dst[i+1] = uint8(offset)
|
||||
dst[i+2] = uint8(offset >> 8)
|
||||
i += 3
|
||||
length -= x
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// Encode returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
// It is valid to pass a nil dst.
|
||||
func Encode(dst, src []byte) ([]byte, error) {
|
||||
if n := MaxEncodedLen(len(src)); len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
// Return early if src is short.
|
||||
if len(src) <= 4 {
|
||||
if len(src) != 0 {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
}
|
||||
return dst[:d], nil
|
||||
}
|
||||
|
||||
// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
|
||||
const maxTableSize = 1 << 14
|
||||
shift, tableSize := uint(32-8), 1<<8
|
||||
for tableSize < maxTableSize && tableSize < len(src) {
|
||||
shift--
|
||||
tableSize *= 2
|
||||
}
|
||||
var table [maxTableSize]int
|
||||
|
||||
// Iterate over the source bytes.
|
||||
var (
|
||||
s int // The iterator position.
|
||||
t int // The last position with the same hash as s.
|
||||
lit int // The start position of any pending literal bytes.
|
||||
)
|
||||
for s+3 < len(src) {
|
||||
// Update the hash table.
|
||||
b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
|
||||
h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
|
||||
p := &table[(h*0x1e35a7bd)>>shift]
|
||||
// We need to to store values in [-1, inf) in table. To save
|
||||
// some initialization time, (re)use the table's zero value
|
||||
// and shift the values against this zero: add 1 on writes,
|
||||
// subtract 1 on reads.
|
||||
t, *p = *p-1, s+1
|
||||
// If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
|
||||
if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
|
||||
s++
|
||||
continue
|
||||
}
|
||||
// Otherwise, we have a match. First, emit any pending literal bytes.
|
||||
if lit != s {
|
||||
d += emitLiteral(dst[d:], src[lit:s])
|
||||
}
|
||||
// Extend the match to be as long as possible.
|
||||
s0 := s
|
||||
s, t = s+4, t+4
|
||||
for s < len(src) && src[s] == src[t] {
|
||||
s++
|
||||
t++
|
||||
}
|
||||
// Emit the copied bytes.
|
||||
d += emitCopy(dst[d:], s-t, s-s0)
|
||||
lit = s
|
||||
}
|
||||
|
||||
// Emit any final pending literal bytes and return.
|
||||
if lit != len(src) {
|
||||
d += emitLiteral(dst[d:], src[lit:])
|
||||
}
|
||||
return dst[:d], nil
|
||||
}
|
||||
|
||||
// MaxEncodedLen returns the maximum length of a snappy block, given its
|
||||
// uncompressed length.
|
||||
func MaxEncodedLen(srcLen int) int {
|
||||
// Compressed data can be defined as:
|
||||
// compressed := item* literal*
|
||||
// item := literal* copy
|
||||
//
|
||||
// The trailing literal sequence has a space blowup of at most 62/60
|
||||
// since a literal of length 60 needs one tag byte + one extra byte
|
||||
// for length information.
|
||||
//
|
||||
// Item blowup is trickier to measure. Suppose the "copy" op copies
|
||||
// 4 bytes of data. Because of a special check in the encoding code,
|
||||
// we produce a 4-byte copy only if the offset is < 65536. Therefore
|
||||
// the copy op takes 3 bytes to encode, and this type of item leads
|
||||
// to at most the 62/60 blowup for representing literals.
|
||||
//
|
||||
// Suppose the "copy" op copies 5 bytes of data. If the offset is big
|
||||
// enough, it will take 5 bytes to encode the copy op. Therefore the
|
||||
// worst case here is a one-byte literal followed by a five-byte copy.
|
||||
// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
|
||||
//
|
||||
// This last factor dominates the blowup, so the final estimate is:
|
||||
return 32 + srcLen + srcLen/6
|
||||
}
|
||||
-38
@@ -1,38 +0,0 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package snappy implements the snappy block-based compression format.
|
||||
// It aims for very high speeds and reasonable compression.
|
||||
//
|
||||
// The C++ snappy implementation is at http://code.google.com/p/snappy/
|
||||
package snappy
|
||||
|
||||
/*
|
||||
Each encoded block begins with the varint-encoded length of the decoded data,
|
||||
followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
|
||||
first byte of each chunk is broken into its 2 least and 6 most significant bits
|
||||
called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
|
||||
Zero means a literal tag. All other values mean a copy tag.
|
||||
|
||||
For literal tags:
|
||||
- If m < 60, the next 1 + m bytes are literal bytes.
|
||||
- Otherwise, let n be the little-endian unsigned integer denoted by the next
|
||||
m - 59 bytes. The next 1 + n bytes after that are literal bytes.
|
||||
|
||||
For copy tags, length bytes are copied from offset bytes ago, in the style of
|
||||
Lempel-Ziv compression algorithms. In particular:
|
||||
- For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
|
||||
The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
|
||||
of the offset. The next byte is bits 0-7 of the offset.
|
||||
- For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
|
||||
The length is 1 + m. The offset is the little-endian unsigned integer
|
||||
denoted by the next 2 bytes.
|
||||
- For l == 3, this tag is a legacy format that is no longer supported.
|
||||
*/
|
||||
const (
|
||||
tagLiteral = 0x00
|
||||
tagCopy1 = 0x01
|
||||
tagCopy2 = 0x02
|
||||
tagCopy4 = 0x03
|
||||
)
|
||||
-261
@@ -1,261 +0,0 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package snappy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
|
||||
|
||||
func roundtrip(b, ebuf, dbuf []byte) error {
|
||||
e, err := Encode(ebuf, b)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding error: %v", err)
|
||||
}
|
||||
d, err := Decode(dbuf, e)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decoding error: %v", err)
|
||||
}
|
||||
if !bytes.Equal(b, d) {
|
||||
return fmt.Errorf("roundtrip mismatch:\n\twant %v\n\tgot %v", b, d)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestEmpty(t *testing.T) {
|
||||
if err := roundtrip(nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmallCopy(t *testing.T) {
|
||||
for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
|
||||
for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
|
||||
for i := 0; i < 32; i++ {
|
||||
s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb"
|
||||
if err := roundtrip([]byte(s), ebuf, dbuf); err != nil {
|
||||
t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmallRand(t *testing.T) {
|
||||
rand.Seed(27354294)
|
||||
for n := 1; n < 20000; n += 23 {
|
||||
b := make([]byte, n)
|
||||
for i, _ := range b {
|
||||
b[i] = uint8(rand.Uint32())
|
||||
}
|
||||
if err := roundtrip(b, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmallRegular(t *testing.T) {
|
||||
for n := 1; n < 20000; n += 23 {
|
||||
b := make([]byte, n)
|
||||
for i, _ := range b {
|
||||
b[i] = uint8(i%10 + 'a')
|
||||
}
|
||||
if err := roundtrip(b, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func benchDecode(b *testing.B, src []byte) {
|
||||
encoded, err := Encode(nil, src)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// Bandwidth is in amount of uncompressed data.
|
||||
b.SetBytes(int64(len(src)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
Decode(src, encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func benchEncode(b *testing.B, src []byte) {
|
||||
// Bandwidth is in amount of uncompressed data.
|
||||
b.SetBytes(int64(len(src)))
|
||||
dst := make([]byte, MaxEncodedLen(len(src)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
Encode(dst, src)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(b *testing.B, filename string) []byte {
|
||||
src, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
b.Fatalf("failed reading %s: %s", filename, err)
|
||||
}
|
||||
if len(src) == 0 {
|
||||
b.Fatalf("%s has zero length", filename)
|
||||
}
|
||||
return src
|
||||
}
|
||||
|
||||
// expand returns a slice of length n containing repeated copies of src.
|
||||
func expand(src []byte, n int) []byte {
|
||||
dst := make([]byte, n)
|
||||
for x := dst; len(x) > 0; {
|
||||
i := copy(x, src)
|
||||
x = x[i:]
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func benchWords(b *testing.B, n int, decode bool) {
|
||||
// Note: the file is OS-language dependent so the resulting values are not
|
||||
// directly comparable for non-US-English OS installations.
|
||||
data := expand(readFile(b, "/usr/share/dict/words"), n)
|
||||
if decode {
|
||||
benchDecode(b, data)
|
||||
} else {
|
||||
benchEncode(b, data)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) }
|
||||
func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) }
|
||||
func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) }
|
||||
func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) }
|
||||
func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) }
|
||||
func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) }
|
||||
func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
|
||||
func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
|
||||
|
||||
// testFiles' values are copied directly from
|
||||
// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc.
|
||||
// The label field is unused in snappy-go.
|
||||
var testFiles = []struct {
|
||||
label string
|
||||
filename string
|
||||
}{
|
||||
{"html", "html"},
|
||||
{"urls", "urls.10K"},
|
||||
{"jpg", "house.jpg"},
|
||||
{"pdf", "mapreduce-osdi-1.pdf"},
|
||||
{"html4", "html_x_4"},
|
||||
{"cp", "cp.html"},
|
||||
{"c", "fields.c"},
|
||||
{"lsp", "grammar.lsp"},
|
||||
{"xls", "kennedy.xls"},
|
||||
{"txt1", "alice29.txt"},
|
||||
{"txt2", "asyoulik.txt"},
|
||||
{"txt3", "lcet10.txt"},
|
||||
{"txt4", "plrabn12.txt"},
|
||||
{"bin", "ptt5"},
|
||||
{"sum", "sum"},
|
||||
{"man", "xargs.1"},
|
||||
{"pb", "geo.protodata"},
|
||||
{"gaviota", "kppkn.gtb"},
|
||||
}
|
||||
|
||||
// The test data files are present at this canonical URL.
|
||||
const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/"
|
||||
|
||||
func downloadTestdata(basename string) (errRet error) {
|
||||
filename := filepath.Join("testdata", basename)
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create %s: %s", filename, err)
|
||||
}
|
||||
defer f.Close()
|
||||
defer func() {
|
||||
if errRet != nil {
|
||||
os.Remove(filename)
|
||||
}
|
||||
}()
|
||||
resp, err := http.Get(baseURL + basename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download %s: %s", baseURL+basename, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, err = io.Copy(f, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write %s: %s", filename, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func benchFile(b *testing.B, n int, decode bool) {
|
||||
filename := filepath.Join("testdata", testFiles[n].filename)
|
||||
if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
|
||||
if !*download {
|
||||
b.Fatal("test data not found; skipping benchmark without the -download flag")
|
||||
}
|
||||
// Download the official snappy C++ implementation reference test data
|
||||
// files for benchmarking.
|
||||
if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
|
||||
b.Fatalf("failed to create testdata: %s", err)
|
||||
}
|
||||
for _, tf := range testFiles {
|
||||
if err := downloadTestdata(tf.filename); err != nil {
|
||||
b.Fatalf("failed to download testdata: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
data := readFile(b, filename)
|
||||
if decode {
|
||||
benchDecode(b, data)
|
||||
} else {
|
||||
benchEncode(b, data)
|
||||
}
|
||||
}
|
||||
|
||||
// Naming convention is kept similar to what snappy's C++ implementation uses.
|
||||
func Benchmark_UFlat0(b *testing.B) { benchFile(b, 0, true) }
|
||||
func Benchmark_UFlat1(b *testing.B) { benchFile(b, 1, true) }
|
||||
func Benchmark_UFlat2(b *testing.B) { benchFile(b, 2, true) }
|
||||
func Benchmark_UFlat3(b *testing.B) { benchFile(b, 3, true) }
|
||||
func Benchmark_UFlat4(b *testing.B) { benchFile(b, 4, true) }
|
||||
func Benchmark_UFlat5(b *testing.B) { benchFile(b, 5, true) }
|
||||
func Benchmark_UFlat6(b *testing.B) { benchFile(b, 6, true) }
|
||||
func Benchmark_UFlat7(b *testing.B) { benchFile(b, 7, true) }
|
||||
func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) }
|
||||
func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) }
|
||||
func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
|
||||
func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
|
||||
func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) }
|
||||
func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) }
|
||||
func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) }
|
||||
func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) }
|
||||
func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) }
|
||||
func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) }
|
||||
func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) }
|
||||
func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) }
|
||||
func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) }
|
||||
func Benchmark_ZFlat3(b *testing.B) { benchFile(b, 3, false) }
|
||||
func Benchmark_ZFlat4(b *testing.B) { benchFile(b, 4, false) }
|
||||
func Benchmark_ZFlat5(b *testing.B) { benchFile(b, 5, false) }
|
||||
func Benchmark_ZFlat6(b *testing.B) { benchFile(b, 6, false) }
|
||||
func Benchmark_ZFlat7(b *testing.B) { benchFile(b, 7, false) }
|
||||
func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) }
|
||||
func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) }
|
||||
func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
|
||||
func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }
|
||||
func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) }
|
||||
func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) }
|
||||
func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) }
|
||||
func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) }
|
||||
func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) }
|
||||
func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lru
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
type simpleStruct struct {
|
||||
int
|
||||
string
|
||||
}
|
||||
|
||||
type complexStruct struct {
|
||||
int
|
||||
simpleStruct
|
||||
}
|
||||
|
||||
var getTests = []struct {
|
||||
name string
|
||||
keyToAdd interface{}
|
||||
keyToGet interface{}
|
||||
expectedOk bool
|
||||
}{
|
||||
{"string_hit", "myKey", "myKey", true},
|
||||
{"string_miss", "myKey", "nonsense", false},
|
||||
{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
|
||||
{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
|
||||
{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
|
||||
complexStruct{1, simpleStruct{2, "three"}}, true},
|
||||
}
|
||||
|
||||
func TestGet(t *testing.T) {
|
||||
for _, tt := range getTests {
|
||||
lru := New(0)
|
||||
lru.Add(tt.keyToAdd, 1234)
|
||||
val, ok := lru.Get(tt.keyToGet)
|
||||
if ok != tt.expectedOk {
|
||||
t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
|
||||
} else if ok && val != 1234 {
|
||||
t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemove(t *testing.T) {
|
||||
lru := New(0)
|
||||
lru.Add("myKey", 1234)
|
||||
if val, ok := lru.Get("myKey"); !ok {
|
||||
t.Fatal("TestRemove returned no match")
|
||||
} else if val != 1234 {
|
||||
t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val)
|
||||
}
|
||||
|
||||
lru.Remove("myKey")
|
||||
if _, ok := lru.Get("myKey"); ok {
|
||||
t.Fatal("TestRemove returned a removed entry")
|
||||
}
|
||||
}
|
||||
-185
@@ -1,185 +0,0 @@
|
||||
This software is licensed under the LGPLv3, included below.
|
||||
|
||||
As a special exception to the GNU Lesser General Public License version 3
|
||||
("LGPL3"), the copyright holders of this Library give you permission to
|
||||
convey to a third party a Combined Work that links statically or dynamically
|
||||
to this Library without providing any Minimal Corresponding Source or
|
||||
Minimal Application Code as set out in 4d or providing the installation
|
||||
information set out in section 4e, provided that you comply with the other
|
||||
provisions of LGPL3 and provided that you meet, for the Application the
|
||||
terms and conditions of the license(s) which apply to the Application.
|
||||
|
||||
Except as stated in this special exception, the provisions of LGPL3 will
|
||||
continue to comply in full to this Library. If you modify this Library, you
|
||||
may apply this exception to your version of this Library, but you are not
|
||||
obliged to do so. If you do not wish to do so, delete this exception
|
||||
statement from your version. This exception does not (and cannot) modify any
|
||||
license terms which apply to the Application, with which you must still
|
||||
comply.
|
||||
|
||||
|
||||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
|
||||
This version of the GNU Lesser General Public License incorporates
|
||||
the terms and conditions of version 3 of the GNU General Public
|
||||
License, supplemented by the additional permissions listed below.
|
||||
|
||||
0. Additional Definitions.
|
||||
|
||||
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||
General Public License.
|
||||
|
||||
"The Library" refers to a covered work governed by this License,
|
||||
other than an Application or a Combined Work as defined below.
|
||||
|
||||
An "Application" is any work that makes use of an interface provided
|
||||
by the Library, but which is not otherwise based on the Library.
|
||||
Defining a subclass of a class defined by the Library is deemed a mode
|
||||
of using an interface provided by the Library.
|
||||
|
||||
A "Combined Work" is a work produced by combining or linking an
|
||||
Application with the Library. The particular version of the Library
|
||||
with which the Combined Work was made is also called the "Linked
|
||||
Version".
|
||||
|
||||
The "Minimal Corresponding Source" for a Combined Work means the
|
||||
Corresponding Source for the Combined Work, excluding any source code
|
||||
for portions of the Combined Work that, considered in isolation, are
|
||||
based on the Application, and not on the Linked Version.
|
||||
|
||||
The "Corresponding Application Code" for a Combined Work means the
|
||||
object code and/or source code for the Application, including any data
|
||||
and utility programs needed for reproducing the Combined Work from the
|
||||
Application, but excluding the System Libraries of the Combined Work.
|
||||
|
||||
1. Exception to Section 3 of the GNU GPL.
|
||||
|
||||
You may convey a covered work under sections 3 and 4 of this License
|
||||
without being bound by section 3 of the GNU GPL.
|
||||
|
||||
2. Conveying Modified Versions.
|
||||
|
||||
If you modify a copy of the Library, and, in your modifications, a
|
||||
facility refers to a function or data to be supplied by an Application
|
||||
that uses the facility (other than as an argument passed when the
|
||||
facility is invoked), then you may convey a copy of the modified
|
||||
version:
|
||||
|
||||
a) under this License, provided that you make a good faith effort to
|
||||
ensure that, in the event an Application does not supply the
|
||||
function or data, the facility still operates, and performs
|
||||
whatever part of its purpose remains meaningful, or
|
||||
|
||||
b) under the GNU GPL, with none of the additional permissions of
|
||||
this License applicable to that copy.
|
||||
|
||||
3. Object Code Incorporating Material from Library Header Files.
|
||||
|
||||
The object code form of an Application may incorporate material from
|
||||
a header file that is part of the Library. You may convey such object
|
||||
code under terms of your choice, provided that, if the incorporated
|
||||
material is not limited to numerical parameters, data structure
|
||||
layouts and accessors, or small macros, inline functions and templates
|
||||
(ten or fewer lines in length), you do both of the following:
|
||||
|
||||
a) Give prominent notice with each copy of the object code that the
|
||||
Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
4. Combined Works.
|
||||
|
||||
You may convey a Combined Work under terms of your choice that,
|
||||
taken together, effectively do not restrict modification of the
|
||||
portions of the Library contained in the Combined Work and reverse
|
||||
engineering for debugging such modifications, if you also do each of
|
||||
the following:
|
||||
|
||||
a) Give prominent notice with each copy of the Combined Work that
|
||||
the Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
c) For a Combined Work that displays copyright notices during
|
||||
execution, include the copyright notice for the Library among
|
||||
these notices, as well as a reference directing the user to the
|
||||
copies of the GNU GPL and this license document.
|
||||
|
||||
d) Do one of the following:
|
||||
|
||||
0) Convey the Minimal Corresponding Source under the terms of this
|
||||
License, and the Corresponding Application Code in a form
|
||||
suitable for, and under terms that permit, the user to
|
||||
recombine or relink the Application with a modified version of
|
||||
the Linked Version to produce a modified Combined Work, in the
|
||||
manner specified by section 6 of the GNU GPL for conveying
|
||||
Corresponding Source.
|
||||
|
||||
1) Use a suitable shared library mechanism for linking with the
|
||||
Library. A suitable mechanism is one that (a) uses at run time
|
||||
a copy of the Library already present on the user's computer
|
||||
system, and (b) will operate properly with a modified version
|
||||
of the Library that is interface-compatible with the Linked
|
||||
Version.
|
||||
|
||||
e) Provide Installation Information, but only if you would otherwise
|
||||
be required to provide such information under section 6 of the
|
||||
GNU GPL, and only to the extent that such information is
|
||||
necessary to install and execute a modified version of the
|
||||
Combined Work produced by recombining or relinking the
|
||||
Application with a modified version of the Linked Version. (If
|
||||
you use option 4d0, the Installation Information must accompany
|
||||
the Minimal Corresponding Source and Corresponding Application
|
||||
Code. If you use option 4d1, you must provide the Installation
|
||||
Information in the manner specified by section 6 of the GNU GPL
|
||||
for conveying Corresponding Source.)
|
||||
|
||||
5. Combined Libraries.
|
||||
|
||||
You may place library facilities that are a work based on the
|
||||
Library side by side in a single library together with other library
|
||||
facilities that are not Applications and are not covered by this
|
||||
License, and convey such a combined library under terms of your
|
||||
choice, if you do both of the following:
|
||||
|
||||
a) Accompany the combined library with a copy of the same work based
|
||||
on the Library, uncombined with any other library facilities,
|
||||
conveyed under the terms of this License.
|
||||
|
||||
b) Give prominent notice with the combined library that part of it
|
||||
is a work based on the Library, and explaining where to find the
|
||||
accompanying uncombined form of the same work.
|
||||
|
||||
6. Revised Versions of the GNU Lesser General Public License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions
|
||||
of the GNU Lesser General Public License from time to time. Such new
|
||||
versions will be similar in spirit to the present version, but may
|
||||
differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Library as you received it specifies that a certain numbered version
|
||||
of the GNU Lesser General Public License "or any later version"
|
||||
applies to it, you have the option of following the terms and
|
||||
conditions either of that published version or of any later version
|
||||
published by the Free Software Foundation. If the Library as you
|
||||
received it does not specify a version number of the GNU Lesser
|
||||
General Public License, you may choose any version of the GNU Lesser
|
||||
General Public License ever published by the Free Software Foundation.
|
||||
|
||||
If the Library as you received it specifies that a proxy can decide
|
||||
whether future versions of the GNU Lesser General Public License shall
|
||||
apply, that proxy's public statement of acceptance of any version is
|
||||
permanent authorization for you to choose that version for the
|
||||
Library.
|
||||
-117
@@ -1,117 +0,0 @@
|
||||
# ratelimit
|
||||
--
|
||||
import "github.com/juju/ratelimit"
|
||||
|
||||
The ratelimit package provides an efficient token bucket implementation. See
|
||||
http://en.wikipedia.org/wiki/Token_bucket.
|
||||
|
||||
## Usage
|
||||
|
||||
#### func Reader
|
||||
|
||||
```go
|
||||
func Reader(r io.Reader, bucket *Bucket) io.Reader
|
||||
```
|
||||
Reader returns a reader that is rate limited by the given token bucket. Each
|
||||
token in the bucket represents one byte.
|
||||
|
||||
#### func Writer
|
||||
|
||||
```go
|
||||
func Writer(w io.Writer, bucket *Bucket) io.Writer
|
||||
```
|
||||
Writer returns a reader that is rate limited by the given token bucket. Each
|
||||
token in the bucket represents one byte.
|
||||
|
||||
#### type Bucket
|
||||
|
||||
```go
|
||||
type Bucket struct {
|
||||
}
|
||||
```
|
||||
|
||||
Bucket represents a token bucket that fills at a predetermined rate. Methods on
|
||||
Bucket may be called concurrently.
|
||||
|
||||
#### func NewBucket
|
||||
|
||||
```go
|
||||
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket
|
||||
```
|
||||
NewBucket returns a new token bucket that fills at the rate of one token every
|
||||
fillInterval, up to the given maximum capacity. Both arguments must be positive.
|
||||
The bucket is initially full.
|
||||
|
||||
#### func NewBucketWithQuantum
|
||||
|
||||
```go
|
||||
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket
|
||||
```
|
||||
NewBucketWithQuantum is similar to NewBucket, but allows the specification of
|
||||
the quantum size - quantum tokens are added every fillInterval.
|
||||
|
||||
#### func NewBucketWithRate
|
||||
|
||||
```go
|
||||
func NewBucketWithRate(rate float64, capacity int64) *Bucket
|
||||
```
|
||||
NewBucketWithRate returns a token bucket that fills the bucket at the rate of
|
||||
rate tokens per second up to the given maximum capacity. Because of limited
|
||||
clock resolution, at high rates, the actual rate may be up to 1% different from
|
||||
the specified rate.
|
||||
|
||||
#### func (*Bucket) Rate
|
||||
|
||||
```go
|
||||
func (tb *Bucket) Rate() float64
|
||||
```
|
||||
Rate returns the fill rate of the bucket, in tokens per second.
|
||||
|
||||
#### func (*Bucket) Take
|
||||
|
||||
```go
|
||||
func (tb *Bucket) Take(count int64) time.Duration
|
||||
```
|
||||
Take takes count tokens from the bucket without blocking. It returns the time
|
||||
that the caller should wait until the tokens are actually available.
|
||||
|
||||
Note that if the request is irrevocable - there is no way to return tokens to
|
||||
the bucket once this method commits us to taking them.
|
||||
|
||||
#### func (*Bucket) TakeAvailable
|
||||
|
||||
```go
|
||||
func (tb *Bucket) TakeAvailable(count int64) int64
|
||||
```
|
||||
TakeAvailable takes up to count immediately available tokens from the bucket. It
|
||||
returns the number of tokens removed, or zero if there are no available tokens.
|
||||
It does not block.
|
||||
|
||||
#### func (*Bucket) TakeMaxDuration
|
||||
|
||||
```go
|
||||
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool)
|
||||
```
|
||||
TakeMaxDuration is like Take, except that it will only take tokens from the
|
||||
bucket if the wait time for the tokens is no greater than maxWait.
|
||||
|
||||
If it would take longer than maxWait for the tokens to become available, it does
|
||||
nothing and reports false, otherwise it returns the time that the caller should
|
||||
wait until the tokens are actually available, and reports true.
|
||||
|
||||
#### func (*Bucket) Wait
|
||||
|
||||
```go
|
||||
func (tb *Bucket) Wait(count int64)
|
||||
```
|
||||
Wait takes count tokens from the bucket, waiting until they are available.
|
||||
|
||||
#### func (*Bucket) WaitMaxDuration
|
||||
|
||||
```go
|
||||
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool
|
||||
```
|
||||
WaitMaxDuration is like Wait except that it will only take tokens from the
|
||||
bucket if it needs to wait for no greater than maxWait. It reports whether any
|
||||
tokens have been removed from the bucket If no tokens have been removed, it
|
||||
returns immediately.
|
||||
-226
@@ -1,226 +0,0 @@
|
||||
// Copyright 2014 Canonical Ltd.
|
||||
// Licensed under the LGPLv3 with static-linking exception.
|
||||
// See LICENCE file for details.
|
||||
|
||||
// The ratelimit package provides an efficient token bucket implementation.
|
||||
// See http://en.wikipedia.org/wiki/Token_bucket.
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Bucket represents a token bucket that fills at a predetermined rate.
|
||||
// Methods on Bucket may be called concurrently.
|
||||
type Bucket struct {
|
||||
startTime time.Time
|
||||
capacity int64
|
||||
quantum int64
|
||||
fillInterval time.Duration
|
||||
|
||||
// The mutex guards the fields following it.
|
||||
mu sync.Mutex
|
||||
|
||||
// avail holds the number of available tokens
|
||||
// in the bucket, as of availTick ticks from startTime.
|
||||
// It will be negative when there are consumers
|
||||
// waiting for tokens.
|
||||
avail int64
|
||||
availTick int64
|
||||
}
|
||||
|
||||
// NewBucket returns a new token bucket that fills at the
|
||||
// rate of one token every fillInterval, up to the given
|
||||
// maximum capacity. Both arguments must be
|
||||
// positive. The bucket is initially full.
|
||||
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket {
|
||||
return NewBucketWithQuantum(fillInterval, capacity, 1)
|
||||
}
|
||||
|
||||
// rateMargin specifes the allowed variance of actual
|
||||
// rate from specified rate. 1% seems reasonable.
|
||||
const rateMargin = 0.01
|
||||
|
||||
// NewBucketWithRate returns a token bucket that fills the bucket
|
||||
// at the rate of rate tokens per second up to the given
|
||||
// maximum capacity. Because of limited clock resolution,
|
||||
// at high rates, the actual rate may be up to 1% different from the
|
||||
// specified rate.
|
||||
func NewBucketWithRate(rate float64, capacity int64) *Bucket {
|
||||
for quantum := int64(1); quantum < 1<<50; quantum = nextQuantum(quantum) {
|
||||
fillInterval := time.Duration(1e9 * float64(quantum) / rate)
|
||||
if fillInterval <= 0 {
|
||||
continue
|
||||
}
|
||||
tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
|
||||
if diff := abs(tb.Rate() - rate); diff/rate <= rateMargin {
|
||||
return tb
|
||||
}
|
||||
}
|
||||
panic("cannot find suitable quantum for " + strconv.FormatFloat(rate, 'g', -1, 64))
|
||||
}
|
||||
|
||||
// nextQuantum returns the next quantum to try after q.
|
||||
// We grow the quantum exponentially, but slowly, so we
|
||||
// get a good fit in the lower numbers.
|
||||
func nextQuantum(q int64) int64 {
|
||||
q1 := q * 11 / 10
|
||||
if q1 == q {
|
||||
q1++
|
||||
}
|
||||
return q1
|
||||
}
|
||||
|
||||
// NewBucketWithQuantum is similar to NewBucket, but allows
|
||||
// the specification of the quantum size - quantum tokens
|
||||
// are added every fillInterval.
|
||||
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket {
|
||||
if fillInterval <= 0 {
|
||||
panic("token bucket fill interval is not > 0")
|
||||
}
|
||||
if capacity <= 0 {
|
||||
panic("token bucket capacity is not > 0")
|
||||
}
|
||||
if quantum <= 0 {
|
||||
panic("token bucket quantum is not > 0")
|
||||
}
|
||||
return &Bucket{
|
||||
startTime: time.Now(),
|
||||
capacity: capacity,
|
||||
quantum: quantum,
|
||||
avail: capacity,
|
||||
fillInterval: fillInterval,
|
||||
}
|
||||
}
|
||||
|
||||
// Wait takes count tokens from the bucket, waiting until they are
|
||||
// available.
|
||||
func (tb *Bucket) Wait(count int64) {
|
||||
if d := tb.Take(count); d > 0 {
|
||||
time.Sleep(d)
|
||||
}
|
||||
}
|
||||
|
||||
// WaitMaxDuration is like Wait except that it will
|
||||
// only take tokens from the bucket if it needs to wait
|
||||
// for no greater than maxWait. It reports whether
|
||||
// any tokens have been removed from the bucket
|
||||
// If no tokens have been removed, it returns immediately.
|
||||
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool {
|
||||
d, ok := tb.TakeMaxDuration(count, maxWait)
|
||||
if d > 0 {
|
||||
time.Sleep(d)
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
const infinityDuration time.Duration = 0x7fffffffffffffff
|
||||
|
||||
// Take takes count tokens from the bucket without blocking. It returns
|
||||
// the time that the caller should wait until the tokens are actually
|
||||
// available.
|
||||
//
|
||||
// Note that if the request is irrevocable - there is no way to return
|
||||
// tokens to the bucket once this method commits us to taking them.
|
||||
func (tb *Bucket) Take(count int64) time.Duration {
|
||||
d, _ := tb.take(time.Now(), count, infinityDuration)
|
||||
return d
|
||||
}
|
||||
|
||||
// TakeMaxDuration is like Take, except that
|
||||
// it will only take tokens from the bucket if the wait
|
||||
// time for the tokens is no greater than maxWait.
|
||||
//
|
||||
// If it would take longer than maxWait for the tokens
|
||||
// to become available, it does nothing and reports false,
|
||||
// otherwise it returns the time that the caller should
|
||||
// wait until the tokens are actually available, and reports
|
||||
// true.
|
||||
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool) {
|
||||
return tb.take(time.Now(), count, maxWait)
|
||||
}
|
||||
|
||||
// TakeAvailable takes up to count immediately available tokens from the
|
||||
// bucket. It returns the number of tokens removed, or zero if there are
|
||||
// no available tokens. It does not block.
|
||||
func (tb *Bucket) TakeAvailable(count int64) int64 {
|
||||
return tb.takeAvailable(time.Now(), count)
|
||||
}
|
||||
|
||||
// takeAvailable is the internal version of TakeAvailable - it takes the
|
||||
// current time as an argument to enable easy testing.
|
||||
func (tb *Bucket) takeAvailable(now time.Time, count int64) int64 {
|
||||
if count <= 0 {
|
||||
return 0
|
||||
}
|
||||
tb.mu.Lock()
|
||||
defer tb.mu.Unlock()
|
||||
|
||||
tb.adjust(now)
|
||||
if tb.avail <= 0 {
|
||||
return 0
|
||||
}
|
||||
if count > tb.avail {
|
||||
count = tb.avail
|
||||
}
|
||||
tb.avail -= count
|
||||
return count
|
||||
}
|
||||
|
||||
// Rate returns the fill rate of the bucket, in tokens per second.
|
||||
func (tb *Bucket) Rate() float64 {
|
||||
return 1e9 * float64(tb.quantum) / float64(tb.fillInterval)
|
||||
}
|
||||
|
||||
// take is the internal version of Take - it takes the current time as
|
||||
// an argument to enable easy testing.
|
||||
func (tb *Bucket) take(now time.Time, count int64, maxWait time.Duration) (time.Duration, bool) {
|
||||
if count <= 0 {
|
||||
return 0, true
|
||||
}
|
||||
tb.mu.Lock()
|
||||
defer tb.mu.Unlock()
|
||||
|
||||
currentTick := tb.adjust(now)
|
||||
avail := tb.avail - count
|
||||
if avail >= 0 {
|
||||
tb.avail = avail
|
||||
return 0, true
|
||||
}
|
||||
// Round up the missing tokens to the nearest multiple
|
||||
// of quantum - the tokens won't be available until
|
||||
// that tick.
|
||||
endTick := currentTick + (-avail+tb.quantum-1)/tb.quantum
|
||||
endTime := tb.startTime.Add(time.Duration(endTick) * tb.fillInterval)
|
||||
waitTime := endTime.Sub(now)
|
||||
if waitTime > maxWait {
|
||||
return 0, false
|
||||
}
|
||||
tb.avail = avail
|
||||
return waitTime, true
|
||||
}
|
||||
|
||||
// adjust adjusts the current bucket capacity based on the current time.
|
||||
// It returns the current tick.
|
||||
func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
|
||||
currentTick = int64(now.Sub(tb.startTime) / tb.fillInterval)
|
||||
|
||||
if tb.avail >= tb.capacity {
|
||||
return
|
||||
}
|
||||
tb.avail += (currentTick - tb.availTick) * tb.quantum
|
||||
if tb.avail > tb.capacity {
|
||||
tb.avail = tb.capacity
|
||||
}
|
||||
tb.availTick = currentTick
|
||||
return
|
||||
}
|
||||
|
||||
func abs(f float64) float64 {
|
||||
if f < 0 {
|
||||
return -f
|
||||
}
|
||||
return f
|
||||
}
|
||||
-328
@@ -1,328 +0,0 @@
|
||||
// Copyright 2014 Canonical Ltd.
|
||||
// Licensed under the LGPLv3 with static-linking exception.
|
||||
// See LICENCE file for details.
|
||||
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
gc "launchpad.net/gocheck"
|
||||
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPackage(t *testing.T) {
|
||||
gc.TestingT(t)
|
||||
}
|
||||
|
||||
type rateLimitSuite struct{}
|
||||
|
||||
var _ = gc.Suite(rateLimitSuite{})
|
||||
|
||||
type takeReq struct {
|
||||
time time.Duration
|
||||
count int64
|
||||
expectWait time.Duration
|
||||
}
|
||||
|
||||
var takeTests = []struct {
|
||||
about string
|
||||
fillInterval time.Duration
|
||||
capacity int64
|
||||
reqs []takeReq
|
||||
}{{
|
||||
about: "serial requests",
|
||||
fillInterval: 250 * time.Millisecond,
|
||||
capacity: 10,
|
||||
reqs: []takeReq{{
|
||||
time: 0,
|
||||
count: 0,
|
||||
expectWait: 0,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 10,
|
||||
expectWait: 0,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 1,
|
||||
expectWait: 250 * time.Millisecond,
|
||||
}, {
|
||||
time: 250 * time.Millisecond,
|
||||
count: 1,
|
||||
expectWait: 250 * time.Millisecond,
|
||||
}},
|
||||
}, {
|
||||
about: "concurrent requests",
|
||||
fillInterval: 250 * time.Millisecond,
|
||||
capacity: 10,
|
||||
reqs: []takeReq{{
|
||||
time: 0,
|
||||
count: 10,
|
||||
expectWait: 0,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 2,
|
||||
expectWait: 500 * time.Millisecond,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 2,
|
||||
expectWait: 1000 * time.Millisecond,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 1,
|
||||
expectWait: 1250 * time.Millisecond,
|
||||
}},
|
||||
}, {
|
||||
about: "more than capacity",
|
||||
fillInterval: 1 * time.Millisecond,
|
||||
capacity: 10,
|
||||
reqs: []takeReq{{
|
||||
time: 0,
|
||||
count: 10,
|
||||
expectWait: 0,
|
||||
}, {
|
||||
time: 20 * time.Millisecond,
|
||||
count: 15,
|
||||
expectWait: 5 * time.Millisecond,
|
||||
}},
|
||||
}, {
|
||||
about: "sub-quantum time",
|
||||
fillInterval: 10 * time.Millisecond,
|
||||
capacity: 10,
|
||||
reqs: []takeReq{{
|
||||
time: 0,
|
||||
count: 10,
|
||||
expectWait: 0,
|
||||
}, {
|
||||
time: 7 * time.Millisecond,
|
||||
count: 1,
|
||||
expectWait: 3 * time.Millisecond,
|
||||
}, {
|
||||
time: 8 * time.Millisecond,
|
||||
count: 1,
|
||||
expectWait: 12 * time.Millisecond,
|
||||
}},
|
||||
}, {
|
||||
about: "within capacity",
|
||||
fillInterval: 10 * time.Millisecond,
|
||||
capacity: 5,
|
||||
reqs: []takeReq{{
|
||||
time: 0,
|
||||
count: 5,
|
||||
expectWait: 0,
|
||||
}, {
|
||||
time: 60 * time.Millisecond,
|
||||
count: 5,
|
||||
expectWait: 0,
|
||||
}, {
|
||||
time: 60 * time.Millisecond,
|
||||
count: 1,
|
||||
expectWait: 10 * time.Millisecond,
|
||||
}, {
|
||||
time: 80 * time.Millisecond,
|
||||
count: 2,
|
||||
expectWait: 10 * time.Millisecond,
|
||||
}},
|
||||
}}
|
||||
|
||||
func (rateLimitSuite) TestTake(c *gc.C) {
|
||||
for i, test := range takeTests {
|
||||
tb := NewBucket(test.fillInterval, test.capacity)
|
||||
for j, req := range test.reqs {
|
||||
d, ok := tb.take(tb.startTime.Add(req.time), req.count, infinityDuration)
|
||||
c.Assert(ok, gc.Equals, true)
|
||||
if d != req.expectWait {
|
||||
c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rateLimitSuite) TestTakeMaxDuration(c *gc.C) {
|
||||
for i, test := range takeTests {
|
||||
tb := NewBucket(test.fillInterval, test.capacity)
|
||||
for j, req := range test.reqs {
|
||||
if req.expectWait > 0 {
|
||||
d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait-1)
|
||||
c.Assert(ok, gc.Equals, false)
|
||||
c.Assert(d, gc.Equals, time.Duration(0))
|
||||
}
|
||||
d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait)
|
||||
c.Assert(ok, gc.Equals, true)
|
||||
if d != req.expectWait {
|
||||
c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type takeAvailableReq struct {
|
||||
time time.Duration
|
||||
count int64
|
||||
expect int64
|
||||
}
|
||||
|
||||
var takeAvailableTests = []struct {
|
||||
about string
|
||||
fillInterval time.Duration
|
||||
capacity int64
|
||||
reqs []takeAvailableReq
|
||||
}{{
|
||||
about: "serial requests",
|
||||
fillInterval: 250 * time.Millisecond,
|
||||
capacity: 10,
|
||||
reqs: []takeAvailableReq{{
|
||||
time: 0,
|
||||
count: 0,
|
||||
expect: 0,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 10,
|
||||
expect: 10,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 1,
|
||||
expect: 0,
|
||||
}, {
|
||||
time: 250 * time.Millisecond,
|
||||
count: 1,
|
||||
expect: 1,
|
||||
}},
|
||||
}, {
|
||||
about: "concurrent requests",
|
||||
fillInterval: 250 * time.Millisecond,
|
||||
capacity: 10,
|
||||
reqs: []takeAvailableReq{{
|
||||
time: 0,
|
||||
count: 5,
|
||||
expect: 5,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 2,
|
||||
expect: 2,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 5,
|
||||
expect: 3,
|
||||
}, {
|
||||
time: 0,
|
||||
count: 1,
|
||||
expect: 0,
|
||||
}},
|
||||
}, {
|
||||
about: "more than capacity",
|
||||
fillInterval: 1 * time.Millisecond,
|
||||
capacity: 10,
|
||||
reqs: []takeAvailableReq{{
|
||||
time: 0,
|
||||
count: 10,
|
||||
expect: 10,
|
||||
}, {
|
||||
time: 20 * time.Millisecond,
|
||||
count: 15,
|
||||
expect: 10,
|
||||
}},
|
||||
}, {
|
||||
about: "within capacity",
|
||||
fillInterval: 10 * time.Millisecond,
|
||||
capacity: 5,
|
||||
reqs: []takeAvailableReq{{
|
||||
time: 0,
|
||||
count: 5,
|
||||
expect: 5,
|
||||
}, {
|
||||
time: 60 * time.Millisecond,
|
||||
count: 5,
|
||||
expect: 5,
|
||||
}, {
|
||||
time: 70 * time.Millisecond,
|
||||
count: 1,
|
||||
expect: 1,
|
||||
}},
|
||||
}}
|
||||
|
||||
func (rateLimitSuite) TestTakeAvailable(c *gc.C) {
|
||||
for i, test := range takeAvailableTests {
|
||||
tb := NewBucket(test.fillInterval, test.capacity)
|
||||
for j, req := range test.reqs {
|
||||
d := tb.takeAvailable(tb.startTime.Add(req.time), req.count)
|
||||
if d != req.expect {
|
||||
c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expect)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rateLimitSuite) TestPanics(c *gc.C) {
|
||||
c.Assert(func() { NewBucket(0, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
|
||||
c.Assert(func() { NewBucket(-2, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
|
||||
c.Assert(func() { NewBucket(1, 0) }, gc.PanicMatches, "token bucket capacity is not > 0")
|
||||
c.Assert(func() { NewBucket(1, -2) }, gc.PanicMatches, "token bucket capacity is not > 0")
|
||||
}
|
||||
|
||||
func isCloseTo(x, y, tolerance float64) bool {
|
||||
return abs(x-y)/y < tolerance
|
||||
}
|
||||
|
||||
func (rateLimitSuite) TestRate(c *gc.C) {
|
||||
tb := NewBucket(1, 1)
|
||||
if !isCloseTo(tb.Rate(), 1e9, 0.00001) {
|
||||
c.Fatalf("got %v want 1e9", tb.Rate())
|
||||
}
|
||||
tb = NewBucket(2*time.Second, 1)
|
||||
if !isCloseTo(tb.Rate(), 0.5, 0.00001) {
|
||||
c.Fatalf("got %v want 0.5", tb.Rate())
|
||||
}
|
||||
tb = NewBucketWithQuantum(100*time.Millisecond, 1, 5)
|
||||
if !isCloseTo(tb.Rate(), 50, 0.00001) {
|
||||
c.Fatalf("got %v want 50", tb.Rate())
|
||||
}
|
||||
}
|
||||
|
||||
func checkRate(c *gc.C, rate float64) {
|
||||
tb := NewBucketWithRate(rate, 1<<62)
|
||||
if !isCloseTo(tb.Rate(), rate, rateMargin) {
|
||||
c.Fatalf("got %g want %v", tb.Rate(), rate)
|
||||
}
|
||||
d, ok := tb.take(tb.startTime, 1<<62, infinityDuration)
|
||||
c.Assert(ok, gc.Equals, true)
|
||||
c.Assert(d, gc.Equals, time.Duration(0))
|
||||
|
||||
// Check that the actual rate is as expected by
|
||||
// asking for a not-quite multiple of the bucket's
|
||||
// quantum and checking that the wait time
|
||||
// correct.
|
||||
d, ok = tb.take(tb.startTime, tb.quantum*2-tb.quantum/2, infinityDuration)
|
||||
c.Assert(ok, gc.Equals, true)
|
||||
expectTime := 1e9 * float64(tb.quantum) * 2 / rate
|
||||
if !isCloseTo(float64(d), expectTime, rateMargin) {
|
||||
c.Fatalf("rate %g: got %g want %v", rate, float64(d), expectTime)
|
||||
}
|
||||
}
|
||||
|
||||
func (rateLimitSuite) TestNewWithRate(c *gc.C) {
|
||||
for rate := float64(1); rate < 1e6; rate += 7 {
|
||||
checkRate(c, rate)
|
||||
}
|
||||
for _, rate := range []float64{
|
||||
1024 * 1024 * 1024,
|
||||
1e-5,
|
||||
0.9e-5,
|
||||
0.5,
|
||||
0.9,
|
||||
0.9e8,
|
||||
3e12,
|
||||
4e18,
|
||||
} {
|
||||
checkRate(c, rate)
|
||||
checkRate(c, rate/3)
|
||||
checkRate(c, rate*1.3)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWait(b *testing.B) {
|
||||
tb := NewBucket(1, 16*1024)
|
||||
for i := b.N - 1; i >= 0; i-- {
|
||||
tb.Wait(1)
|
||||
}
|
||||
}
|
||||
-51
@@ -1,51 +0,0 @@
|
||||
// Copyright 2014 Canonical Ltd.
|
||||
// Licensed under the LGPLv3 with static-linking exception.
|
||||
// See LICENCE file for details.
|
||||
|
||||
package ratelimit
|
||||
|
||||
import "io"
|
||||
|
||||
type reader struct {
|
||||
r io.Reader
|
||||
bucket *Bucket
|
||||
}
|
||||
|
||||
// Reader returns a reader that is rate limited by
|
||||
// the given token bucket. Each token in the bucket
|
||||
// represents one byte.
|
||||
func Reader(r io.Reader, bucket *Bucket) io.Reader {
|
||||
return &reader{
|
||||
r: r,
|
||||
bucket: bucket,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reader) Read(buf []byte) (int, error) {
|
||||
n, err := r.r.Read(buf)
|
||||
if n <= 0 {
|
||||
return n, err
|
||||
}
|
||||
r.bucket.Wait(int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
w io.Writer
|
||||
bucket *Bucket
|
||||
}
|
||||
|
||||
// Writer returns a reader that is rate limited by
|
||||
// the given token bucket. Each token in the bucket
|
||||
// represents one byte.
|
||||
func Writer(w io.Writer, bucket *Bucket) io.Writer {
|
||||
return &writer{
|
||||
w: w,
|
||||
bucket: bucket,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *writer) Write(buf []byte) (int, error) {
|
||||
w.bucket.Wait(int64(len(buf)))
|
||||
return w.w.Write(buf)
|
||||
}
|
||||
-216
@@ -1,216 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
)
|
||||
|
||||
var (
|
||||
errBatchTooShort = errors.New("leveldb: batch is too short")
|
||||
errBatchBadRecord = errors.New("leveldb: bad record in batch")
|
||||
)
|
||||
|
||||
const kBatchHdrLen = 8 + 4
|
||||
|
||||
type batchReplay interface {
|
||||
put(key, value []byte, seq uint64)
|
||||
delete(key []byte, seq uint64)
|
||||
}
|
||||
|
||||
// Batch is a write batch.
|
||||
type Batch struct {
|
||||
buf []byte
|
||||
rLen, bLen int
|
||||
seq uint64
|
||||
sync bool
|
||||
}
|
||||
|
||||
func (b *Batch) grow(n int) {
|
||||
off := len(b.buf)
|
||||
if off == 0 {
|
||||
// include headers
|
||||
off = kBatchHdrLen
|
||||
n += off
|
||||
}
|
||||
if cap(b.buf)-off >= n {
|
||||
return
|
||||
}
|
||||
buf := make([]byte, 2*cap(b.buf)+n)
|
||||
copy(buf, b.buf)
|
||||
b.buf = buf[:off]
|
||||
}
|
||||
|
||||
func (b *Batch) appendRec(t vType, key, value []byte) {
|
||||
n := 1 + binary.MaxVarintLen32 + len(key)
|
||||
if t == tVal {
|
||||
n += binary.MaxVarintLen32 + len(value)
|
||||
}
|
||||
b.grow(n)
|
||||
off := len(b.buf)
|
||||
buf := b.buf[:off+n]
|
||||
buf[off] = byte(t)
|
||||
off += 1
|
||||
off += binary.PutUvarint(buf[off:], uint64(len(key)))
|
||||
copy(buf[off:], key)
|
||||
off += len(key)
|
||||
if t == tVal {
|
||||
off += binary.PutUvarint(buf[off:], uint64(len(value)))
|
||||
copy(buf[off:], value)
|
||||
off += len(value)
|
||||
}
|
||||
b.buf = buf[:off]
|
||||
b.rLen++
|
||||
// Include 8-byte ikey header
|
||||
b.bLen += len(key) + len(value) + 8
|
||||
}
|
||||
|
||||
// Put appends 'put operation' of the given key/value pair to the batch.
|
||||
// It is safe to modify the contents of the argument after Put returns.
|
||||
func (b *Batch) Put(key, value []byte) {
|
||||
b.appendRec(tVal, key, value)
|
||||
}
|
||||
|
||||
// Delete appends 'delete operation' of the given key to the batch.
|
||||
// It is safe to modify the contents of the argument after Delete returns.
|
||||
func (b *Batch) Delete(key []byte) {
|
||||
b.appendRec(tDel, key, nil)
|
||||
}
|
||||
|
||||
// Reset resets the batch.
|
||||
func (b *Batch) Reset() {
|
||||
b.buf = nil
|
||||
b.seq = 0
|
||||
b.rLen = 0
|
||||
b.bLen = 0
|
||||
b.sync = false
|
||||
}
|
||||
|
||||
func (b *Batch) init(sync bool) {
|
||||
b.sync = sync
|
||||
}
|
||||
|
||||
func (b *Batch) put(key, value []byte, seq uint64) {
|
||||
if b.rLen == 0 {
|
||||
b.seq = seq
|
||||
}
|
||||
b.Put(key, value)
|
||||
}
|
||||
|
||||
func (b *Batch) delete(key []byte, seq uint64) {
|
||||
if b.rLen == 0 {
|
||||
b.seq = seq
|
||||
}
|
||||
b.Delete(key)
|
||||
}
|
||||
|
||||
func (b *Batch) append(p *Batch) {
|
||||
if p.rLen > 0 {
|
||||
b.grow(len(p.buf) - kBatchHdrLen)
|
||||
b.buf = append(b.buf, p.buf[kBatchHdrLen:]...)
|
||||
b.rLen += p.rLen
|
||||
}
|
||||
if p.sync {
|
||||
b.sync = true
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Batch) len() int {
|
||||
return b.rLen
|
||||
}
|
||||
|
||||
func (b *Batch) size() int {
|
||||
return b.bLen
|
||||
}
|
||||
|
||||
func (b *Batch) encode() []byte {
|
||||
b.grow(0)
|
||||
binary.LittleEndian.PutUint64(b.buf, b.seq)
|
||||
binary.LittleEndian.PutUint32(b.buf[8:], uint32(b.rLen))
|
||||
|
||||
return b.buf
|
||||
}
|
||||
|
||||
func (b *Batch) decode(buf []byte) error {
|
||||
if len(buf) < kBatchHdrLen {
|
||||
return errBatchTooShort
|
||||
}
|
||||
|
||||
b.seq = binary.LittleEndian.Uint64(buf)
|
||||
b.rLen = int(binary.LittleEndian.Uint32(buf[8:]))
|
||||
// No need to be precise at this point, it won't be used anyway
|
||||
b.bLen = len(buf) - kBatchHdrLen
|
||||
b.buf = buf
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Batch) decodeRec(f func(i int, t vType, key, value []byte)) error {
|
||||
off := kBatchHdrLen
|
||||
for i := 0; i < b.rLen; i++ {
|
||||
if off >= len(b.buf) {
|
||||
return errors.New("leveldb: invalid batch record length")
|
||||
}
|
||||
|
||||
t := vType(b.buf[off])
|
||||
if t > tVal {
|
||||
return errors.New("leveldb: invalid batch record type in batch")
|
||||
}
|
||||
off += 1
|
||||
|
||||
x, n := binary.Uvarint(b.buf[off:])
|
||||
off += n
|
||||
if n <= 0 || off+int(x) > len(b.buf) {
|
||||
return errBatchBadRecord
|
||||
}
|
||||
key := b.buf[off : off+int(x)]
|
||||
off += int(x)
|
||||
|
||||
var value []byte
|
||||
if t == tVal {
|
||||
x, n := binary.Uvarint(b.buf[off:])
|
||||
off += n
|
||||
if n <= 0 || off+int(x) > len(b.buf) {
|
||||
return errBatchBadRecord
|
||||
}
|
||||
value = b.buf[off : off+int(x)]
|
||||
off += int(x)
|
||||
}
|
||||
|
||||
f(i, t, key, value)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Batch) replay(to batchReplay) error {
|
||||
return b.decodeRec(func(i int, t vType, key, value []byte) {
|
||||
switch t {
|
||||
case tVal:
|
||||
to.put(key, value, b.seq+uint64(i))
|
||||
case tDel:
|
||||
to.delete(key, b.seq+uint64(i))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Batch) memReplay(to *memdb.DB) error {
|
||||
return b.decodeRec(func(i int, t vType, key, value []byte) {
|
||||
ikey := newIKey(key, b.seq+uint64(i), t)
|
||||
to.Put(ikey, value)
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Batch) revertMemReplay(to *memdb.DB) error {
|
||||
return b.decodeRec(func(i int, t vType, key, value []byte) {
|
||||
ikey := newIKey(key, b.seq+uint64(i), t)
|
||||
to.Delete(ikey)
|
||||
})
|
||||
}
|
||||
-120
@@ -1,120 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
)
|
||||
|
||||
type tbRec struct {
|
||||
t vType
|
||||
key, value []byte
|
||||
}
|
||||
|
||||
type testBatch struct {
|
||||
rec []*tbRec
|
||||
}
|
||||
|
||||
func (p *testBatch) put(key, value []byte, seq uint64) {
|
||||
p.rec = append(p.rec, &tbRec{tVal, key, value})
|
||||
}
|
||||
|
||||
func (p *testBatch) delete(key []byte, seq uint64) {
|
||||
p.rec = append(p.rec, &tbRec{tDel, key, nil})
|
||||
}
|
||||
|
||||
func compareBatch(t *testing.T, b1, b2 *Batch) {
|
||||
if b1.seq != b2.seq {
|
||||
t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
|
||||
}
|
||||
if b1.len() != b2.len() {
|
||||
t.Fatalf("invalid record length want %d, got %d", b1.len(), b2.len())
|
||||
}
|
||||
p1, p2 := new(testBatch), new(testBatch)
|
||||
err := b1.replay(p1)
|
||||
if err != nil {
|
||||
t.Fatal("error when replaying batch 1: ", err)
|
||||
}
|
||||
err = b2.replay(p2)
|
||||
if err != nil {
|
||||
t.Fatal("error when replaying batch 2: ", err)
|
||||
}
|
||||
for i := range p1.rec {
|
||||
r1, r2 := p1.rec[i], p2.rec[i]
|
||||
if r1.t != r2.t {
|
||||
t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.t, r2.t)
|
||||
}
|
||||
if !bytes.Equal(r1.key, r2.key) {
|
||||
t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
|
||||
}
|
||||
if r1.t == tVal {
|
||||
if !bytes.Equal(r1.value, r2.value) {
|
||||
t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatch_EncodeDecode(t *testing.T) {
|
||||
b1 := new(Batch)
|
||||
b1.seq = 10009
|
||||
b1.Put([]byte("key1"), []byte("value1"))
|
||||
b1.Put([]byte("key2"), []byte("value2"))
|
||||
b1.Delete([]byte("key1"))
|
||||
b1.Put([]byte("k"), []byte(""))
|
||||
b1.Put([]byte("zzzzzzzzzzz"), []byte("zzzzzzzzzzzzzzzzzzzzzzzz"))
|
||||
b1.Delete([]byte("key10000"))
|
||||
b1.Delete([]byte("k"))
|
||||
buf := b1.encode()
|
||||
b2 := new(Batch)
|
||||
err := b2.decode(buf)
|
||||
if err != nil {
|
||||
t.Error("error when decoding batch: ", err)
|
||||
}
|
||||
compareBatch(t, b1, b2)
|
||||
}
|
||||
|
||||
func TestBatch_Append(t *testing.T) {
|
||||
b1 := new(Batch)
|
||||
b1.seq = 10009
|
||||
b1.Put([]byte("key1"), []byte("value1"))
|
||||
b1.Put([]byte("key2"), []byte("value2"))
|
||||
b1.Delete([]byte("key1"))
|
||||
b1.Put([]byte("foo"), []byte("foovalue"))
|
||||
b1.Put([]byte("bar"), []byte("barvalue"))
|
||||
b2a := new(Batch)
|
||||
b2a.seq = 10009
|
||||
b2a.Put([]byte("key1"), []byte("value1"))
|
||||
b2a.Put([]byte("key2"), []byte("value2"))
|
||||
b2a.Delete([]byte("key1"))
|
||||
b2b := new(Batch)
|
||||
b2b.Put([]byte("foo"), []byte("foovalue"))
|
||||
b2b.Put([]byte("bar"), []byte("barvalue"))
|
||||
b2a.append(b2b)
|
||||
compareBatch(t, b1, b2a)
|
||||
}
|
||||
|
||||
func TestBatch_Size(t *testing.T) {
|
||||
b := new(Batch)
|
||||
for i := 0; i < 2; i++ {
|
||||
b.Put([]byte("key1"), []byte("value1"))
|
||||
b.Put([]byte("key2"), []byte("value2"))
|
||||
b.Delete([]byte("key1"))
|
||||
b.Put([]byte("foo"), []byte("foovalue"))
|
||||
b.Put([]byte("bar"), []byte("barvalue"))
|
||||
mem := memdb.New(&iComparer{comparer.DefaultComparer}, 0)
|
||||
b.memReplay(mem)
|
||||
if b.size() != mem.Size() {
|
||||
t.Errorf("invalid batch size calculation, want=%d got=%d", mem.Size(), b.size())
|
||||
}
|
||||
b.Reset()
|
||||
}
|
||||
}
|
||||
-461
@@ -1,461 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
func randomString(r *rand.Rand, n int) []byte {
|
||||
b := new(bytes.Buffer)
|
||||
for i := 0; i < n; i++ {
|
||||
b.WriteByte(' ' + byte(r.Intn(95)))
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func compressibleStr(r *rand.Rand, frac float32, n int) []byte {
|
||||
nn := int(float32(n) * frac)
|
||||
rb := randomString(r, nn)
|
||||
b := make([]byte, 0, n+nn)
|
||||
for len(b) < n {
|
||||
b = append(b, rb...)
|
||||
}
|
||||
return b[:n]
|
||||
}
|
||||
|
||||
type valueGen struct {
|
||||
src []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
func newValueGen(frac float32) *valueGen {
|
||||
v := new(valueGen)
|
||||
r := rand.New(rand.NewSource(301))
|
||||
v.src = make([]byte, 0, 1048576+100)
|
||||
for len(v.src) < 1048576 {
|
||||
v.src = append(v.src, compressibleStr(r, frac, 100)...)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (v *valueGen) get(n int) []byte {
|
||||
if v.pos+n > len(v.src) {
|
||||
v.pos = 0
|
||||
}
|
||||
v.pos += n
|
||||
return v.src[v.pos-n : v.pos]
|
||||
}
|
||||
|
||||
var benchDB = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbbench-%d", os.Getuid()))
|
||||
|
||||
type dbBench struct {
|
||||
b *testing.B
|
||||
stor storage.Storage
|
||||
db *DB
|
||||
|
||||
o *opt.Options
|
||||
ro *opt.ReadOptions
|
||||
wo *opt.WriteOptions
|
||||
|
||||
keys, values [][]byte
|
||||
}
|
||||
|
||||
func openDBBench(b *testing.B, noCompress bool) *dbBench {
|
||||
_, err := os.Stat(benchDB)
|
||||
if err == nil {
|
||||
err = os.RemoveAll(benchDB)
|
||||
if err != nil {
|
||||
b.Fatal("cannot remove old db: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
p := &dbBench{
|
||||
b: b,
|
||||
o: &opt.Options{},
|
||||
ro: &opt.ReadOptions{},
|
||||
wo: &opt.WriteOptions{},
|
||||
}
|
||||
p.stor, err = storage.OpenFile(benchDB)
|
||||
if err != nil {
|
||||
b.Fatal("cannot open stor: ", err)
|
||||
}
|
||||
if noCompress {
|
||||
p.o.Compression = opt.NoCompression
|
||||
}
|
||||
|
||||
p.db, err = Open(p.stor, p.o)
|
||||
if err != nil {
|
||||
b.Fatal("cannot open db: ", err)
|
||||
}
|
||||
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *dbBench) reopen() {
|
||||
p.db.Close()
|
||||
var err error
|
||||
p.db, err = Open(p.stor, p.o)
|
||||
if err != nil {
|
||||
p.b.Fatal("Reopen: got error: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *dbBench) populate(n int) {
|
||||
p.keys, p.values = make([][]byte, n), make([][]byte, n)
|
||||
v := newValueGen(0.5)
|
||||
for i := range p.keys {
|
||||
p.keys[i], p.values[i] = []byte(fmt.Sprintf("%016d", i)), v.get(100)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *dbBench) randomize() {
|
||||
m := len(p.keys)
|
||||
times := m * 2
|
||||
r1, r2 := rand.New(rand.NewSource(0xdeadbeef)), rand.New(rand.NewSource(0xbeefface))
|
||||
for n := 0; n < times; n++ {
|
||||
i, j := r1.Int()%m, r2.Int()%m
|
||||
if i == j {
|
||||
continue
|
||||
}
|
||||
p.keys[i], p.keys[j] = p.keys[j], p.keys[i]
|
||||
p.values[i], p.values[j] = p.values[j], p.values[i]
|
||||
}
|
||||
}
|
||||
|
||||
func (p *dbBench) writes(perBatch int) {
|
||||
b := p.b
|
||||
db := p.db
|
||||
|
||||
n := len(p.keys)
|
||||
m := n / perBatch
|
||||
if n%perBatch > 0 {
|
||||
m++
|
||||
}
|
||||
batches := make([]Batch, m)
|
||||
j := 0
|
||||
for i := range batches {
|
||||
first := true
|
||||
for ; j < n && ((j+1)%perBatch != 0 || first); j++ {
|
||||
first = false
|
||||
batches[i].Put(p.keys[j], p.values[j])
|
||||
}
|
||||
}
|
||||
runtime.GC()
|
||||
|
||||
b.ResetTimer()
|
||||
b.StartTimer()
|
||||
for i := range batches {
|
||||
err := db.Write(&(batches[i]), p.wo)
|
||||
if err != nil {
|
||||
b.Fatal("write failed: ", err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
}
|
||||
|
||||
func (p *dbBench) drop() {
|
||||
p.keys, p.values = nil, nil
|
||||
runtime.GC()
|
||||
}
|
||||
|
||||
func (p *dbBench) puts() {
|
||||
b := p.b
|
||||
db := p.db
|
||||
|
||||
b.ResetTimer()
|
||||
b.StartTimer()
|
||||
for i := range p.keys {
|
||||
err := db.Put(p.keys[i], p.values[i], p.wo)
|
||||
if err != nil {
|
||||
b.Fatal("put failed: ", err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
}
|
||||
|
||||
func (p *dbBench) fill() {
|
||||
b := p.b
|
||||
db := p.db
|
||||
|
||||
perBatch := 10000
|
||||
batch := new(Batch)
|
||||
for i, n := 0, len(p.keys); i < n; {
|
||||
first := true
|
||||
for ; i < n && ((i+1)%perBatch != 0 || first); i++ {
|
||||
first = false
|
||||
batch.Put(p.keys[i], p.values[i])
|
||||
}
|
||||
err := db.Write(batch, p.wo)
|
||||
if err != nil {
|
||||
b.Fatal("write failed: ", err)
|
||||
}
|
||||
batch.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *dbBench) gets() {
|
||||
b := p.b
|
||||
db := p.db
|
||||
|
||||
b.ResetTimer()
|
||||
for i := range p.keys {
|
||||
_, err := db.Get(p.keys[i], p.ro)
|
||||
if err != nil {
|
||||
b.Error("got error: ", err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
func (p *dbBench) seeks() {
|
||||
b := p.b
|
||||
|
||||
iter := p.newIter()
|
||||
defer iter.Release()
|
||||
b.ResetTimer()
|
||||
for i := range p.keys {
|
||||
if !iter.Seek(p.keys[i]) {
|
||||
b.Error("value not found for: ", string(p.keys[i]))
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
func (p *dbBench) newIter() iterator.Iterator {
|
||||
iter := p.db.NewIterator(nil, p.ro)
|
||||
err := iter.Error()
|
||||
if err != nil {
|
||||
p.b.Fatal("cannot create iterator: ", err)
|
||||
}
|
||||
return iter
|
||||
}
|
||||
|
||||
func (p *dbBench) close() {
|
||||
p.db.Close()
|
||||
p.stor.Close()
|
||||
os.RemoveAll(benchDB)
|
||||
p.db = nil
|
||||
p.keys = nil
|
||||
p.values = nil
|
||||
runtime.GC()
|
||||
runtime.GOMAXPROCS(1)
|
||||
}
|
||||
|
||||
func BenchmarkDBWrite(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.writes(1)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBWriteBatch(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.writes(1000)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBWriteUncompressed(b *testing.B) {
|
||||
p := openDBBench(b, true)
|
||||
p.populate(b.N)
|
||||
p.writes(1)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBWriteBatchUncompressed(b *testing.B) {
|
||||
p := openDBBench(b, true)
|
||||
p.populate(b.N)
|
||||
p.writes(1000)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBWriteRandom(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.randomize()
|
||||
p.writes(1)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBWriteRandomSync(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.wo.Sync = true
|
||||
p.populate(b.N)
|
||||
p.writes(1)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBOverwrite(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.writes(1)
|
||||
p.writes(1)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBOverwriteRandom(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.writes(1)
|
||||
p.randomize()
|
||||
p.writes(1)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBPut(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.puts()
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBRead(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.drop()
|
||||
|
||||
iter := p.newIter()
|
||||
b.ResetTimer()
|
||||
for iter.Next() {
|
||||
}
|
||||
iter.Release()
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBReadGC(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
|
||||
iter := p.newIter()
|
||||
b.ResetTimer()
|
||||
for iter.Next() {
|
||||
}
|
||||
iter.Release()
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBReadUncompressed(b *testing.B) {
|
||||
p := openDBBench(b, true)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.drop()
|
||||
|
||||
iter := p.newIter()
|
||||
b.ResetTimer()
|
||||
for iter.Next() {
|
||||
}
|
||||
iter.Release()
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBReadTable(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.reopen()
|
||||
p.drop()
|
||||
|
||||
iter := p.newIter()
|
||||
b.ResetTimer()
|
||||
for iter.Next() {
|
||||
}
|
||||
iter.Release()
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBReadReverse(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.drop()
|
||||
|
||||
iter := p.newIter()
|
||||
b.ResetTimer()
|
||||
iter.Last()
|
||||
for iter.Prev() {
|
||||
}
|
||||
iter.Release()
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBReadReverseTable(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.reopen()
|
||||
p.drop()
|
||||
|
||||
iter := p.newIter()
|
||||
b.ResetTimer()
|
||||
iter.Last()
|
||||
for iter.Prev() {
|
||||
}
|
||||
iter.Release()
|
||||
b.StopTimer()
|
||||
b.SetBytes(116)
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBSeek(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.seeks()
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBSeekRandom(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.randomize()
|
||||
p.seeks()
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBGet(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.gets()
|
||||
p.close()
|
||||
}
|
||||
|
||||
func BenchmarkDBGetRandom(b *testing.B) {
|
||||
p := openDBBench(b, false)
|
||||
p.populate(b.N)
|
||||
p.fill()
|
||||
p.randomize()
|
||||
p.gets()
|
||||
p.close()
|
||||
}
|
||||
-125
@@ -1,125 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package cache provides interface and implementation of a cache algorithms.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// SetFunc used by Namespace.Get method to create a cache object. SetFunc
|
||||
// may return ok false, in that case the cache object will not be created.
|
||||
type SetFunc func() (ok bool, value interface{}, charge int, fin SetFin)
|
||||
|
||||
// SetFin will be called when corresponding cache object are released.
|
||||
type SetFin func()
|
||||
|
||||
// DelFin will be called when corresponding cache object are released.
|
||||
// DelFin will be called after SetFin. The exist is true if the corresponding
|
||||
// cache object is actually exist in the cache tree.
|
||||
type DelFin func(exist bool)
|
||||
|
||||
// PurgeFin will be called when corresponding cache object are released.
|
||||
// PurgeFin will be called after SetFin. If PurgeFin present DelFin will
|
||||
// not be executed but passed to the PurgeFin, it is up to the caller
|
||||
// to call it or not.
|
||||
type PurgeFin func(ns, key uint64, delfin DelFin)
|
||||
|
||||
// Cache is a cache tree.
|
||||
type Cache interface {
|
||||
// SetCapacity sets cache capacity.
|
||||
SetCapacity(capacity int)
|
||||
|
||||
// GetNamespace gets or creates a cache namespace for the given id.
|
||||
GetNamespace(id uint64) Namespace
|
||||
|
||||
// Purge purges all cache namespaces, read Namespace.Purge method documentation.
|
||||
Purge(fin PurgeFin)
|
||||
|
||||
// Zap zaps all cache namespaces, read Namespace.Zap method documentation.
|
||||
Zap(closed bool)
|
||||
}
|
||||
|
||||
// Namespace is a cache namespace.
|
||||
type Namespace interface {
|
||||
// Get gets cache object for the given key. The given SetFunc (if not nil) will
|
||||
// be called if the given key does not exist.
|
||||
// If the given key does not exist, SetFunc is nil or SetFunc return ok false, Get
|
||||
// will return ok false.
|
||||
Get(key uint64, setf SetFunc) (obj Object, ok bool)
|
||||
|
||||
// Get deletes cache object for the given key. If exist the cache object will
|
||||
// be deleted later when all of its handles have been released (i.e. no one use
|
||||
// it anymore) and the given DelFin (if not nil) will finally be executed. If
|
||||
// such cache object does not exist the given DelFin will be executed anyway.
|
||||
//
|
||||
// Delete returns true if such cache object exist.
|
||||
Delete(key uint64, fin DelFin) bool
|
||||
|
||||
// Purge deletes all cache objects, read Delete method documentation.
|
||||
Purge(fin PurgeFin)
|
||||
|
||||
// Zap detaches the namespace from the cache tree and delete all its cache
|
||||
// objects. The cache objects deletion and finalizers execution are happen
|
||||
// immediately, even if its existing handles haven't yet been released.
|
||||
// A zapped namespace can't never be filled again.
|
||||
// If closed is false then the Get function will always call the given SetFunc
|
||||
// if it is not nil, but resultant of the SetFunc will not be cached.
|
||||
Zap(closed bool)
|
||||
}
|
||||
|
||||
// Object is a cache object.
|
||||
type Object interface {
|
||||
// Release releases the cache object. Other methods should not be called
|
||||
// after the cache object has been released.
|
||||
Release()
|
||||
|
||||
// Value returns value of the cache object.
|
||||
Value() interface{}
|
||||
}
|
||||
|
||||
// Namespace state.
|
||||
type nsState int
|
||||
|
||||
const (
|
||||
nsEffective nsState = iota
|
||||
nsZapped
|
||||
nsClosed
|
||||
)
|
||||
|
||||
// Node state.
|
||||
type nodeState int
|
||||
|
||||
const (
|
||||
nodeEffective nodeState = iota
|
||||
nodeEvicted
|
||||
nodeRemoved
|
||||
)
|
||||
|
||||
// Fake object.
|
||||
type fakeObject struct {
|
||||
value interface{}
|
||||
fin func()
|
||||
once uint32
|
||||
}
|
||||
|
||||
func (o *fakeObject) Value() interface{} {
|
||||
if atomic.LoadUint32(&o.once) == 0 {
|
||||
return o.value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *fakeObject) Release() {
|
||||
if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
|
||||
return
|
||||
}
|
||||
if o.fin != nil {
|
||||
o.fin()
|
||||
o.fin = nil
|
||||
}
|
||||
}
|
||||
-236
@@ -1,236 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func set(ns Namespace, key uint64, value interface{}, charge int, fin func()) Object {
|
||||
obj, _ := ns.Get(key, func() (bool, interface{}, int, SetFin) {
|
||||
return true, value, charge, fin
|
||||
})
|
||||
return obj
|
||||
}
|
||||
|
||||
func TestCache_HitMiss(t *testing.T) {
|
||||
cases := []struct {
|
||||
key uint64
|
||||
value string
|
||||
}{
|
||||
{1, "vvvvvvvvv"},
|
||||
{100, "v1"},
|
||||
{0, "v2"},
|
||||
{12346, "v3"},
|
||||
{777, "v4"},
|
||||
{999, "v5"},
|
||||
{7654, "v6"},
|
||||
{2, "v7"},
|
||||
{3, "v8"},
|
||||
{9, "v9"},
|
||||
}
|
||||
|
||||
setfin := 0
|
||||
c := NewLRUCache(1000)
|
||||
ns := c.GetNamespace(0)
|
||||
for i, x := range cases {
|
||||
set(ns, x.key, x.value, len(x.value), func() {
|
||||
setfin++
|
||||
}).Release()
|
||||
for j, y := range cases {
|
||||
r, ok := ns.Get(y.key, nil)
|
||||
if j <= i {
|
||||
// should hit
|
||||
if !ok {
|
||||
t.Errorf("case '%d' iteration '%d' is miss", i, j)
|
||||
} else if r.Value().(string) != y.value {
|
||||
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
|
||||
}
|
||||
} else {
|
||||
// should miss
|
||||
if ok {
|
||||
t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, r.Value().(string))
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
r.Release()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i, x := range cases {
|
||||
finalizerOk := false
|
||||
ns.Delete(x.key, func(exist bool) {
|
||||
finalizerOk = true
|
||||
})
|
||||
|
||||
if !finalizerOk {
|
||||
t.Errorf("case %d delete finalizer not executed", i)
|
||||
}
|
||||
|
||||
for j, y := range cases {
|
||||
r, ok := ns.Get(y.key, nil)
|
||||
if j > i {
|
||||
// should hit
|
||||
if !ok {
|
||||
t.Errorf("case '%d' iteration '%d' is miss", i, j)
|
||||
} else if r.Value().(string) != y.value {
|
||||
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
|
||||
}
|
||||
} else {
|
||||
// should miss
|
||||
if ok {
|
||||
t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, r.Value().(string))
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
r.Release()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if setfin != len(cases) {
|
||||
t.Errorf("some set finalizer may not be executed, want=%d got=%d", len(cases), setfin)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLRUCache_Eviction(t *testing.T) {
|
||||
c := NewLRUCache(12)
|
||||
ns := c.GetNamespace(0)
|
||||
o1 := set(ns, 1, 1, 1, nil)
|
||||
set(ns, 2, 2, 1, nil).Release()
|
||||
set(ns, 3, 3, 1, nil).Release()
|
||||
set(ns, 4, 4, 1, nil).Release()
|
||||
set(ns, 5, 5, 1, nil).Release()
|
||||
if r, ok := ns.Get(2, nil); ok { // 1,3,4,5,2
|
||||
r.Release()
|
||||
}
|
||||
set(ns, 9, 9, 10, nil).Release() // 5,2,9
|
||||
|
||||
for _, x := range []uint64{9, 2, 5, 1} {
|
||||
r, ok := ns.Get(x, nil)
|
||||
if !ok {
|
||||
t.Errorf("miss for key '%d'", x)
|
||||
} else {
|
||||
if r.Value().(int) != int(x) {
|
||||
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
|
||||
}
|
||||
r.Release()
|
||||
}
|
||||
}
|
||||
o1.Release()
|
||||
for _, x := range []uint64{1, 2, 5} {
|
||||
r, ok := ns.Get(x, nil)
|
||||
if !ok {
|
||||
t.Errorf("miss for key '%d'", x)
|
||||
} else {
|
||||
if r.Value().(int) != int(x) {
|
||||
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
|
||||
}
|
||||
r.Release()
|
||||
}
|
||||
}
|
||||
for _, x := range []uint64{3, 4, 9} {
|
||||
r, ok := ns.Get(x, nil)
|
||||
if ok {
|
||||
t.Errorf("hit for key '%d'", x)
|
||||
if r.Value().(int) != int(x) {
|
||||
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
|
||||
}
|
||||
r.Release()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLRUCache_SetGet(t *testing.T) {
|
||||
c := NewLRUCache(13)
|
||||
ns := c.GetNamespace(0)
|
||||
for i := 0; i < 200; i++ {
|
||||
n := uint64(rand.Intn(99999) % 20)
|
||||
set(ns, n, n, 1, nil).Release()
|
||||
if p, ok := ns.Get(n, nil); ok {
|
||||
if p.Value() == nil {
|
||||
t.Errorf("key '%d' contains nil value", n)
|
||||
} else {
|
||||
got := p.Value().(uint64)
|
||||
if got != n {
|
||||
t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, got)
|
||||
}
|
||||
}
|
||||
p.Release()
|
||||
} else {
|
||||
t.Errorf("key '%d' doesn't exist", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLRUCache_Purge(t *testing.T) {
|
||||
c := NewLRUCache(3)
|
||||
ns1 := c.GetNamespace(0)
|
||||
o1 := set(ns1, 1, 1, 1, nil)
|
||||
o2 := set(ns1, 2, 2, 1, nil)
|
||||
ns1.Purge(nil)
|
||||
set(ns1, 3, 3, 1, nil).Release()
|
||||
for _, x := range []uint64{1, 2, 3} {
|
||||
r, ok := ns1.Get(x, nil)
|
||||
if !ok {
|
||||
t.Errorf("miss for key '%d'", x)
|
||||
} else {
|
||||
if r.Value().(int) != int(x) {
|
||||
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
|
||||
}
|
||||
r.Release()
|
||||
}
|
||||
}
|
||||
o1.Release()
|
||||
o2.Release()
|
||||
for _, x := range []uint64{1, 2} {
|
||||
r, ok := ns1.Get(x, nil)
|
||||
if ok {
|
||||
t.Errorf("hit for key '%d'", x)
|
||||
if r.Value().(int) != int(x) {
|
||||
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
|
||||
}
|
||||
r.Release()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLRUCache_SetRelease(b *testing.B) {
|
||||
capacity := b.N / 100
|
||||
if capacity <= 0 {
|
||||
capacity = 10
|
||||
}
|
||||
c := NewLRUCache(capacity)
|
||||
ns := c.GetNamespace(0)
|
||||
b.ResetTimer()
|
||||
for i := uint64(0); i < uint64(b.N); i++ {
|
||||
set(ns, i, nil, 1, nil).Release()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
|
||||
capacity := b.N / 100
|
||||
if capacity <= 0 {
|
||||
capacity = 10
|
||||
}
|
||||
c := NewLRUCache(capacity)
|
||||
ns := c.GetNamespace(0)
|
||||
b.ResetTimer()
|
||||
|
||||
na := b.N / 2
|
||||
nb := b.N - na
|
||||
|
||||
for i := uint64(0); i < uint64(na); i++ {
|
||||
set(ns, i, nil, 1, nil).Release()
|
||||
}
|
||||
|
||||
for i := uint64(0); i < uint64(nb); i++ {
|
||||
set(ns, i, nil, 1, nil).Release()
|
||||
}
|
||||
}
|
||||
-246
@@ -1,246 +0,0 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
type emptyCache struct {
|
||||
sync.Mutex
|
||||
table map[uint64]*emptyNS
|
||||
}
|
||||
|
||||
// NewEmptyCache creates a new initialized empty cache.
|
||||
func NewEmptyCache() Cache {
|
||||
return &emptyCache{
|
||||
table: make(map[uint64]*emptyNS),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *emptyCache) GetNamespace(id uint64) Namespace {
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
|
||||
if ns, ok := c.table[id]; ok {
|
||||
return ns
|
||||
}
|
||||
|
||||
ns := &emptyNS{
|
||||
cache: c,
|
||||
id: id,
|
||||
table: make(map[uint64]*emptyNode),
|
||||
}
|
||||
c.table[id] = ns
|
||||
return ns
|
||||
}
|
||||
|
||||
func (c *emptyCache) Purge(fin PurgeFin) {
|
||||
c.Lock()
|
||||
for _, ns := range c.table {
|
||||
ns.purgeNB(fin)
|
||||
}
|
||||
c.Unlock()
|
||||
}
|
||||
|
||||
func (c *emptyCache) Zap(closed bool) {
|
||||
c.Lock()
|
||||
for _, ns := range c.table {
|
||||
ns.zapNB(closed)
|
||||
}
|
||||
c.table = make(map[uint64]*emptyNS)
|
||||
c.Unlock()
|
||||
}
|
||||
|
||||
func (*emptyCache) SetCapacity(capacity int) {}
|
||||
|
||||
type emptyNS struct {
|
||||
cache *emptyCache
|
||||
id uint64
|
||||
table map[uint64]*emptyNode
|
||||
state nsState
|
||||
}
|
||||
|
||||
func (ns *emptyNS) Get(key uint64, setf SetFunc) (o Object, ok bool) {
|
||||
ns.cache.Lock()
|
||||
|
||||
switch ns.state {
|
||||
case nsZapped:
|
||||
ns.cache.Unlock()
|
||||
if setf == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var value interface{}
|
||||
var fin func()
|
||||
ok, value, _, fin = setf()
|
||||
if ok {
|
||||
o = &fakeObject{
|
||||
value: value,
|
||||
fin: fin,
|
||||
}
|
||||
}
|
||||
return
|
||||
case nsClosed:
|
||||
ns.cache.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
n, ok := ns.table[key]
|
||||
if ok {
|
||||
n.ref++
|
||||
} else {
|
||||
if setf == nil {
|
||||
ns.cache.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
var value interface{}
|
||||
var fin func()
|
||||
ok, value, _, fin = setf()
|
||||
if !ok {
|
||||
ns.cache.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
n = &emptyNode{
|
||||
ns: ns,
|
||||
key: key,
|
||||
value: value,
|
||||
setfin: fin,
|
||||
ref: 1,
|
||||
}
|
||||
ns.table[key] = n
|
||||
}
|
||||
|
||||
ns.cache.Unlock()
|
||||
o = &emptyObject{node: n}
|
||||
return
|
||||
}
|
||||
|
||||
func (ns *emptyNS) Delete(key uint64, fin DelFin) bool {
|
||||
ns.cache.Lock()
|
||||
|
||||
if ns.state != nsEffective {
|
||||
ns.cache.Unlock()
|
||||
if fin != nil {
|
||||
fin(false)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
n, ok := ns.table[key]
|
||||
if !ok {
|
||||
ns.cache.Unlock()
|
||||
if fin != nil {
|
||||
fin(false)
|
||||
}
|
||||
return false
|
||||
}
|
||||
n.delfin = fin
|
||||
ns.cache.Unlock()
|
||||
return true
|
||||
}
|
||||
|
||||
func (ns *emptyNS) purgeNB(fin PurgeFin) {
|
||||
if ns.state != nsEffective {
|
||||
return
|
||||
}
|
||||
for _, n := range ns.table {
|
||||
n.purgefin = fin
|
||||
}
|
||||
}
|
||||
|
||||
func (ns *emptyNS) Purge(fin PurgeFin) {
|
||||
ns.cache.Lock()
|
||||
ns.purgeNB(fin)
|
||||
ns.cache.Unlock()
|
||||
}
|
||||
|
||||
func (ns *emptyNS) zapNB(closed bool) {
|
||||
if ns.state != nsEffective {
|
||||
return
|
||||
}
|
||||
for _, n := range ns.table {
|
||||
n.execFin()
|
||||
}
|
||||
if closed {
|
||||
ns.state = nsClosed
|
||||
} else {
|
||||
ns.state = nsZapped
|
||||
}
|
||||
ns.table = nil
|
||||
}
|
||||
|
||||
func (ns *emptyNS) Zap(closed bool) {
|
||||
ns.cache.Lock()
|
||||
ns.zapNB(closed)
|
||||
delete(ns.cache.table, ns.id)
|
||||
ns.cache.Unlock()
|
||||
}
|
||||
|
||||
type emptyNode struct {
|
||||
ns *emptyNS
|
||||
key uint64
|
||||
value interface{}
|
||||
ref int
|
||||
setfin SetFin
|
||||
delfin DelFin
|
||||
purgefin PurgeFin
|
||||
}
|
||||
|
||||
func (n *emptyNode) execFin() {
|
||||
if n.setfin != nil {
|
||||
n.setfin()
|
||||
n.setfin = nil
|
||||
}
|
||||
if n.purgefin != nil {
|
||||
n.purgefin(n.ns.id, n.key, n.delfin)
|
||||
n.delfin = nil
|
||||
n.purgefin = nil
|
||||
} else if n.delfin != nil {
|
||||
n.delfin(true)
|
||||
n.delfin = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (n *emptyNode) evict() {
|
||||
n.ns.cache.Lock()
|
||||
n.ref--
|
||||
if n.ref == 0 {
|
||||
if n.ns.state == nsEffective {
|
||||
// Remove elem.
|
||||
delete(n.ns.table, n.key)
|
||||
// Execute finalizer.
|
||||
n.execFin()
|
||||
}
|
||||
} else if n.ref < 0 {
|
||||
panic("leveldb/cache: emptyNode: negative node reference")
|
||||
}
|
||||
n.ns.cache.Unlock()
|
||||
}
|
||||
|
||||
type emptyObject struct {
|
||||
node *emptyNode
|
||||
once uint32
|
||||
}
|
||||
|
||||
func (o *emptyObject) Value() interface{} {
|
||||
if atomic.LoadUint32(&o.once) == 0 {
|
||||
return o.node.value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *emptyObject) Release() {
|
||||
if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
|
||||
return
|
||||
}
|
||||
o.node.evict()
|
||||
o.node = nil
|
||||
}
|
||||
-354
@@ -1,354 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// lruCache represent a LRU cache state.
|
||||
type lruCache struct {
|
||||
sync.Mutex
|
||||
|
||||
recent lruNode
|
||||
table map[uint64]*lruNs
|
||||
capacity int
|
||||
size int
|
||||
}
|
||||
|
||||
// NewLRUCache creates a new initialized LRU cache with the given capacity.
|
||||
func NewLRUCache(capacity int) Cache {
|
||||
c := &lruCache{
|
||||
table: make(map[uint64]*lruNs),
|
||||
capacity: capacity,
|
||||
}
|
||||
c.recent.rNext = &c.recent
|
||||
c.recent.rPrev = &c.recent
|
||||
return c
|
||||
}
|
||||
|
||||
// SetCapacity set cache capacity.
|
||||
func (c *lruCache) SetCapacity(capacity int) {
|
||||
c.Lock()
|
||||
c.capacity = capacity
|
||||
c.evict()
|
||||
c.Unlock()
|
||||
}
|
||||
|
||||
// GetNamespace return namespace object for given id.
|
||||
func (c *lruCache) GetNamespace(id uint64) Namespace {
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
|
||||
if p, ok := c.table[id]; ok {
|
||||
return p
|
||||
}
|
||||
|
||||
p := &lruNs{
|
||||
lru: c,
|
||||
id: id,
|
||||
table: make(map[uint64]*lruNode),
|
||||
}
|
||||
c.table[id] = p
|
||||
return p
|
||||
}
|
||||
|
||||
// Purge purge entire cache.
|
||||
func (c *lruCache) Purge(fin PurgeFin) {
|
||||
c.Lock()
|
||||
for _, ns := range c.table {
|
||||
ns.purgeNB(fin)
|
||||
}
|
||||
c.Unlock()
|
||||
}
|
||||
|
||||
func (c *lruCache) Zap(closed bool) {
|
||||
c.Lock()
|
||||
for _, ns := range c.table {
|
||||
ns.zapNB(closed)
|
||||
}
|
||||
c.table = make(map[uint64]*lruNs)
|
||||
c.Unlock()
|
||||
}
|
||||
|
||||
func (c *lruCache) evict() {
|
||||
top := &c.recent
|
||||
for n := c.recent.rPrev; c.size > c.capacity && n != top; {
|
||||
n.state = nodeEvicted
|
||||
n.rRemove()
|
||||
n.evictNB()
|
||||
c.size -= n.charge
|
||||
n = c.recent.rPrev
|
||||
}
|
||||
}
|
||||
|
||||
type lruNs struct {
|
||||
lru *lruCache
|
||||
id uint64
|
||||
table map[uint64]*lruNode
|
||||
state nsState
|
||||
}
|
||||
|
||||
func (ns *lruNs) Get(key uint64, setf SetFunc) (o Object, ok bool) {
|
||||
lru := ns.lru
|
||||
lru.Lock()
|
||||
|
||||
switch ns.state {
|
||||
case nsZapped:
|
||||
lru.Unlock()
|
||||
if setf == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var value interface{}
|
||||
var fin func()
|
||||
ok, value, _, fin = setf()
|
||||
if ok {
|
||||
o = &fakeObject{
|
||||
value: value,
|
||||
fin: fin,
|
||||
}
|
||||
}
|
||||
return
|
||||
case nsClosed:
|
||||
lru.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
n, ok := ns.table[key]
|
||||
if ok {
|
||||
switch n.state {
|
||||
case nodeEvicted:
|
||||
// Insert to recent list.
|
||||
n.state = nodeEffective
|
||||
n.ref++
|
||||
lru.size += n.charge
|
||||
lru.evict()
|
||||
fallthrough
|
||||
case nodeEffective:
|
||||
// Bump to front
|
||||
n.rRemove()
|
||||
n.rInsert(&lru.recent)
|
||||
}
|
||||
n.ref++
|
||||
} else {
|
||||
if setf == nil {
|
||||
lru.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
var value interface{}
|
||||
var charge int
|
||||
var fin func()
|
||||
ok, value, charge, fin = setf()
|
||||
if !ok {
|
||||
lru.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
n = &lruNode{
|
||||
ns: ns,
|
||||
key: key,
|
||||
value: value,
|
||||
charge: charge,
|
||||
setfin: fin,
|
||||
ref: 2,
|
||||
}
|
||||
ns.table[key] = n
|
||||
n.rInsert(&lru.recent)
|
||||
|
||||
lru.size += charge
|
||||
lru.evict()
|
||||
}
|
||||
|
||||
lru.Unlock()
|
||||
o = &lruObject{node: n}
|
||||
return
|
||||
}
|
||||
|
||||
func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
|
||||
lru := ns.lru
|
||||
lru.Lock()
|
||||
|
||||
if ns.state != nsEffective {
|
||||
lru.Unlock()
|
||||
if fin != nil {
|
||||
fin(false)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
n, ok := ns.table[key]
|
||||
if !ok {
|
||||
lru.Unlock()
|
||||
if fin != nil {
|
||||
fin(false)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
n.delfin = fin
|
||||
switch n.state {
|
||||
case nodeRemoved:
|
||||
lru.Unlock()
|
||||
return false
|
||||
case nodeEffective:
|
||||
lru.size -= n.charge
|
||||
n.rRemove()
|
||||
n.evictNB()
|
||||
}
|
||||
n.state = nodeRemoved
|
||||
|
||||
lru.Unlock()
|
||||
return true
|
||||
}
|
||||
|
||||
func (ns *lruNs) purgeNB(fin PurgeFin) {
|
||||
lru := ns.lru
|
||||
if ns.state != nsEffective {
|
||||
return
|
||||
}
|
||||
|
||||
for _, n := range ns.table {
|
||||
n.purgefin = fin
|
||||
if n.state == nodeEffective {
|
||||
lru.size -= n.charge
|
||||
n.rRemove()
|
||||
n.evictNB()
|
||||
}
|
||||
n.state = nodeRemoved
|
||||
}
|
||||
}
|
||||
|
||||
func (ns *lruNs) Purge(fin PurgeFin) {
|
||||
ns.lru.Lock()
|
||||
ns.purgeNB(fin)
|
||||
ns.lru.Unlock()
|
||||
}
|
||||
|
||||
func (ns *lruNs) zapNB(closed bool) {
|
||||
lru := ns.lru
|
||||
if ns.state != nsEffective {
|
||||
return
|
||||
}
|
||||
|
||||
if closed {
|
||||
ns.state = nsClosed
|
||||
} else {
|
||||
ns.state = nsZapped
|
||||
}
|
||||
for _, n := range ns.table {
|
||||
if n.state == nodeEffective {
|
||||
lru.size -= n.charge
|
||||
n.rRemove()
|
||||
}
|
||||
n.state = nodeRemoved
|
||||
n.execFin()
|
||||
}
|
||||
ns.table = nil
|
||||
}
|
||||
|
||||
func (ns *lruNs) Zap(closed bool) {
|
||||
ns.lru.Lock()
|
||||
ns.zapNB(closed)
|
||||
delete(ns.lru.table, ns.id)
|
||||
ns.lru.Unlock()
|
||||
}
|
||||
|
||||
type lruNode struct {
|
||||
ns *lruNs
|
||||
|
||||
rNext, rPrev *lruNode
|
||||
|
||||
key uint64
|
||||
value interface{}
|
||||
charge int
|
||||
ref int
|
||||
state nodeState
|
||||
setfin SetFin
|
||||
delfin DelFin
|
||||
purgefin PurgeFin
|
||||
}
|
||||
|
||||
func (n *lruNode) rInsert(at *lruNode) {
|
||||
x := at.rNext
|
||||
at.rNext = n
|
||||
n.rPrev = at
|
||||
n.rNext = x
|
||||
x.rPrev = n
|
||||
}
|
||||
|
||||
func (n *lruNode) rRemove() bool {
|
||||
// only remove if not already removed
|
||||
if n.rPrev == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
n.rPrev.rNext = n.rNext
|
||||
n.rNext.rPrev = n.rPrev
|
||||
n.rPrev = nil
|
||||
n.rNext = nil
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (n *lruNode) execFin() {
|
||||
if n.setfin != nil {
|
||||
n.setfin()
|
||||
n.setfin = nil
|
||||
}
|
||||
if n.purgefin != nil {
|
||||
n.purgefin(n.ns.id, n.key, n.delfin)
|
||||
n.delfin = nil
|
||||
n.purgefin = nil
|
||||
} else if n.delfin != nil {
|
||||
n.delfin(true)
|
||||
n.delfin = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (n *lruNode) evictNB() {
|
||||
n.ref--
|
||||
if n.ref == 0 {
|
||||
if n.ns.state == nsEffective {
|
||||
// remove elem
|
||||
delete(n.ns.table, n.key)
|
||||
// execute finalizer
|
||||
n.execFin()
|
||||
}
|
||||
} else if n.ref < 0 {
|
||||
panic("leveldb/cache: lruCache: negative node reference")
|
||||
}
|
||||
}
|
||||
|
||||
func (n *lruNode) evict() {
|
||||
n.ns.lru.Lock()
|
||||
n.evictNB()
|
||||
n.ns.lru.Unlock()
|
||||
}
|
||||
|
||||
type lruObject struct {
|
||||
node *lruNode
|
||||
once uint32
|
||||
}
|
||||
|
||||
func (o *lruObject) Value() interface{} {
|
||||
if atomic.LoadUint32(&o.once) == 0 {
|
||||
return o.node.value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *lruObject) Release() {
|
||||
if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
|
||||
return
|
||||
}
|
||||
|
||||
o.node.evict()
|
||||
o.node = nil
|
||||
}
|
||||
-75
@@ -1,75 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import "github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
|
||||
type iComparer struct {
|
||||
ucmp comparer.Comparer
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uName() string {
|
||||
return icmp.ucmp.Name()
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uCompare(a, b []byte) int {
|
||||
return icmp.ucmp.Compare(a, b)
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte {
|
||||
return icmp.ucmp.Separator(dst, a, b)
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uSuccessor(dst, b []byte) []byte {
|
||||
return icmp.ucmp.Successor(dst, b)
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Name() string {
|
||||
return icmp.uName()
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Compare(a, b []byte) int {
|
||||
x := icmp.ucmp.Compare(iKey(a).ukey(), iKey(b).ukey())
|
||||
if x == 0 {
|
||||
if m, n := iKey(a).num(), iKey(b).num(); m > n {
|
||||
x = -1
|
||||
} else if m < n {
|
||||
x = 1
|
||||
}
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Separator(dst, a, b []byte) []byte {
|
||||
ua, ub := iKey(a).ukey(), iKey(b).ukey()
|
||||
dst = icmp.ucmp.Separator(dst, ua, ub)
|
||||
if dst == nil {
|
||||
return nil
|
||||
}
|
||||
if len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 {
|
||||
dst = append(dst, kMaxNumBytes...)
|
||||
} else {
|
||||
// Did not close possibilities that n maybe longer than len(ub).
|
||||
dst = append(dst, a[len(a)-8:]...)
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Successor(dst, b []byte) []byte {
|
||||
ub := iKey(b).ukey()
|
||||
dst = icmp.ucmp.Successor(dst, ub)
|
||||
if dst == nil {
|
||||
return nil
|
||||
}
|
||||
if len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 {
|
||||
dst = append(dst, kMaxNumBytes...)
|
||||
} else {
|
||||
// Did not close possibilities that n maybe longer than len(ub).
|
||||
dst = append(dst, b[len(b)-8:]...)
|
||||
}
|
||||
return dst
|
||||
}
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
const (
|
||||
kNumLevels = 7
|
||||
|
||||
// Level-0 compaction is started when we hit this many files.
|
||||
kL0_CompactionTrigger float64 = 4
|
||||
|
||||
// Soft limit on number of level-0 files. We slow down writes at this point.
|
||||
kL0_SlowdownWritesTrigger = 8
|
||||
|
||||
// Maximum number of level-0 files. We stop writes at this point.
|
||||
kL0_StopWritesTrigger = 12
|
||||
|
||||
// Maximum level to which a new compacted memdb is pushed if it
|
||||
// does not create overlap. We try to push to level 2 to avoid the
|
||||
// relatively expensive level 0=>1 compactions and to avoid some
|
||||
// expensive manifest file operations. We do not push all the way to
|
||||
// the largest level since that can generate a lot of wasted disk
|
||||
// space if the same key space is being repeatedly overwritten.
|
||||
kMaxMemCompactLevel = 2
|
||||
|
||||
// Maximum size of a table.
|
||||
kMaxTableSize = 2 * 1048576
|
||||
|
||||
// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
|
||||
// stop building a single file in a level->level+1 compaction.
|
||||
kMaxGrandParentOverlapBytes = 10 * kMaxTableSize
|
||||
|
||||
// Maximum number of bytes in all compacted files. We avoid expanding
|
||||
// the lower level file set of a compaction if it would make the
|
||||
// total compaction cover more than this many bytes.
|
||||
kExpCompactionMaxBytes = 25 * kMaxTableSize
|
||||
)
|
||||
-750
@@ -1,750 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/journal"
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/table"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// DB is a LevelDB database.
|
||||
type DB struct {
|
||||
// Need 64-bit alignment.
|
||||
seq uint64
|
||||
|
||||
s *session
|
||||
|
||||
// MemDB
|
||||
memMu sync.RWMutex
|
||||
mem *memdb.DB
|
||||
frozenMem *memdb.DB
|
||||
journal *journal.Writer
|
||||
journalWriter storage.Writer
|
||||
journalFile storage.File
|
||||
frozenJournalFile storage.File
|
||||
frozenSeq uint64
|
||||
|
||||
// Snapshot
|
||||
snapsMu sync.Mutex
|
||||
snapsRoot snapshotElement
|
||||
|
||||
// Write
|
||||
writeC chan *Batch
|
||||
writeMergedC chan bool
|
||||
writeLockC chan struct{}
|
||||
writeAckC chan error
|
||||
journalC chan *Batch
|
||||
journalAckC chan error
|
||||
|
||||
// Compaction
|
||||
tcompCmdC chan cCmd
|
||||
tcompPauseC chan chan<- struct{}
|
||||
tcompTriggerC chan struct{}
|
||||
mcompCmdC chan cCmd
|
||||
mcompTriggerC chan struct{}
|
||||
compErrC chan error
|
||||
compErrSetC chan error
|
||||
compStats [kNumLevels]cStats
|
||||
|
||||
// Close
|
||||
closeW sync.WaitGroup
|
||||
closeC chan struct{}
|
||||
closed uint32
|
||||
closer io.Closer
|
||||
}
|
||||
|
||||
func openDB(s *session) (*DB, error) {
|
||||
s.log("db@open opening")
|
||||
start := time.Now()
|
||||
db := &DB{
|
||||
s: s,
|
||||
// Initial sequence
|
||||
seq: s.stSeq,
|
||||
// Write
|
||||
writeC: make(chan *Batch),
|
||||
writeMergedC: make(chan bool),
|
||||
writeLockC: make(chan struct{}, 1),
|
||||
writeAckC: make(chan error),
|
||||
journalC: make(chan *Batch),
|
||||
journalAckC: make(chan error),
|
||||
// Compaction
|
||||
tcompCmdC: make(chan cCmd),
|
||||
tcompPauseC: make(chan chan<- struct{}),
|
||||
tcompTriggerC: make(chan struct{}, 1),
|
||||
mcompCmdC: make(chan cCmd),
|
||||
mcompTriggerC: make(chan struct{}, 1),
|
||||
compErrC: make(chan error),
|
||||
compErrSetC: make(chan error),
|
||||
// Close
|
||||
closeC: make(chan struct{}),
|
||||
}
|
||||
db.initSnapshot()
|
||||
|
||||
if err := db.recoverJournal(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Remove any obsolete files.
|
||||
if err := db.checkAndCleanFiles(); err != nil {
|
||||
// Close journal.
|
||||
if db.journal != nil {
|
||||
db.journal.Close()
|
||||
db.journalWriter.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Don't include compaction error goroutine into wait group.
|
||||
go db.compactionError()
|
||||
|
||||
db.closeW.Add(3)
|
||||
go db.tCompaction()
|
||||
go db.mCompaction()
|
||||
go db.jWriter()
|
||||
|
||||
s.logf("db@open done T·%v", time.Since(start))
|
||||
|
||||
runtime.SetFinalizer(db, (*DB).Close)
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Open opens or creates a DB for the given storage.
|
||||
// The DB will be created if not exist, unless ErrorIfMissing is true.
|
||||
// Also, if ErrorIfExist is true and the DB exist Open will returns
|
||||
// os.ErrExist error.
|
||||
//
|
||||
// Open will return an error with type of ErrCorrupted if corruption
|
||||
// detected in the DB. Corrupted DB can be recovered with Recover
|
||||
// function.
|
||||
//
|
||||
// The DB must be closed after use, by calling Close method.
|
||||
func Open(p storage.Storage, o *opt.Options) (db *DB, err error) {
|
||||
s, err := newSession(p, o)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
s.close()
|
||||
s.release()
|
||||
}
|
||||
}()
|
||||
|
||||
err = s.recover()
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) || s.o.GetErrorIfMissing() {
|
||||
return
|
||||
}
|
||||
err = s.create()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else if s.o.GetErrorIfExist() {
|
||||
err = os.ErrExist
|
||||
return
|
||||
}
|
||||
|
||||
return openDB(s)
|
||||
}
|
||||
|
||||
// OpenFile opens or creates a DB for the given path.
|
||||
// The DB will be created if not exist, unless ErrorIfMissing is true.
|
||||
// Also, if ErrorIfExist is true and the DB exist OpenFile will returns
|
||||
// os.ErrExist error.
|
||||
//
|
||||
// OpenFile uses standard file-system backed storage implementation as
|
||||
// desribed in the leveldb/storage package.
|
||||
//
|
||||
// OpenFile will return an error with type of ErrCorrupted if corruption
|
||||
// detected in the DB. Corrupted DB can be recovered with Recover
|
||||
// function.
|
||||
//
|
||||
// The DB must be closed after use, by calling Close method.
|
||||
func OpenFile(path string, o *opt.Options) (db *DB, err error) {
|
||||
stor, err := storage.OpenFile(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
db, err = Open(stor, o)
|
||||
if err != nil {
|
||||
stor.Close()
|
||||
} else {
|
||||
db.closer = stor
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Recover recovers and opens a DB with missing or corrupted manifest files
|
||||
// for the given storage. It will ignore any manifest files, valid or not.
|
||||
// The DB must already exist or it will returns an error.
|
||||
// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
|
||||
//
|
||||
// The DB must be closed after use, by calling Close method.
|
||||
func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) {
|
||||
s, err := newSession(p, o)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
s.close()
|
||||
s.release()
|
||||
}
|
||||
}()
|
||||
|
||||
err = recoverTable(s, o)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return openDB(s)
|
||||
}
|
||||
|
||||
// RecoverFile recovers and opens a DB with missing or corrupted manifest files
|
||||
// for the given path. It will ignore any manifest files, valid or not.
|
||||
// The DB must already exist or it will returns an error.
|
||||
// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
|
||||
//
|
||||
// RecoverFile uses standard file-system backed storage implementation as desribed
|
||||
// in the leveldb/storage package.
|
||||
//
|
||||
// The DB must be closed after use, by calling Close method.
|
||||
func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
|
||||
stor, err := storage.OpenFile(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
db, err = Recover(stor, o)
|
||||
if err != nil {
|
||||
stor.Close()
|
||||
} else {
|
||||
db.closer = stor
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func recoverTable(s *session, o *opt.Options) error {
|
||||
ff0, err := s.getFiles(storage.TypeTable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ff1 := files(ff0)
|
||||
ff1.sort()
|
||||
|
||||
var mSeq uint64
|
||||
var good, corrupted int
|
||||
rec := new(sessionRecord)
|
||||
buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
|
||||
tmp = s.newTemp()
|
||||
writer, err := tmp.Create()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
writer.Close()
|
||||
if err != nil {
|
||||
tmp.Remove()
|
||||
tmp = nil
|
||||
}
|
||||
}()
|
||||
tw := table.NewWriter(writer, o)
|
||||
// Copy records.
|
||||
for iter.Next() {
|
||||
key := iter.Key()
|
||||
if validIkey(key) {
|
||||
err = tw.Append(key, iter.Value())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
err = iter.Error()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = tw.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = writer.Sync()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size = int64(tw.BytesLen())
|
||||
return
|
||||
}
|
||||
recoverTable := func(file storage.File) error {
|
||||
s.logf("table@recovery recovering @%d", file.Num())
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
// Get file size.
|
||||
size, err := reader.Seek(0, 2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tSeq uint64
|
||||
var tgood, tcorrupted, blockerr int
|
||||
var min, max []byte
|
||||
tr := table.NewReader(reader, size, nil, o)
|
||||
iter := tr.NewIterator(nil, nil)
|
||||
iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
|
||||
s.logf("table@recovery found error @%d %q", file.Num(), err)
|
||||
blockerr++
|
||||
})
|
||||
// Scan the table.
|
||||
for iter.Next() {
|
||||
key := iter.Key()
|
||||
_, seq, _, ok := parseIkey(key)
|
||||
if !ok {
|
||||
tcorrupted++
|
||||
continue
|
||||
}
|
||||
tgood++
|
||||
if seq > tSeq {
|
||||
tSeq = seq
|
||||
}
|
||||
if min == nil {
|
||||
min = append([]byte{}, key...)
|
||||
}
|
||||
max = append(max[:0], key...)
|
||||
}
|
||||
if err := iter.Error(); err != nil {
|
||||
iter.Release()
|
||||
return err
|
||||
}
|
||||
iter.Release()
|
||||
if tgood > 0 {
|
||||
if tcorrupted > 0 || blockerr > 0 {
|
||||
// Rebuild the table.
|
||||
s.logf("table@recovery rebuilding @%d", file.Num())
|
||||
iter := tr.NewIterator(nil, nil)
|
||||
tmp, newSize, err := buildTable(iter)
|
||||
iter.Release()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader.Close()
|
||||
if err := file.Replace(tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
size = newSize
|
||||
}
|
||||
if tSeq > mSeq {
|
||||
mSeq = tSeq
|
||||
}
|
||||
// Add table to level 0.
|
||||
rec.addTable(0, file.Num(), uint64(size), min, max)
|
||||
s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq)
|
||||
} else {
|
||||
s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size)
|
||||
}
|
||||
|
||||
good += tgood
|
||||
corrupted += tcorrupted
|
||||
|
||||
return nil
|
||||
}
|
||||
// Recover all tables.
|
||||
if len(ff1) > 0 {
|
||||
s.logf("table@recovery F·%d", len(ff1))
|
||||
s.markFileNum(ff1[len(ff1)-1].Num())
|
||||
for _, file := range ff1 {
|
||||
if err := recoverTable(file); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(ff1), good, corrupted, mSeq)
|
||||
}
|
||||
// Set sequence number.
|
||||
rec.setSeq(mSeq + 1)
|
||||
// Create new manifest.
|
||||
if err := s.create(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Commit.
|
||||
return s.commit(rec)
|
||||
}
|
||||
|
||||
func (d *DB) recoverJournal() error {
|
||||
s := d.s
|
||||
|
||||
ff0, err := s.getFiles(storage.TypeJournal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ff1 := files(ff0)
|
||||
ff1.sort()
|
||||
ff2 := make([]storage.File, 0, len(ff1))
|
||||
for _, file := range ff1 {
|
||||
if file.Num() >= s.stJournalNum || file.Num() == s.stPrevJournalNum {
|
||||
s.markFileNum(file.Num())
|
||||
ff2 = append(ff2, file)
|
||||
}
|
||||
}
|
||||
|
||||
var jr *journal.Reader
|
||||
var of storage.File
|
||||
var mem *memdb.DB
|
||||
batch := new(Batch)
|
||||
cm := newCMem(s)
|
||||
buf := new(util.Buffer)
|
||||
// Options.
|
||||
strict := s.o.GetStrict(opt.StrictJournal)
|
||||
checksum := s.o.GetStrict(opt.StrictJournalChecksum)
|
||||
writeBuffer := s.o.GetWriteBuffer()
|
||||
recoverJournal := func(file storage.File) error {
|
||||
s.logf("journal@recovery recovering @%d", file.Num())
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
if jr == nil {
|
||||
jr = journal.NewReader(reader, dropper{s, file}, strict, checksum)
|
||||
} else {
|
||||
jr.Reset(reader, dropper{s, file}, strict, checksum)
|
||||
}
|
||||
if of != nil {
|
||||
if mem.Len() > 0 {
|
||||
if err := cm.flush(mem, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := cm.commit(file.Num(), d.seq); err != nil {
|
||||
return err
|
||||
}
|
||||
cm.reset()
|
||||
of.Remove()
|
||||
of = nil
|
||||
}
|
||||
// Reset memdb.
|
||||
mem.Reset()
|
||||
for {
|
||||
r, err := jr.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
buf.Reset()
|
||||
if _, err := buf.ReadFrom(r); err != nil {
|
||||
if strict {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := batch.decode(buf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := batch.memReplay(mem); err != nil {
|
||||
return err
|
||||
}
|
||||
d.seq = batch.seq + uint64(batch.len())
|
||||
if mem.Size() >= writeBuffer {
|
||||
// Large enough, flush it.
|
||||
if err := cm.flush(mem, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
// Reset memdb.
|
||||
mem.Reset()
|
||||
}
|
||||
}
|
||||
of = file
|
||||
return nil
|
||||
}
|
||||
// Recover all journals.
|
||||
if len(ff2) > 0 {
|
||||
s.logf("journal@recovery F·%d", len(ff2))
|
||||
mem = memdb.New(s.icmp, writeBuffer)
|
||||
for _, file := range ff2 {
|
||||
if err := recoverJournal(file); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Flush the last journal.
|
||||
if mem.Len() > 0 {
|
||||
if err := cm.flush(mem, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
// Create a new journal.
|
||||
if _, err := d.newMem(0); err != nil {
|
||||
return err
|
||||
}
|
||||
// Commit.
|
||||
if err := cm.commit(d.journalFile.Num(), d.seq); err != nil {
|
||||
return err
|
||||
}
|
||||
// Remove the last journal.
|
||||
if of != nil {
|
||||
of.Remove()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
|
||||
s := d.s
|
||||
|
||||
ikey := newIKey(key, seq, tSeek)
|
||||
|
||||
em, fm := d.getMems()
|
||||
for _, m := range [...]*memdb.DB{em, fm} {
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
mk, mv, me := m.Find(ikey)
|
||||
if me == nil {
|
||||
ukey, _, t, ok := parseIkey(mk)
|
||||
if ok && s.icmp.uCompare(ukey, key) == 0 {
|
||||
if t == tDel {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return mv, nil
|
||||
}
|
||||
} else if me != ErrNotFound {
|
||||
return nil, me
|
||||
}
|
||||
}
|
||||
|
||||
v := s.version()
|
||||
value, cSched, err := v.get(ikey, ro)
|
||||
v.release()
|
||||
if cSched {
|
||||
// Trigger table compaction.
|
||||
d.compTrigger(d.tcompTriggerC)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Get gets the value for the given key. It returns ErrNotFound if the
|
||||
// DB does not contain the key.
|
||||
//
|
||||
// The caller should not modify the contents of the returned slice, but
|
||||
// it is safe to modify the contents of the argument after Get returns.
|
||||
func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
|
||||
err = d.ok()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return d.get(key, d.getSeq(), ro)
|
||||
}
|
||||
|
||||
// NewIterator returns an iterator for the latest snapshot of the
|
||||
// uderlying DB.
|
||||
// The returned iterator is not goroutine-safe, but it is safe to use
|
||||
// multiple iterators concurrently, with each in a dedicated goroutine.
|
||||
// It is also safe to use an iterator concurrently with modifying its
|
||||
// underlying DB. The resultant key/value pairs are guaranteed to be
|
||||
// consistent.
|
||||
//
|
||||
// Slice allows slicing the iterator to only contains keys in the given
|
||||
// range. A nil Range.Start is treated as a key before all keys in the
|
||||
// DB. And a nil Range.Limit is treated as a key after all keys in
|
||||
// the DB.
|
||||
//
|
||||
// The iterator must be released after use, by calling Release method.
|
||||
//
|
||||
// Also read Iterator documentation of the leveldb/iterator package.
|
||||
func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
if err := d.ok(); err != nil {
|
||||
return iterator.NewEmptyIterator(err)
|
||||
}
|
||||
|
||||
p := d.newSnapshot()
|
||||
defer p.Release()
|
||||
return p.NewIterator(slice, ro)
|
||||
}
|
||||
|
||||
// GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
|
||||
// is a frozen snapshot of a DB state at a particular point in time. The
|
||||
// content of snapshot are guaranteed to be consistent.
|
||||
//
|
||||
// The snapshot must be released after use, by calling Release method.
|
||||
func (d *DB) GetSnapshot() (*Snapshot, error) {
|
||||
if err := d.ok(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return d.newSnapshot(), nil
|
||||
}
|
||||
|
||||
// GetProperty returns value of the given property name.
|
||||
//
|
||||
// Property names:
|
||||
// leveldb.num-files-at-level{n}
|
||||
// Returns the number of filer at level 'n'.
|
||||
// leveldb.stats
|
||||
// Returns statistics of the underlying DB.
|
||||
// leveldb.sstables
|
||||
// Returns sstables list for each level.
|
||||
func (d *DB) GetProperty(name string) (value string, err error) {
|
||||
err = d.ok()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
const prefix = "leveldb."
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
return "", errors.New("leveldb: GetProperty: unknown property: " + name)
|
||||
}
|
||||
|
||||
p := name[len(prefix):]
|
||||
|
||||
s := d.s
|
||||
v := s.version()
|
||||
defer v.release()
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(p, "num-files-at-level"):
|
||||
var level uint
|
||||
var rest string
|
||||
n, _ := fmt.Scanf("%d%s", &level, &rest)
|
||||
if n != 1 || level >= kNumLevels {
|
||||
err = errors.New("leveldb: GetProperty: invalid property: " + name)
|
||||
} else {
|
||||
value = fmt.Sprint(v.tLen(int(level)))
|
||||
}
|
||||
case p == "stats":
|
||||
value = "Compactions\n" +
|
||||
" Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" +
|
||||
"-------+------------+---------------+---------------+---------------+---------------\n"
|
||||
for level, tt := range v.tables {
|
||||
duration, read, write := d.compStats[level].get()
|
||||
if len(tt) == 0 && duration == 0 {
|
||||
continue
|
||||
}
|
||||
value += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n",
|
||||
level, len(tt), float64(tt.size())/1048576.0, duration.Seconds(),
|
||||
float64(read)/1048576.0, float64(write)/1048576.0)
|
||||
}
|
||||
case p == "sstables":
|
||||
for level, tt := range v.tables {
|
||||
value += fmt.Sprintf("--- level %d ---\n", level)
|
||||
for _, t := range tt {
|
||||
value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.min, t.max)
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = errors.New("leveldb: GetProperty: unknown property: " + name)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// SizeOf calculates approximate sizes of the given key ranges.
|
||||
// The length of the returned sizes are equal with the length of the given
|
||||
// ranges. The returned sizes measure storage space usage, so if the user
|
||||
// data compresses by a factor of ten, the returned sizes will be one-tenth
|
||||
// the size of the corresponding user data size.
|
||||
// The results may not include the sizes of recently written data.
|
||||
func (d *DB) SizeOf(ranges []util.Range) (Sizes, error) {
|
||||
if err := d.ok(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
v := d.s.version()
|
||||
defer v.release()
|
||||
|
||||
sizes := make(Sizes, 0, len(ranges))
|
||||
for _, r := range ranges {
|
||||
min := newIKey(r.Start, kMaxSeq, tSeek)
|
||||
max := newIKey(r.Limit, kMaxSeq, tSeek)
|
||||
start, err := v.offsetOf(min)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
limit, err := v.offsetOf(max)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var size uint64
|
||||
if limit >= start {
|
||||
size = limit - start
|
||||
}
|
||||
sizes = append(sizes, size)
|
||||
}
|
||||
|
||||
return sizes, nil
|
||||
}
|
||||
|
||||
// Close closes the DB. This will also releases any outstanding snapshot.
|
||||
//
|
||||
// It is not safe to close a DB until all outstanding iterators are released.
|
||||
// It is valid to call Close multiple times. Other methods should not be
|
||||
// called after the DB has been closed.
|
||||
func (d *DB) Close() error {
|
||||
if !d.setClosed() {
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
s := d.s
|
||||
start := time.Now()
|
||||
s.log("db@close closing")
|
||||
|
||||
// Clear the finalizer.
|
||||
runtime.SetFinalizer(d, nil)
|
||||
|
||||
// Get compaction error.
|
||||
var err error
|
||||
select {
|
||||
case err = <-d.compErrC:
|
||||
default:
|
||||
}
|
||||
|
||||
close(d.closeC)
|
||||
|
||||
// Wait for the close WaitGroup.
|
||||
d.closeW.Wait()
|
||||
|
||||
// Close journal.
|
||||
if d.journal != nil {
|
||||
d.journal.Close()
|
||||
d.journalWriter.Close()
|
||||
}
|
||||
|
||||
// Close session.
|
||||
s.close()
|
||||
s.logf("db@close done T·%v", time.Since(start))
|
||||
s.release()
|
||||
|
||||
if d.closer != nil {
|
||||
if err1 := d.closer.Close(); err == nil {
|
||||
err = err1
|
||||
}
|
||||
}
|
||||
|
||||
d.s = nil
|
||||
d.mem = nil
|
||||
d.frozenMem = nil
|
||||
d.journal = nil
|
||||
d.journalWriter = nil
|
||||
d.journalFile = nil
|
||||
d.frozenJournalFile = nil
|
||||
d.snapsRoot = snapshotElement{}
|
||||
d.closer = nil
|
||||
|
||||
return err
|
||||
}
|
||||
-688
@@ -1,688 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
)
|
||||
|
||||
var (
|
||||
errCompactionTransactExiting = errors.New("leveldb: compaction transact exiting")
|
||||
)
|
||||
|
||||
type cStats struct {
|
||||
sync.Mutex
|
||||
duration time.Duration
|
||||
read uint64
|
||||
write uint64
|
||||
}
|
||||
|
||||
func (p *cStats) add(n *cStatsStaging) {
|
||||
p.Lock()
|
||||
p.duration += n.duration
|
||||
p.read += n.read
|
||||
p.write += n.write
|
||||
p.Unlock()
|
||||
}
|
||||
|
||||
func (p *cStats) get() (duration time.Duration, read, write uint64) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
return p.duration, p.read, p.write
|
||||
}
|
||||
|
||||
type cStatsStaging struct {
|
||||
start time.Time
|
||||
duration time.Duration
|
||||
on bool
|
||||
read uint64
|
||||
write uint64
|
||||
}
|
||||
|
||||
func (p *cStatsStaging) startTimer() {
|
||||
if !p.on {
|
||||
p.start = time.Now()
|
||||
p.on = true
|
||||
}
|
||||
}
|
||||
|
||||
func (p *cStatsStaging) stopTimer() {
|
||||
if p.on {
|
||||
p.duration += time.Since(p.start)
|
||||
p.on = false
|
||||
}
|
||||
}
|
||||
|
||||
type cMem struct {
|
||||
s *session
|
||||
level int
|
||||
rec *sessionRecord
|
||||
}
|
||||
|
||||
func newCMem(s *session) *cMem {
|
||||
return &cMem{s: s, rec: new(sessionRecord)}
|
||||
}
|
||||
|
||||
func (c *cMem) flush(mem *memdb.DB, level int) error {
|
||||
s := c.s
|
||||
|
||||
// Write memdb to table
|
||||
iter := mem.NewIterator(nil)
|
||||
defer iter.Release()
|
||||
t, n, err := s.tops.createFrom(iter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if level < 0 {
|
||||
level = s.version_NB().pickLevel(t.min.ukey(), t.max.ukey())
|
||||
}
|
||||
c.rec.addTableFile(level, t)
|
||||
|
||||
s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.min, t.max)
|
||||
|
||||
c.level = level
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *cMem) reset() {
|
||||
c.rec = new(sessionRecord)
|
||||
}
|
||||
|
||||
func (c *cMem) commit(journal, seq uint64) error {
|
||||
c.rec.setJournalNum(journal)
|
||||
c.rec.setSeq(seq)
|
||||
// Commit changes
|
||||
return c.s.commit(c.rec)
|
||||
}
|
||||
|
||||
func (d *DB) compactionError() {
|
||||
var err error
|
||||
noerr:
|
||||
for {
|
||||
select {
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
case err = <-d.compErrSetC:
|
||||
if err != nil {
|
||||
goto haserr
|
||||
}
|
||||
}
|
||||
}
|
||||
haserr:
|
||||
for {
|
||||
select {
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
case err = <-d.compErrSetC:
|
||||
if err == nil {
|
||||
goto noerr
|
||||
}
|
||||
case d.compErrC <- err:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type compactionTransactCounter int
|
||||
|
||||
func (cnt *compactionTransactCounter) incr() {
|
||||
*cnt++
|
||||
}
|
||||
|
||||
func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
|
||||
s := d.s
|
||||
defer func() {
|
||||
if x := recover(); x != nil {
|
||||
if x == errCompactionTransactExiting && rollback != nil {
|
||||
if err := rollback(); err != nil {
|
||||
s.logf("%s rollback error %q", name, err)
|
||||
}
|
||||
}
|
||||
panic(x)
|
||||
}
|
||||
}()
|
||||
const (
|
||||
backoffMin = 1 * time.Second
|
||||
backoffMax = 8 * time.Second
|
||||
backoffMul = 2 * time.Second
|
||||
)
|
||||
backoff := backoffMin
|
||||
backoffT := time.NewTimer(backoff)
|
||||
lastCnt := compactionTransactCounter(0)
|
||||
for n := 0; ; n++ {
|
||||
// Check wether the DB is closed.
|
||||
if d.isClosed() {
|
||||
s.logf("%s exiting", name)
|
||||
d.compactionExitTransact()
|
||||
} else if n > 0 {
|
||||
s.logf("%s retrying N·%d", name, n)
|
||||
}
|
||||
|
||||
// Execute.
|
||||
cnt := compactionTransactCounter(0)
|
||||
err := exec(&cnt)
|
||||
|
||||
// Set compaction error status.
|
||||
select {
|
||||
case d.compErrSetC <- err:
|
||||
case _, _ = <-d.closeC:
|
||||
s.logf("%s exiting", name)
|
||||
d.compactionExitTransact()
|
||||
}
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
s.logf("%s error I·%d %q", name, cnt, err)
|
||||
|
||||
// Reset backoff duration if counter is advancing.
|
||||
if cnt > lastCnt {
|
||||
backoff = backoffMin
|
||||
lastCnt = cnt
|
||||
}
|
||||
|
||||
// Backoff.
|
||||
backoffT.Reset(backoff)
|
||||
if backoff < backoffMax {
|
||||
backoff *= backoffMul
|
||||
if backoff > backoffMax {
|
||||
backoff = backoffMax
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-backoffT.C:
|
||||
case _, _ = <-d.closeC:
|
||||
s.logf("%s exiting", name)
|
||||
d.compactionExitTransact()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) compactionExitTransact() {
|
||||
panic(errCompactionTransactExiting)
|
||||
}
|
||||
|
||||
func (d *DB) memCompaction() {
|
||||
mem := d.getFrozenMem()
|
||||
if mem == nil {
|
||||
return
|
||||
}
|
||||
|
||||
s := d.s
|
||||
c := newCMem(s)
|
||||
stats := new(cStatsStaging)
|
||||
|
||||
s.logf("mem@flush N·%d S·%s", mem.Len(), shortenb(mem.Size()))
|
||||
|
||||
// Don't compact empty memdb.
|
||||
if mem.Len() == 0 {
|
||||
s.logf("mem@flush skipping")
|
||||
// drop frozen mem
|
||||
d.dropFrozenMem()
|
||||
return
|
||||
}
|
||||
|
||||
// Pause table compaction.
|
||||
ch := make(chan struct{})
|
||||
select {
|
||||
case d.tcompPauseC <- (chan<- struct{})(ch):
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
}
|
||||
|
||||
d.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
|
||||
stats.startTimer()
|
||||
defer stats.stopTimer()
|
||||
return c.flush(mem, -1)
|
||||
}, func() error {
|
||||
for _, r := range c.rec.addedTables {
|
||||
s.logf("mem@flush rollback @%d", r.num)
|
||||
f := s.getTableFile(r.num)
|
||||
if err := f.Remove(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
d.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
|
||||
stats.startTimer()
|
||||
defer stats.stopTimer()
|
||||
return c.commit(d.journalFile.Num(), d.frozenSeq)
|
||||
}, nil)
|
||||
|
||||
s.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
|
||||
|
||||
for _, r := range c.rec.addedTables {
|
||||
stats.write += r.size
|
||||
}
|
||||
d.compStats[c.level].add(stats)
|
||||
|
||||
// Drop frozen mem.
|
||||
d.dropFrozenMem()
|
||||
|
||||
// Resume table compaction.
|
||||
select {
|
||||
case <-ch:
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
}
|
||||
|
||||
// Trigger table compaction.
|
||||
d.compTrigger(d.mcompTriggerC)
|
||||
}
|
||||
|
||||
func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
|
||||
s := d.s
|
||||
|
||||
rec := new(sessionRecord)
|
||||
rec.addCompactionPointer(c.level, c.max)
|
||||
|
||||
if !noTrivial && c.trivial() {
|
||||
t := c.tables[0][0]
|
||||
s.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
|
||||
rec.deleteTable(c.level, t.file.Num())
|
||||
rec.addTableFile(c.level+1, t)
|
||||
d.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
|
||||
return s.commit(rec)
|
||||
}, nil)
|
||||
return
|
||||
}
|
||||
|
||||
var stats [2]cStatsStaging
|
||||
for i, tt := range c.tables {
|
||||
for _, t := range tt {
|
||||
stats[i].read += t.size
|
||||
// Insert deleted tables into record
|
||||
rec.deleteTable(c.level+i, t.file.Num())
|
||||
}
|
||||
}
|
||||
sourceSize := int(stats[0].read + stats[1].read)
|
||||
minSeq := d.minSeq()
|
||||
s.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
|
||||
|
||||
var snapUkey []byte
|
||||
var snapHasUkey bool
|
||||
var snapSeq uint64
|
||||
var snapIter int
|
||||
var snapDropCnt int
|
||||
var dropCnt int
|
||||
d.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
|
||||
ukey := append([]byte{}, snapUkey...)
|
||||
hasUkey := snapHasUkey
|
||||
lseq := snapSeq
|
||||
dropCnt = snapDropCnt
|
||||
snapSched := snapIter == 0
|
||||
|
||||
var tw *tWriter
|
||||
finish := func() error {
|
||||
t, err := tw.finish()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rec.addTableFile(c.level+1, t)
|
||||
stats[1].write += t.size
|
||||
s.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.min, t.max)
|
||||
return nil
|
||||
}
|
||||
|
||||
defer func() {
|
||||
stats[1].stopTimer()
|
||||
if tw != nil {
|
||||
tw.drop()
|
||||
tw = nil
|
||||
}
|
||||
}()
|
||||
|
||||
stats[1].startTimer()
|
||||
iter := c.newIterator()
|
||||
defer iter.Release()
|
||||
for i := 0; iter.Next(); i++ {
|
||||
// Incr transact counter.
|
||||
cnt.incr()
|
||||
|
||||
// Skip until last state.
|
||||
if i < snapIter {
|
||||
continue
|
||||
}
|
||||
|
||||
key := iKey(iter.Key())
|
||||
|
||||
if c.shouldStopBefore(key) && tw != nil {
|
||||
err = finish()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
snapSched = true
|
||||
tw = nil
|
||||
}
|
||||
|
||||
// Scheduled for snapshot, snapshot will used to retry compaction
|
||||
// if error occured.
|
||||
if snapSched {
|
||||
snapUkey = append(snapUkey[:0], ukey...)
|
||||
snapHasUkey = hasUkey
|
||||
snapSeq = lseq
|
||||
snapIter = i
|
||||
snapDropCnt = dropCnt
|
||||
snapSched = false
|
||||
}
|
||||
|
||||
if seq, t, ok := key.parseNum(); !ok {
|
||||
// Don't drop error keys
|
||||
ukey = ukey[:0]
|
||||
hasUkey = false
|
||||
lseq = kMaxSeq
|
||||
} else {
|
||||
if !hasUkey || s.icmp.uCompare(key.ukey(), ukey) != 0 {
|
||||
// First occurrence of this user key
|
||||
ukey = append(ukey[:0], key.ukey()...)
|
||||
hasUkey = true
|
||||
lseq = kMaxSeq
|
||||
}
|
||||
|
||||
drop := false
|
||||
if lseq <= minSeq {
|
||||
// Dropped because newer entry for same user key exist
|
||||
drop = true // (A)
|
||||
} else if t == tDel && seq <= minSeq && c.isBaseLevelForKey(ukey) {
|
||||
// For this user key:
|
||||
// (1) there is no data in higher levels
|
||||
// (2) data in lower levels will have larger seq numbers
|
||||
// (3) data in layers that are being compacted here and have
|
||||
// smaller seq numbers will be dropped in the next
|
||||
// few iterations of this loop (by rule (A) above).
|
||||
// Therefore this deletion marker is obsolete and can be dropped.
|
||||
drop = true
|
||||
}
|
||||
|
||||
lseq = seq
|
||||
if drop {
|
||||
dropCnt++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Create new table if not already
|
||||
if tw == nil {
|
||||
// Check for pause event.
|
||||
select {
|
||||
case ch := <-d.tcompPauseC:
|
||||
d.pauseCompaction(ch)
|
||||
case _, _ = <-d.closeC:
|
||||
d.compactionExitTransact()
|
||||
default:
|
||||
}
|
||||
|
||||
// Create new table.
|
||||
tw, err = s.tops.create()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Write key/value into table
|
||||
err = tw.add(key, iter.Value())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Finish table if it is big enough
|
||||
if tw.tw.BytesLen() >= kMaxTableSize {
|
||||
err = finish()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
snapSched = true
|
||||
tw = nil
|
||||
}
|
||||
}
|
||||
|
||||
err = iter.Error()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Finish last table
|
||||
if tw != nil && !tw.empty() {
|
||||
err = finish()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tw = nil
|
||||
}
|
||||
return
|
||||
}, func() error {
|
||||
for _, r := range rec.addedTables {
|
||||
s.logf("table@build rollback @%d", r.num)
|
||||
f := s.getTableFile(r.num)
|
||||
if err := f.Remove(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Commit changes
|
||||
d.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
|
||||
stats[1].startTimer()
|
||||
defer stats[1].stopTimer()
|
||||
return s.commit(rec)
|
||||
}, nil)
|
||||
|
||||
resultSize := int(int(stats[1].write))
|
||||
s.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
|
||||
|
||||
// Save compaction stats
|
||||
for i := range stats {
|
||||
d.compStats[c.level+1].add(&stats[i])
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) tableRangeCompaction(level int, min, max []byte) {
|
||||
s := d.s
|
||||
s.logf("table@compaction range L%d %q:%q", level, min, max)
|
||||
|
||||
if level >= 0 {
|
||||
if c := s.getCompactionRange(level, min, max); c != nil {
|
||||
d.tableCompaction(c, true)
|
||||
}
|
||||
} else {
|
||||
v := s.version_NB()
|
||||
m := 1
|
||||
for i, t := range v.tables[1:] {
|
||||
if t.isOverlaps(min, max, true, s.icmp) {
|
||||
m = i + 1
|
||||
}
|
||||
}
|
||||
for level := 0; level < m; level++ {
|
||||
if c := s.getCompactionRange(level, min, max); c != nil {
|
||||
d.tableCompaction(c, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) tableAutoCompaction() {
|
||||
if c := d.s.pickCompaction(); c != nil {
|
||||
d.tableCompaction(c, false)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) tableNeedCompaction() bool {
|
||||
return d.s.version_NB().needCompaction()
|
||||
}
|
||||
|
||||
func (d *DB) pauseCompaction(ch chan<- struct{}) {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
case _, _ = <-d.closeC:
|
||||
d.compactionExitTransact()
|
||||
}
|
||||
}
|
||||
|
||||
type cCmd interface {
|
||||
ack(err error)
|
||||
}
|
||||
|
||||
type cIdle struct {
|
||||
ackC chan<- error
|
||||
}
|
||||
|
||||
func (r cIdle) ack(err error) {
|
||||
r.ackC <- err
|
||||
}
|
||||
|
||||
type cRange struct {
|
||||
level int
|
||||
min, max []byte
|
||||
ackC chan<- error
|
||||
}
|
||||
|
||||
func (r cRange) ack(err error) {
|
||||
defer func() {
|
||||
recover()
|
||||
}()
|
||||
if r.ackC != nil {
|
||||
r.ackC <- err
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) compSendIdle(compC chan<- cCmd) error {
|
||||
ch := make(chan error)
|
||||
defer close(ch)
|
||||
// Send cmd.
|
||||
select {
|
||||
case compC <- cIdle{ch}:
|
||||
case err := <-d.compErrC:
|
||||
return err
|
||||
case _, _ = <-d.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
// Wait cmd.
|
||||
return <-ch
|
||||
}
|
||||
|
||||
func (d *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
|
||||
ch := make(chan error)
|
||||
defer close(ch)
|
||||
// Send cmd.
|
||||
select {
|
||||
case compC <- cRange{level, min, max, ch}:
|
||||
case err := <-d.compErrC:
|
||||
return err
|
||||
case _, _ = <-d.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
// Wait cmd.
|
||||
select {
|
||||
case err = <-d.compErrC:
|
||||
case err = <-ch:
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *DB) compTrigger(compTriggerC chan struct{}) {
|
||||
select {
|
||||
case compTriggerC <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) mCompaction() {
|
||||
var x cCmd
|
||||
|
||||
defer func() {
|
||||
if x := recover(); x != nil {
|
||||
if x != errCompactionTransactExiting {
|
||||
panic(x)
|
||||
}
|
||||
}
|
||||
if x != nil {
|
||||
x.ack(ErrClosed)
|
||||
}
|
||||
d.closeW.Done()
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
case x = <-d.mcompCmdC:
|
||||
d.memCompaction()
|
||||
x.ack(nil)
|
||||
x = nil
|
||||
case <-d.mcompTriggerC:
|
||||
d.memCompaction()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) tCompaction() {
|
||||
var x cCmd
|
||||
var ackQ []cCmd
|
||||
|
||||
defer func() {
|
||||
if x := recover(); x != nil {
|
||||
if x != errCompactionTransactExiting {
|
||||
panic(x)
|
||||
}
|
||||
}
|
||||
for i := range ackQ {
|
||||
ackQ[i].ack(ErrClosed)
|
||||
ackQ[i] = nil
|
||||
}
|
||||
if x != nil {
|
||||
x.ack(ErrClosed)
|
||||
}
|
||||
d.closeW.Done()
|
||||
}()
|
||||
|
||||
for {
|
||||
if d.tableNeedCompaction() {
|
||||
select {
|
||||
case x = <-d.tcompCmdC:
|
||||
case <-d.tcompTriggerC:
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
case ch := <-d.tcompPauseC:
|
||||
d.pauseCompaction(ch)
|
||||
continue
|
||||
default:
|
||||
}
|
||||
} else {
|
||||
for i := range ackQ {
|
||||
ackQ[i].ack(nil)
|
||||
ackQ[i] = nil
|
||||
}
|
||||
ackQ = ackQ[:0]
|
||||
select {
|
||||
case x = <-d.tcompCmdC:
|
||||
case <-d.tcompTriggerC:
|
||||
case ch := <-d.tcompPauseC:
|
||||
d.pauseCompaction(ch)
|
||||
continue
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
}
|
||||
}
|
||||
if x != nil {
|
||||
switch cmd := x.(type) {
|
||||
case cIdle:
|
||||
ackQ = append(ackQ, x)
|
||||
case cRange:
|
||||
d.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
|
||||
x.ack(nil)
|
||||
}
|
||||
x = nil
|
||||
}
|
||||
d.tableAutoCompaction()
|
||||
}
|
||||
}
|
||||
-310
@@ -1,310 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"runtime"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
|
||||
)
|
||||
|
||||
func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
s := db.s
|
||||
|
||||
em, fm := db.getMems()
|
||||
v := s.version()
|
||||
|
||||
ti := v.getIterators(slice, ro)
|
||||
n := len(ti) + 2
|
||||
i := make([]iterator.Iterator, 0, n)
|
||||
i = append(i, em.NewIterator(slice))
|
||||
if fm != nil {
|
||||
i = append(i, fm.NewIterator(slice))
|
||||
}
|
||||
i = append(i, ti...)
|
||||
strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
|
||||
mi := iterator.NewMergedIterator(i, s.icmp, strict)
|
||||
mi.SetReleaser(&versionReleaser{v: v})
|
||||
return mi
|
||||
}
|
||||
|
||||
func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
|
||||
var slice_ *util.Range
|
||||
if slice != nil {
|
||||
slice_ = &util.Range{}
|
||||
if slice.Start != nil {
|
||||
slice_.Start = newIKey(slice.Start, kMaxSeq, tSeek)
|
||||
}
|
||||
if slice.Limit != nil {
|
||||
slice_.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
|
||||
}
|
||||
}
|
||||
rawIter := db.newRawIterator(slice_, ro)
|
||||
iter := &dbIter{
|
||||
icmp: db.s.icmp,
|
||||
iter: rawIter,
|
||||
seq: seq,
|
||||
strict: db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator),
|
||||
key: make([]byte, 0),
|
||||
value: make([]byte, 0),
|
||||
}
|
||||
runtime.SetFinalizer(iter, (*dbIter).Release)
|
||||
return iter
|
||||
}
|
||||
|
||||
type dir int
|
||||
|
||||
const (
|
||||
dirReleased dir = iota - 1
|
||||
dirSOI
|
||||
dirEOI
|
||||
dirBackward
|
||||
dirForward
|
||||
)
|
||||
|
||||
// dbIter represent an interator states over a database session.
|
||||
type dbIter struct {
|
||||
icmp *iComparer
|
||||
iter iterator.Iterator
|
||||
seq uint64
|
||||
strict bool
|
||||
|
||||
dir dir
|
||||
key []byte
|
||||
value []byte
|
||||
err error
|
||||
releaser util.Releaser
|
||||
}
|
||||
|
||||
func (i *dbIter) setErr(err error) {
|
||||
i.err = err
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
}
|
||||
|
||||
func (i *dbIter) iterErr() {
|
||||
if err := i.iter.Error(); err != nil {
|
||||
i.setErr(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *dbIter) Valid() bool {
|
||||
return i.err == nil && i.dir > dirEOI
|
||||
}
|
||||
|
||||
func (i *dbIter) First() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.iter.First() {
|
||||
i.dir = dirSOI
|
||||
return i.next()
|
||||
}
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) Last() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.iter.Last() {
|
||||
return i.prev()
|
||||
}
|
||||
i.dir = dirSOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) Seek(key []byte) bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
ikey := newIKey(key, i.seq, tSeek)
|
||||
if i.iter.Seek(ikey) {
|
||||
i.dir = dirSOI
|
||||
return i.next()
|
||||
}
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) next() bool {
|
||||
for {
|
||||
ukey, seq, t, ok := parseIkey(i.iter.Key())
|
||||
if ok {
|
||||
if seq <= i.seq {
|
||||
switch t {
|
||||
case tDel:
|
||||
// Skip deleted key.
|
||||
i.key = append(i.key[:0], ukey...)
|
||||
i.dir = dirForward
|
||||
case tVal:
|
||||
if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
|
||||
i.key = append(i.key[:0], ukey...)
|
||||
i.value = append(i.value[:0], i.iter.Value()...)
|
||||
i.dir = dirForward
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if i.strict {
|
||||
i.setErr(errInvalidIkey)
|
||||
break
|
||||
}
|
||||
if !i.iter.Next() {
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
break
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) Next() bool {
|
||||
if i.dir == dirEOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.iter.Next() || (i.dir == dirBackward && !i.iter.Next()) {
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
return i.next()
|
||||
}
|
||||
|
||||
func (i *dbIter) prev() bool {
|
||||
i.dir = dirBackward
|
||||
del := true
|
||||
if i.iter.Valid() {
|
||||
for {
|
||||
ukey, seq, t, ok := parseIkey(i.iter.Key())
|
||||
if ok {
|
||||
if seq <= i.seq {
|
||||
if !del && i.icmp.uCompare(ukey, i.key) < 0 {
|
||||
return true
|
||||
}
|
||||
del = (t == tDel)
|
||||
if !del {
|
||||
i.key = append(i.key[:0], ukey...)
|
||||
i.value = append(i.value[:0], i.iter.Value()...)
|
||||
}
|
||||
}
|
||||
} else if i.strict {
|
||||
i.setErr(errInvalidIkey)
|
||||
return false
|
||||
}
|
||||
if !i.iter.Prev() {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if del {
|
||||
i.dir = dirSOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *dbIter) Prev() bool {
|
||||
if i.dir == dirSOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
switch i.dir {
|
||||
case dirEOI:
|
||||
return i.Last()
|
||||
case dirForward:
|
||||
for i.iter.Prev() {
|
||||
ukey, _, _, ok := parseIkey(i.iter.Key())
|
||||
if ok {
|
||||
if i.icmp.uCompare(ukey, i.key) < 0 {
|
||||
goto cont
|
||||
}
|
||||
} else if i.strict {
|
||||
i.setErr(errInvalidIkey)
|
||||
return false
|
||||
}
|
||||
}
|
||||
i.dir = dirSOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
cont:
|
||||
return i.prev()
|
||||
}
|
||||
|
||||
func (i *dbIter) Key() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.key
|
||||
}
|
||||
|
||||
func (i *dbIter) Value() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.value
|
||||
}
|
||||
|
||||
func (i *dbIter) Release() {
|
||||
if i.dir != dirReleased {
|
||||
// Clear the finalizer.
|
||||
runtime.SetFinalizer(i, nil)
|
||||
|
||||
if i.releaser != nil {
|
||||
i.releaser.Release()
|
||||
}
|
||||
|
||||
i.dir = dirReleased
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
i.iter.Release()
|
||||
i.iter = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i *dbIter) SetReleaser(releaser util.Releaser) {
|
||||
if i.dir != dirReleased {
|
||||
i.releaser = releaser
|
||||
}
|
||||
}
|
||||
|
||||
func (i *dbIter) Error() error {
|
||||
return i.err
|
||||
}
|
||||
-165
@@ -1,165 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type snapshotElement struct {
|
||||
seq uint64
|
||||
ref int
|
||||
// Next and previous pointers in the doubly-linked list of elements.
|
||||
next, prev *snapshotElement
|
||||
}
|
||||
|
||||
// Initialize the snapshot.
|
||||
func (db *DB) initSnapshot() {
|
||||
db.snapsRoot.next = &db.snapsRoot
|
||||
db.snapsRoot.prev = &db.snapsRoot
|
||||
}
|
||||
|
||||
// Acquires a snapshot, based on latest sequence.
|
||||
func (db *DB) acquireSnapshot() *snapshotElement {
|
||||
db.snapsMu.Lock()
|
||||
seq := db.getSeq()
|
||||
elem := db.snapsRoot.prev
|
||||
if elem == &db.snapsRoot || elem.seq != seq {
|
||||
at := db.snapsRoot.prev
|
||||
next := at.next
|
||||
elem = &snapshotElement{
|
||||
seq: seq,
|
||||
prev: at,
|
||||
next: next,
|
||||
}
|
||||
at.next = elem
|
||||
next.prev = elem
|
||||
}
|
||||
elem.ref++
|
||||
db.snapsMu.Unlock()
|
||||
return elem
|
||||
}
|
||||
|
||||
// Releases given snapshot element.
|
||||
func (db *DB) releaseSnapshot(elem *snapshotElement) {
|
||||
if !db.isClosed() {
|
||||
db.snapsMu.Lock()
|
||||
elem.ref--
|
||||
if elem.ref == 0 {
|
||||
elem.prev.next = elem.next
|
||||
elem.next.prev = elem.prev
|
||||
elem.next = nil
|
||||
elem.prev = nil
|
||||
} else if elem.ref < 0 {
|
||||
panic("leveldb: Snapshot: negative element reference")
|
||||
}
|
||||
db.snapsMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Gets minimum sequence that not being snapshoted.
|
||||
func (db *DB) minSeq() uint64 {
|
||||
db.snapsMu.Lock()
|
||||
defer db.snapsMu.Unlock()
|
||||
elem := db.snapsRoot.prev
|
||||
if elem != &db.snapsRoot {
|
||||
return elem.seq
|
||||
}
|
||||
return db.getSeq()
|
||||
}
|
||||
|
||||
// Snapshot is a DB snapshot.
|
||||
type Snapshot struct {
|
||||
db *DB
|
||||
elem *snapshotElement
|
||||
mu sync.Mutex
|
||||
released bool
|
||||
}
|
||||
|
||||
// Creates new snapshot object.
|
||||
func (db *DB) newSnapshot() *Snapshot {
|
||||
p := &Snapshot{
|
||||
db: db,
|
||||
elem: db.acquireSnapshot(),
|
||||
}
|
||||
runtime.SetFinalizer(p, (*Snapshot).Release)
|
||||
return p
|
||||
}
|
||||
|
||||
// Get gets the value for the given key. It returns ErrNotFound if
|
||||
// the DB does not contain the key.
|
||||
//
|
||||
// The caller should not modify the contents of the returned slice, but
|
||||
// it is safe to modify the contents of the argument after Get returns.
|
||||
func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
|
||||
db := p.db
|
||||
err = db.ok()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.released {
|
||||
err = ErrSnapshotReleased
|
||||
return
|
||||
}
|
||||
return db.get(key, p.elem.seq, ro)
|
||||
}
|
||||
|
||||
// NewIterator returns an iterator for the snapshot of the uderlying DB.
|
||||
// The returned iterator is not goroutine-safe, but it is safe to use
|
||||
// multiple iterators concurrently, with each in a dedicated goroutine.
|
||||
// It is also safe to use an iterator concurrently with modifying its
|
||||
// underlying DB. The resultant key/value pairs are guaranteed to be
|
||||
// consistent.
|
||||
//
|
||||
// Slice allows slicing the iterator to only contains keys in the given
|
||||
// range. A nil Range.Start is treated as a key before all keys in the
|
||||
// DB. And a nil Range.Limit is treated as a key after all keys in
|
||||
// the DB.
|
||||
//
|
||||
// The iterator must be released after use, by calling Release method.
|
||||
// Releasing the snapshot doesn't mean releasing the iterator too, the
|
||||
// iterator would be still valid until released.
|
||||
//
|
||||
// Also read Iterator documentation of the leveldb/iterator package.
|
||||
func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
db := p.db
|
||||
if err := db.ok(); err != nil {
|
||||
return iterator.NewEmptyIterator(err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.released {
|
||||
return iterator.NewEmptyIterator(ErrSnapshotReleased)
|
||||
}
|
||||
return db.newIterator(p.elem.seq, slice, ro)
|
||||
}
|
||||
|
||||
// Release releases the snapshot. This will not release any returned
|
||||
// iterators, the iterators would still be valid until released or the
|
||||
// underlying DB is closed.
|
||||
//
|
||||
// Other methods should not be called after the snapshot has been released.
|
||||
func (p *Snapshot) Release() {
|
||||
p.mu.Lock()
|
||||
if !p.released {
|
||||
// Clear the finalizer.
|
||||
runtime.SetFinalizer(p, nil)
|
||||
|
||||
p.released = true
|
||||
p.db.releaseSnapshot(p.elem)
|
||||
p.db = nil
|
||||
p.elem = nil
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
-114
@@ -1,114 +0,0 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/journal"
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
)
|
||||
|
||||
// Get latest sequence number.
|
||||
func (d *DB) getSeq() uint64 {
|
||||
return atomic.LoadUint64(&d.seq)
|
||||
}
|
||||
|
||||
// Atomically adds delta to seq.
|
||||
func (d *DB) addSeq(delta uint64) {
|
||||
atomic.AddUint64(&d.seq, delta)
|
||||
}
|
||||
|
||||
// Create new memdb and froze the old one; need external synchronization.
|
||||
// newMem only called synchronously by the writer.
|
||||
func (d *DB) newMem(n int) (mem *memdb.DB, err error) {
|
||||
s := d.s
|
||||
|
||||
num := s.allocFileNum()
|
||||
file := s.getJournalFile(num)
|
||||
w, err := file.Create()
|
||||
if err != nil {
|
||||
s.reuseFileNum(num)
|
||||
return
|
||||
}
|
||||
d.memMu.Lock()
|
||||
if d.journal == nil {
|
||||
d.journal = journal.NewWriter(w)
|
||||
} else {
|
||||
d.journal.Reset(w)
|
||||
d.journalWriter.Close()
|
||||
d.frozenJournalFile = d.journalFile
|
||||
}
|
||||
d.journalWriter = w
|
||||
d.journalFile = file
|
||||
d.frozenMem = d.mem
|
||||
d.mem = memdb.New(s.icmp, maxInt(d.s.o.GetWriteBuffer(), n))
|
||||
mem = d.mem
|
||||
// The seq only incremented by the writer.
|
||||
d.frozenSeq = d.seq
|
||||
d.memMu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Get all memdbs.
|
||||
func (d *DB) getMems() (e *memdb.DB, f *memdb.DB) {
|
||||
d.memMu.RLock()
|
||||
defer d.memMu.RUnlock()
|
||||
return d.mem, d.frozenMem
|
||||
}
|
||||
|
||||
// Get frozen memdb.
|
||||
func (d *DB) getEffectiveMem() *memdb.DB {
|
||||
d.memMu.RLock()
|
||||
defer d.memMu.RUnlock()
|
||||
return d.mem
|
||||
}
|
||||
|
||||
// Check whether we has frozen memdb.
|
||||
func (d *DB) hasFrozenMem() bool {
|
||||
d.memMu.RLock()
|
||||
defer d.memMu.RUnlock()
|
||||
return d.frozenMem != nil
|
||||
}
|
||||
|
||||
// Get frozen memdb.
|
||||
func (d *DB) getFrozenMem() *memdb.DB {
|
||||
d.memMu.RLock()
|
||||
defer d.memMu.RUnlock()
|
||||
return d.frozenMem
|
||||
}
|
||||
|
||||
// Drop frozen memdb; assume that frozen memdb isn't nil.
|
||||
func (d *DB) dropFrozenMem() {
|
||||
d.memMu.Lock()
|
||||
if err := d.frozenJournalFile.Remove(); err != nil {
|
||||
d.s.logf("journal@remove removing @%d %q", d.frozenJournalFile.Num(), err)
|
||||
} else {
|
||||
d.s.logf("journal@remove removed @%d", d.frozenJournalFile.Num())
|
||||
}
|
||||
d.frozenJournalFile = nil
|
||||
d.frozenMem = nil
|
||||
d.memMu.Unlock()
|
||||
}
|
||||
|
||||
// Set closed flag; return true if not already closed.
|
||||
func (d *DB) setClosed() bool {
|
||||
return atomic.CompareAndSwapUint32(&d.closed, 0, 1)
|
||||
}
|
||||
|
||||
// Check whether DB was closed.
|
||||
func (d *DB) isClosed() bool {
|
||||
return atomic.LoadUint32(&d.closed) != 0
|
||||
}
|
||||
|
||||
// Check read ok status.
|
||||
func (d *DB) ok() error {
|
||||
if d.isClosed() {
|
||||
return ErrClosed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-1888
File diff suppressed because it is too large
Load Diff
-95
@@ -1,95 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// Reader is the interface that wraps basic Get and NewIterator methods.
|
||||
// This interface implemented by both DB and Snapshot.
|
||||
type Reader interface {
|
||||
Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
|
||||
NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
|
||||
}
|
||||
|
||||
type Sizes []uint64
|
||||
|
||||
// Sum returns sum of the sizes.
|
||||
func (p Sizes) Sum() (n uint64) {
|
||||
for _, s := range p {
|
||||
n += s
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Check and clean files.
|
||||
func (d *DB) checkAndCleanFiles() error {
|
||||
s := d.s
|
||||
|
||||
v := s.version_NB()
|
||||
tables := make(map[uint64]bool)
|
||||
for _, tt := range v.tables {
|
||||
for _, t := range tt {
|
||||
tables[t.file.Num()] = false
|
||||
}
|
||||
}
|
||||
|
||||
ff, err := s.getFiles(storage.TypeAll)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var nTables int
|
||||
var rem []storage.File
|
||||
for _, f := range ff {
|
||||
keep := true
|
||||
switch f.Type() {
|
||||
case storage.TypeManifest:
|
||||
keep = f.Num() >= s.manifestFile.Num()
|
||||
case storage.TypeJournal:
|
||||
if d.frozenJournalFile != nil {
|
||||
keep = f.Num() >= d.frozenJournalFile.Num()
|
||||
} else {
|
||||
keep = f.Num() >= d.journalFile.Num()
|
||||
}
|
||||
case storage.TypeTable:
|
||||
_, keep = tables[f.Num()]
|
||||
if keep {
|
||||
tables[f.Num()] = true
|
||||
nTables++
|
||||
}
|
||||
}
|
||||
|
||||
if !keep {
|
||||
rem = append(rem, f)
|
||||
}
|
||||
}
|
||||
|
||||
if nTables != len(tables) {
|
||||
for num, present := range tables {
|
||||
if !present {
|
||||
s.logf("db@janitor table missing @%d", num)
|
||||
}
|
||||
}
|
||||
return ErrCorrupted{Type: MissingFiles, Err: errors.New("leveldb: table files missing")}
|
||||
}
|
||||
|
||||
s.logf("db@janitor F·%d G·%d", len(ff), len(rem))
|
||||
for _, f := range rem {
|
||||
s.logf("db@janitor removing %s-%d", f.Type(), f.Num())
|
||||
if err := f.Remove(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-279
@@ -1,279 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
func (d *DB) writeJournal(b *Batch) error {
|
||||
w, err := d.journal.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(b.encode()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.journal.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.sync {
|
||||
return d.journalWriter.Sync()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DB) jWriter() {
|
||||
defer d.closeW.Done()
|
||||
for {
|
||||
select {
|
||||
case b := <-d.journalC:
|
||||
if b != nil {
|
||||
d.journalAckC <- d.writeJournal(b)
|
||||
}
|
||||
case _, _ = <-d.closeC:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DB) rotateMem(n int) (mem *memdb.DB, err error) {
|
||||
// Wait for pending memdb compaction.
|
||||
err = d.compSendIdle(d.mcompCmdC)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Create new memdb and journal.
|
||||
mem, err = d.newMem(n)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Schedule memdb compaction.
|
||||
d.compTrigger(d.mcompTriggerC)
|
||||
return
|
||||
}
|
||||
|
||||
func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
|
||||
s := d.s
|
||||
|
||||
delayed := false
|
||||
flush := func() bool {
|
||||
v := s.version()
|
||||
defer v.release()
|
||||
mem = d.getEffectiveMem()
|
||||
nn = mem.Free()
|
||||
switch {
|
||||
case v.tLen(0) >= kL0_SlowdownWritesTrigger && !delayed:
|
||||
delayed = true
|
||||
time.Sleep(time.Millisecond)
|
||||
case nn >= n:
|
||||
return false
|
||||
case v.tLen(0) >= kL0_StopWritesTrigger:
|
||||
delayed = true
|
||||
err = d.compSendIdle(d.tcompCmdC)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
// Allow memdb to grow if it has no entry.
|
||||
if mem.Len() == 0 {
|
||||
nn = n
|
||||
return false
|
||||
}
|
||||
mem, err = d.rotateMem(n)
|
||||
nn = mem.Free()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
start := time.Now()
|
||||
for flush() {
|
||||
}
|
||||
if delayed {
|
||||
s.logf("db@write delayed T·%v", time.Since(start))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Write apply the given batch to the DB. The batch will be applied
|
||||
// sequentially.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Write returns.
|
||||
func (d *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
|
||||
err = d.ok()
|
||||
if err != nil || b == nil || b.len() == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
b.init(wo.GetSync())
|
||||
|
||||
// The write happen synchronously.
|
||||
retry:
|
||||
select {
|
||||
case d.writeC <- b:
|
||||
if <-d.writeMergedC {
|
||||
return <-d.writeAckC
|
||||
}
|
||||
goto retry
|
||||
case d.writeLockC <- struct{}{}:
|
||||
case _, _ = <-d.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
merged := 0
|
||||
defer func() {
|
||||
<-d.writeLockC
|
||||
for i := 0; i < merged; i++ {
|
||||
d.writeAckC <- err
|
||||
}
|
||||
}()
|
||||
|
||||
mem, memFree, err := d.flush(b.size())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate maximum size of the batch.
|
||||
m := 1 << 20
|
||||
if x := b.size(); x <= 128<<10 {
|
||||
m = x + (128 << 10)
|
||||
}
|
||||
m = minInt(m, memFree)
|
||||
|
||||
// Merge with other batch.
|
||||
drain:
|
||||
for b.size() < m && !b.sync {
|
||||
select {
|
||||
case nb := <-d.writeC:
|
||||
if b.size()+nb.size() <= m {
|
||||
b.append(nb)
|
||||
d.writeMergedC <- true
|
||||
merged++
|
||||
} else {
|
||||
d.writeMergedC <- false
|
||||
break drain
|
||||
}
|
||||
default:
|
||||
break drain
|
||||
}
|
||||
}
|
||||
|
||||
// Set batch first seq number relative from last seq.
|
||||
b.seq = d.seq + 1
|
||||
|
||||
// Write journal concurrently if it is large enough.
|
||||
if b.size() >= (128 << 10) {
|
||||
// Push the write batch to the journal writer
|
||||
select {
|
||||
case _, _ = <-d.closeC:
|
||||
err = ErrClosed
|
||||
return
|
||||
case d.journalC <- b:
|
||||
// Write into memdb
|
||||
b.memReplay(mem)
|
||||
}
|
||||
// Wait for journal writer
|
||||
select {
|
||||
case _, _ = <-d.closeC:
|
||||
err = ErrClosed
|
||||
return
|
||||
case err = <-d.journalAckC:
|
||||
if err != nil {
|
||||
// Revert memdb if error detected
|
||||
b.revertMemReplay(mem)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err = d.writeJournal(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b.memReplay(mem)
|
||||
}
|
||||
|
||||
// Set last seq number.
|
||||
d.addSeq(uint64(b.len()))
|
||||
|
||||
if b.size() >= memFree {
|
||||
d.rotateMem(0)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Put sets the value for the given key. It overwrites any previous value
|
||||
// for that key; a DB is not a multi-map.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Put returns.
|
||||
func (d *DB) Put(key, value []byte, wo *opt.WriteOptions) error {
|
||||
b := new(Batch)
|
||||
b.Put(key, value)
|
||||
return d.Write(b, wo)
|
||||
}
|
||||
|
||||
// Delete deletes the value for the given key. It returns ErrNotFound if
|
||||
// the DB does not contain the key.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Delete returns.
|
||||
func (d *DB) Delete(key []byte, wo *opt.WriteOptions) error {
|
||||
b := new(Batch)
|
||||
b.Delete(key)
|
||||
return d.Write(b, wo)
|
||||
}
|
||||
|
||||
func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool {
|
||||
iter := mem.NewIterator(nil)
|
||||
defer iter.Release()
|
||||
return (max == nil || (iter.First() && icmp.uCompare(max, iKey(iter.Key()).ukey()) >= 0)) &&
|
||||
(min == nil || (iter.Last() && icmp.uCompare(min, iKey(iter.Key()).ukey()) <= 0))
|
||||
}
|
||||
|
||||
// CompactRange compacts the underlying DB for the given key range.
|
||||
// In particular, deleted and overwritten versions are discarded,
|
||||
// and the data is rearranged to reduce the cost of operations
|
||||
// needed to access the data. This operation should typically only
|
||||
// be invoked by users who understand the underlying implementation.
|
||||
//
|
||||
// A nil Range.Start is treated as a key before all keys in the DB.
|
||||
// And a nil Range.Limit is treated as a key after all keys in the DB.
|
||||
// Therefore if both is nil then it will compact entire DB.
|
||||
func (d *DB) CompactRange(r util.Range) error {
|
||||
if err := d.ok(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case d.writeLockC <- struct{}{}:
|
||||
case _, _ = <-d.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
// Check for overlaps in memdb.
|
||||
mem := d.getEffectiveMem()
|
||||
if isMemOverlaps(d.s.icmp, mem, r.Start, r.Limit) {
|
||||
// Memdb compaction.
|
||||
if _, err := d.rotateMem(0); err != nil {
|
||||
<-d.writeLockC
|
||||
return err
|
||||
}
|
||||
<-d.writeLockC
|
||||
if err := d.compSendIdle(d.mcompCmdC); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
<-d.writeLockC
|
||||
}
|
||||
|
||||
// Table compaction.
|
||||
return d.compSendRange(d.tcompCmdC, -1, r.Start, r.Limit)
|
||||
}
|
||||
-80
@@ -1,80 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package leveldb provides implementation of LevelDB key/value database.
|
||||
//
|
||||
// Create or open a database:
|
||||
//
|
||||
// db, err := leveldb.OpenFile("path/to/db", nil)
|
||||
// ...
|
||||
// defer db.Close()
|
||||
// ...
|
||||
//
|
||||
// Read or modify the database content:
|
||||
//
|
||||
// // Remember that the contents of the returned slice should not be modified.
|
||||
// data, err := db.Get([]byte("key"), nil)
|
||||
// ...
|
||||
// err = db.Put([]byte("key"), []byte("value"), nil)
|
||||
// ...
|
||||
// err = db.Delete([]byte("key"), nil)
|
||||
// ...
|
||||
//
|
||||
// Iterate over database content:
|
||||
//
|
||||
// iter := db.NewIterator(nil, nil)
|
||||
// for iter.Next() {
|
||||
// // Remember that the contents of the returned slice should not be modified, and
|
||||
// // only valid until the next call to Next.
|
||||
// key := iter.Key()
|
||||
// value := iter.Value()
|
||||
// ...
|
||||
// }
|
||||
// iter.Release()
|
||||
// err = iter.Error()
|
||||
// ...
|
||||
//
|
||||
// Seek-then-Iterate:
|
||||
//
|
||||
// iter := db.NewIterator(nil, nil)
|
||||
// for ok := iter.Seek(key); ok; ok = iter.Next() {
|
||||
// // Use key/value.
|
||||
// ...
|
||||
// }
|
||||
// iter.Release()
|
||||
// err = iter.Error()
|
||||
// ...
|
||||
//
|
||||
// Iterate over subset of database content:
|
||||
//
|
||||
// iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil)
|
||||
// for iter.Next() {
|
||||
// // Use key/value.
|
||||
// ...
|
||||
// }
|
||||
// iter.Release()
|
||||
// err = iter.Error()
|
||||
// ...
|
||||
//
|
||||
// Batch writes:
|
||||
//
|
||||
// batch := new(leveldb.Batch)
|
||||
// batch.Put([]byte("foo"), []byte("value"))
|
||||
// batch.Put([]byte("bar"), []byte("another value"))
|
||||
// batch.Delete([]byte("baz"))
|
||||
// err = db.Write(batch, nil)
|
||||
// ...
|
||||
//
|
||||
// Use bloom filter:
|
||||
//
|
||||
// o := &opt.Options{
|
||||
// Filter: filter.NewBloomFilter(10),
|
||||
// }
|
||||
// db, err := leveldb.OpenFile("path/to/db", o)
|
||||
// ...
|
||||
// defer db.Close()
|
||||
// ...
|
||||
package leveldb
|
||||
-38
@@ -1,38 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = util.ErrNotFound
|
||||
ErrSnapshotReleased = errors.New("leveldb: snapshot released")
|
||||
ErrIterReleased = errors.New("leveldb: iterator released")
|
||||
ErrClosed = errors.New("leveldb: closed")
|
||||
)
|
||||
|
||||
type CorruptionType int
|
||||
|
||||
const (
|
||||
CorruptedManifest CorruptionType = iota
|
||||
MissingFiles
|
||||
)
|
||||
|
||||
// ErrCorrupted is the type that wraps errors that indicate corruption in
|
||||
// the database.
|
||||
type ErrCorrupted struct {
|
||||
Type CorruptionType
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e ErrCorrupted) Error() string {
|
||||
return e.Err.Error()
|
||||
}
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/testutil"
|
||||
)
|
||||
|
||||
var _ = testutil.Defer(func() {
|
||||
Describe("Leveldb external", func() {
|
||||
o := &opt.Options{
|
||||
BlockCache: opt.NoCache,
|
||||
BlockRestartInterval: 5,
|
||||
BlockSize: 50,
|
||||
Compression: opt.NoCompression,
|
||||
MaxOpenFiles: 0,
|
||||
Strict: opt.StrictAll,
|
||||
WriteBuffer: 1000,
|
||||
}
|
||||
|
||||
Describe("write test", func() {
|
||||
It("should do write correctly", func(done Done) {
|
||||
db := newTestingDB(o, nil, nil)
|
||||
t := testutil.DBTesting{
|
||||
DB: db,
|
||||
Deleted: testutil.KeyValue_Generate(nil, 500, 1, 50, 5, 5).Clone(),
|
||||
}
|
||||
testutil.DoDBTesting(&t)
|
||||
db.TestClose()
|
||||
done <- true
|
||||
}, 9.0)
|
||||
})
|
||||
|
||||
Describe("read test", func() {
|
||||
testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
|
||||
// Building the DB.
|
||||
db := newTestingDB(o, nil, nil)
|
||||
kv.IterateShuffled(nil, func(i int, key, value []byte) {
|
||||
err := db.TestPut(key, value)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
testutil.Defer("teardown", func() {
|
||||
db.TestClose()
|
||||
})
|
||||
|
||||
return db
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
Generated
Vendored
-221
@@ -1,221 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// IteratorIndexer is the interface that wraps CommonIterator and basic Get
|
||||
// method. IteratorIndexer provides index for indexed iterator.
|
||||
type IteratorIndexer interface {
|
||||
CommonIterator
|
||||
|
||||
// Get returns a new data iterator for the current position, or nil if
|
||||
// done.
|
||||
Get() Iterator
|
||||
}
|
||||
|
||||
type indexedIterator struct {
|
||||
util.BasicReleaser
|
||||
index IteratorIndexer
|
||||
strict bool
|
||||
strictGet bool
|
||||
|
||||
data Iterator
|
||||
err error
|
||||
errf func(err error)
|
||||
}
|
||||
|
||||
func (i *indexedIterator) setData() {
|
||||
if i.data != nil {
|
||||
i.data.Release()
|
||||
}
|
||||
i.data = i.index.Get()
|
||||
if i.strictGet {
|
||||
if err := i.data.Error(); err != nil {
|
||||
i.err = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *indexedIterator) clearData() {
|
||||
if i.data != nil {
|
||||
i.data.Release()
|
||||
}
|
||||
i.data = nil
|
||||
}
|
||||
|
||||
func (i *indexedIterator) dataErr() bool {
|
||||
if i.errf != nil {
|
||||
if err := i.data.Error(); err != nil {
|
||||
i.errf(err)
|
||||
}
|
||||
}
|
||||
if i.strict {
|
||||
if err := i.data.Error(); err != nil {
|
||||
i.err = err
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Valid() bool {
|
||||
return i.data != nil && i.data.Valid()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) First() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.index.First() {
|
||||
i.clearData()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
return i.Next()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Last() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.index.Last() {
|
||||
i.clearData()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
if !i.data.Last() {
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
return i.Prev()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Seek(key []byte) bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.index.Seek(key) {
|
||||
i.clearData()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
if !i.data.Seek(key) {
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
return i.Next()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Next() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
switch {
|
||||
case i.data != nil && !i.data.Next():
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
fallthrough
|
||||
case i.data == nil:
|
||||
if !i.index.Next() {
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
return i.Next()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Prev() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
switch {
|
||||
case i.data != nil && !i.data.Prev():
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
fallthrough
|
||||
case i.data == nil:
|
||||
if !i.index.Prev() {
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
if !i.data.Last() {
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
return i.Prev()
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Key() []byte {
|
||||
if i.data == nil {
|
||||
return nil
|
||||
}
|
||||
return i.data.Key()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Value() []byte {
|
||||
if i.data == nil {
|
||||
return nil
|
||||
}
|
||||
return i.data.Value()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Release() {
|
||||
i.clearData()
|
||||
i.index.Release()
|
||||
i.BasicReleaser.Release()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Error() error {
|
||||
if i.err != nil {
|
||||
return i.err
|
||||
}
|
||||
if err := i.index.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *indexedIterator) SetErrorCallback(f func(err error)) {
|
||||
i.errf = f
|
||||
}
|
||||
|
||||
// NewIndexedIterator returns an indexed iterator. An index is iterator
|
||||
// that returns another iterator, a data iterator. A data iterator is the
|
||||
// iterator that contains actual key/value pairs.
|
||||
//
|
||||
// If strict is true then error yield by data iterator will halt the indexed
|
||||
// iterator, on contrary if strict is false then the indexed iterator will
|
||||
// ignore those error and move on to the next index. If strictGet is true and
|
||||
// index.Get() yield an 'error iterator' then the indexed iterator will be halted.
|
||||
// An 'error iterator' is iterator which its Error() method always return non-nil
|
||||
// even before any 'seeks method' is called.
|
||||
func NewIndexedIterator(index IteratorIndexer, strict, strictGet bool) Iterator {
|
||||
return &indexedIterator{index: index, strict: strict, strictGet: strictGet}
|
||||
}
|
||||
-142
@@ -1,142 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package iterator provides interface and implementation to traverse over
|
||||
// contents of a database.
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// IteratorSeeker is the interface that wraps the 'seeks method'.
|
||||
type IteratorSeeker interface {
|
||||
// First moves the iterator to the first key/value pair. If the iterator
|
||||
// only contains one key/value pair then First and Last whould moves
|
||||
// to the same key/value pair.
|
||||
// It returns whether such pair exist.
|
||||
First() bool
|
||||
|
||||
// Last moves the iterator to the last key/value pair. If the iterator
|
||||
// only contains one key/value pair then First and Last whould moves
|
||||
// to the same key/value pair.
|
||||
// It returns whether such pair exist.
|
||||
Last() bool
|
||||
|
||||
// Seek moves the iterator to the first key/value pair whose key is greater
|
||||
// than or equal to the given key.
|
||||
// It returns whether such pair exist.
|
||||
//
|
||||
// It is safe to modify the contents of the argument after Seek returns.
|
||||
Seek(key []byte) bool
|
||||
|
||||
// Next moves the iterator to the next key/value pair.
|
||||
// It returns whether the iterator is exhausted.
|
||||
Next() bool
|
||||
|
||||
// Prev moves the iterator to the previous key/value pair.
|
||||
// It returns whether the iterator is exhausted.
|
||||
Prev() bool
|
||||
}
|
||||
|
||||
// CommonIterator is the interface that wraps common interator methods.
|
||||
type CommonIterator interface {
|
||||
IteratorSeeker
|
||||
|
||||
// util.Releaser is the interface that wraps basic Release method.
|
||||
// When called Release will releases any resources associated with the
|
||||
// iterator.
|
||||
util.Releaser
|
||||
|
||||
// util.ReleaseSetter is the interface that wraps the basic SetReleaser
|
||||
// method.
|
||||
util.ReleaseSetter
|
||||
|
||||
// TODO: Remove this when ready.
|
||||
Valid() bool
|
||||
|
||||
// Error returns any accumulated error. Exhausting all the key/value pairs
|
||||
// is not considered to be an error.
|
||||
Error() error
|
||||
}
|
||||
|
||||
// Iterator iterates over a DB's key/value pairs in key order.
|
||||
//
|
||||
// When encouter an error any 'seeks method' will return false and will
|
||||
// yield no key/value pairs. The error can be queried by calling the Error
|
||||
// method. Calling Release is still necessary.
|
||||
//
|
||||
// An iterator must be released after use, but it is not necessary to read
|
||||
// an iterator until exhaustion.
|
||||
// Also, an iterator is not necessarily goroutine-safe, but it is safe to use
|
||||
// multiple iterators concurrently, with each in a dedicated goroutine.
|
||||
type Iterator interface {
|
||||
CommonIterator
|
||||
|
||||
// Key returns the key of the current key/value pair, or nil if done.
|
||||
// The caller should not modify the contents of the returned slice, and
|
||||
// its contents may change on the next call to any 'seeks method'.
|
||||
Key() []byte
|
||||
|
||||
// Value returns the key of the current key/value pair, or nil if done.
|
||||
// The caller should not modify the contents of the returned slice, and
|
||||
// its contents may change on the next call to any 'seeks method'.
|
||||
Value() []byte
|
||||
}
|
||||
|
||||
// ErrorCallbackSetter is the interface that wraps basic SetErrorCallback
|
||||
// method.
|
||||
//
|
||||
// ErrorCallbackSetter implemented by indexed and merged iterator.
|
||||
type ErrorCallbackSetter interface {
|
||||
// SetErrorCallback allows set an error callback of the coresponding
|
||||
// iterator. Use nil to clear the callback.
|
||||
SetErrorCallback(f func(err error))
|
||||
}
|
||||
|
||||
type emptyIterator struct {
|
||||
releaser util.Releaser
|
||||
released bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (i *emptyIterator) rErr() {
|
||||
if i.err == nil && i.released {
|
||||
i.err = errors.New("leveldb/iterator: iterator released")
|
||||
}
|
||||
}
|
||||
|
||||
func (i *emptyIterator) Release() {
|
||||
if i.releaser != nil {
|
||||
i.releaser.Release()
|
||||
i.releaser = nil
|
||||
}
|
||||
i.released = true
|
||||
}
|
||||
|
||||
func (i *emptyIterator) SetReleaser(releaser util.Releaser) {
|
||||
if !i.released {
|
||||
i.releaser = releaser
|
||||
}
|
||||
}
|
||||
|
||||
func (*emptyIterator) Valid() bool { return false }
|
||||
func (i *emptyIterator) First() bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Last() bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Next() bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Prev() bool { i.rErr(); return false }
|
||||
func (*emptyIterator) Key() []byte { return nil }
|
||||
func (*emptyIterator) Value() []byte { return nil }
|
||||
func (i *emptyIterator) Error() error { return i.err }
|
||||
|
||||
// NewEmptyIterator creates an empty iterator. The err parameter can be
|
||||
// nil, but if not nil the given err will be returned by Error method.
|
||||
func NewEmptyIterator(err error) Iterator {
|
||||
return &emptyIterator{err: err}
|
||||
}
|
||||
Generated
Vendored
-17
@@ -1,17 +0,0 @@
|
||||
package iterator_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/testutil"
|
||||
)
|
||||
|
||||
func TestIterator(t *testing.T) {
|
||||
testutil.RunDefer()
|
||||
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Iterator Suite")
|
||||
}
|
||||
Generated
Vendored
-328
@@ -1,328 +0,0 @@
|
||||
// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record_test.go?r=df1fa28f7f3be6c3935548169002309c12967135
|
||||
// License, authors and contributors informations can be found at bellow URLs respectively:
|
||||
// https://code.google.com/p/leveldb-go/source/browse/LICENSE
|
||||
// https://code.google.com/p/leveldb-go/source/browse/AUTHORS
|
||||
// https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS
|
||||
|
||||
package journal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type dropper struct {
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (d dropper) Drop(err error) {
|
||||
d.t.Log(err)
|
||||
}
|
||||
|
||||
func short(s string) string {
|
||||
if len(s) < 64 {
|
||||
return s
|
||||
}
|
||||
return fmt.Sprintf("%s...(skipping %d bytes)...%s", s[:20], len(s)-40, s[len(s)-20:])
|
||||
}
|
||||
|
||||
// big returns a string of length n, composed of repetitions of partial.
|
||||
func big(partial string, n int) string {
|
||||
return strings.Repeat(partial, n/len(partial)+1)[:n]
|
||||
}
|
||||
|
||||
func TestEmpty(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
r := NewReader(buf, dropper{t}, true, true)
|
||||
if _, err := r.Next(); err != io.EOF {
|
||||
t.Fatalf("got %v, want %v", err, io.EOF)
|
||||
}
|
||||
}
|
||||
|
||||
func testGenerator(t *testing.T, reset func(), gen func() (string, bool)) {
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
reset()
|
||||
w := NewWriter(buf)
|
||||
for {
|
||||
s, ok := gen()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
ww, err := w.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ww.Write([]byte(s)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reset()
|
||||
r := NewReader(buf, dropper{t}, true, true)
|
||||
for {
|
||||
s, ok := gen()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
rr, err := r.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
x, err := ioutil.ReadAll(rr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(x) != s {
|
||||
t.Fatalf("got %q, want %q", short(string(x)), short(s))
|
||||
}
|
||||
}
|
||||
if _, err := r.Next(); err != io.EOF {
|
||||
t.Fatalf("got %v, want %v", err, io.EOF)
|
||||
}
|
||||
}
|
||||
|
||||
func testLiterals(t *testing.T, s []string) {
|
||||
var i int
|
||||
reset := func() {
|
||||
i = 0
|
||||
}
|
||||
gen := func() (string, bool) {
|
||||
if i == len(s) {
|
||||
return "", false
|
||||
}
|
||||
i++
|
||||
return s[i-1], true
|
||||
}
|
||||
testGenerator(t, reset, gen)
|
||||
}
|
||||
|
||||
func TestMany(t *testing.T) {
|
||||
const n = 1e5
|
||||
var i int
|
||||
reset := func() {
|
||||
i = 0
|
||||
}
|
||||
gen := func() (string, bool) {
|
||||
if i == n {
|
||||
return "", false
|
||||
}
|
||||
i++
|
||||
return fmt.Sprintf("%d.", i-1), true
|
||||
}
|
||||
testGenerator(t, reset, gen)
|
||||
}
|
||||
|
||||
func TestRandom(t *testing.T) {
|
||||
const n = 1e2
|
||||
var (
|
||||
i int
|
||||
r *rand.Rand
|
||||
)
|
||||
reset := func() {
|
||||
i, r = 0, rand.New(rand.NewSource(0))
|
||||
}
|
||||
gen := func() (string, bool) {
|
||||
if i == n {
|
||||
return "", false
|
||||
}
|
||||
i++
|
||||
return strings.Repeat(string(uint8(i)), r.Intn(2*blockSize+16)), true
|
||||
}
|
||||
testGenerator(t, reset, gen)
|
||||
}
|
||||
|
||||
func TestBasic(t *testing.T) {
|
||||
testLiterals(t, []string{
|
||||
strings.Repeat("a", 1000),
|
||||
strings.Repeat("b", 97270),
|
||||
strings.Repeat("c", 8000),
|
||||
})
|
||||
}
|
||||
|
||||
func TestBoundary(t *testing.T) {
|
||||
for i := blockSize - 16; i < blockSize+16; i++ {
|
||||
s0 := big("abcd", i)
|
||||
for j := blockSize - 16; j < blockSize+16; j++ {
|
||||
s1 := big("ABCDE", j)
|
||||
testLiterals(t, []string{s0, s1})
|
||||
testLiterals(t, []string{s0, "", s1})
|
||||
testLiterals(t, []string{s0, "x", s1})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlush(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
w := NewWriter(buf)
|
||||
// Write a couple of records. Everything should still be held
|
||||
// in the record.Writer buffer, so that buf.Len should be 0.
|
||||
w0, _ := w.Next()
|
||||
w0.Write([]byte("0"))
|
||||
w1, _ := w.Next()
|
||||
w1.Write([]byte("11"))
|
||||
if got, want := buf.Len(), 0; got != want {
|
||||
t.Fatalf("buffer length #0: got %d want %d", got, want)
|
||||
}
|
||||
// Flush the record.Writer buffer, which should yield 17 bytes.
|
||||
// 17 = 2*7 + 1 + 2, which is two headers and 1 + 2 payload bytes.
|
||||
if err := w.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := buf.Len(), 17; got != want {
|
||||
t.Fatalf("buffer length #1: got %d want %d", got, want)
|
||||
}
|
||||
// Do another write, one that isn't large enough to complete the block.
|
||||
// The write should not have flowed through to buf.
|
||||
w2, _ := w.Next()
|
||||
w2.Write(bytes.Repeat([]byte("2"), 10000))
|
||||
if got, want := buf.Len(), 17; got != want {
|
||||
t.Fatalf("buffer length #2: got %d want %d", got, want)
|
||||
}
|
||||
// Flushing should get us up to 10024 bytes written.
|
||||
// 10024 = 17 + 7 + 10000.
|
||||
if err := w.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := buf.Len(), 10024; got != want {
|
||||
t.Fatalf("buffer length #3: got %d want %d", got, want)
|
||||
}
|
||||
// Do a bigger write, one that completes the current block.
|
||||
// We should now have 32768 bytes (a complete block), without
|
||||
// an explicit flush.
|
||||
w3, _ := w.Next()
|
||||
w3.Write(bytes.Repeat([]byte("3"), 40000))
|
||||
if got, want := buf.Len(), 32768; got != want {
|
||||
t.Fatalf("buffer length #4: got %d want %d", got, want)
|
||||
}
|
||||
// Flushing should get us up to 50038 bytes written.
|
||||
// 50038 = 10024 + 2*7 + 40000. There are two headers because
|
||||
// the one record was split into two chunks.
|
||||
if err := w.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := buf.Len(), 50038; got != want {
|
||||
t.Fatalf("buffer length #5: got %d want %d", got, want)
|
||||
}
|
||||
// Check that reading those records give the right lengths.
|
||||
r := NewReader(buf, dropper{t}, true, true)
|
||||
wants := []int64{1, 2, 10000, 40000}
|
||||
for i, want := range wants {
|
||||
rr, _ := r.Next()
|
||||
n, err := io.Copy(ioutil.Discard, rr)
|
||||
if err != nil {
|
||||
t.Fatalf("read #%d: %v", i, err)
|
||||
}
|
||||
if n != want {
|
||||
t.Fatalf("read #%d: got %d bytes want %d", i, n, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonExhaustiveRead(t *testing.T) {
|
||||
const n = 100
|
||||
buf := new(bytes.Buffer)
|
||||
p := make([]byte, 10)
|
||||
rnd := rand.New(rand.NewSource(1))
|
||||
|
||||
w := NewWriter(buf)
|
||||
for i := 0; i < n; i++ {
|
||||
length := len(p) + rnd.Intn(3*blockSize)
|
||||
s := string(uint8(i)) + "123456789abcdefgh"
|
||||
ww, _ := w.Next()
|
||||
ww.Write([]byte(big(s, length)))
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := NewReader(buf, dropper{t}, true, true)
|
||||
for i := 0; i < n; i++ {
|
||||
rr, _ := r.Next()
|
||||
_, err := io.ReadFull(rr, p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := string(uint8(i)) + "123456789"
|
||||
if got := string(p); got != want {
|
||||
t.Fatalf("read #%d: got %q want %q", i, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleReader(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
w := NewWriter(buf)
|
||||
w0, err := w.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w0.Write([]byte("0"))
|
||||
w1, err := w.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w1.Write([]byte("11"))
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := NewReader(buf, dropper{t}, true, true)
|
||||
r0, err := r.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r1, err := r.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := make([]byte, 1)
|
||||
if _, err := r0.Read(p); err == nil || !strings.Contains(err.Error(), "stale") {
|
||||
t.Fatalf("stale read #0: unexpected error: %v", err)
|
||||
}
|
||||
if _, err := r1.Read(p); err != nil {
|
||||
t.Fatalf("fresh read #1: got %v want nil error", err)
|
||||
}
|
||||
if p[0] != '1' {
|
||||
t.Fatalf("fresh read #1: byte contents: got '%c' want '1'", p[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleWriter(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
w := NewWriter(buf)
|
||||
w0, err := w.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w1, err := w.Next()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w0.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") {
|
||||
t.Fatalf("stale write #0: unexpected error: %v", err)
|
||||
}
|
||||
if _, err := w1.Write([]byte("11")); err != nil {
|
||||
t.Fatalf("fresh write #1: got %v want nil error", err)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
if _, err := w1.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") {
|
||||
t.Fatalf("stale write #1: unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
-139
@@ -1,139 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type vType int
|
||||
|
||||
func (t vType) String() string {
|
||||
switch t {
|
||||
case tDel:
|
||||
return "d"
|
||||
case tVal:
|
||||
return "v"
|
||||
}
|
||||
return "x"
|
||||
}
|
||||
|
||||
// Value types encoded as the last component of internal keys.
|
||||
// Don't modify; this value are saved to disk.
|
||||
const (
|
||||
tDel vType = iota
|
||||
tVal
|
||||
)
|
||||
|
||||
// tSeek defines the vType that should be passed when constructing an
|
||||
// internal key for seeking to a particular sequence number (since we
|
||||
// sort sequence numbers in decreasing order and the value type is
|
||||
// embedded as the low 8 bits in the sequence number in internal keys,
|
||||
// we need to use the highest-numbered ValueType, not the lowest).
|
||||
const tSeek = tVal
|
||||
|
||||
const (
|
||||
// Maximum value possible for sequence number; the 8-bits are
|
||||
// used by value type, so its can packed together in single
|
||||
// 64-bit integer.
|
||||
kMaxSeq uint64 = (uint64(1) << 56) - 1
|
||||
// Maximum value possible for packed sequence number and type.
|
||||
kMaxNum uint64 = (kMaxSeq << 8) | uint64(tSeek)
|
||||
)
|
||||
|
||||
// Maximum number encoded in bytes.
|
||||
var kMaxNumBytes = make([]byte, 8)
|
||||
|
||||
func init() {
|
||||
binary.LittleEndian.PutUint64(kMaxNumBytes, kMaxNum)
|
||||
}
|
||||
|
||||
type iKey []byte
|
||||
|
||||
func newIKey(ukey []byte, seq uint64, t vType) iKey {
|
||||
if seq > kMaxSeq || t > tVal {
|
||||
panic("invalid seq number or value type")
|
||||
}
|
||||
|
||||
b := make(iKey, len(ukey)+8)
|
||||
copy(b, ukey)
|
||||
binary.LittleEndian.PutUint64(b[len(ukey):], (seq<<8)|uint64(t))
|
||||
return b
|
||||
}
|
||||
|
||||
func parseIkey(p []byte) (ukey []byte, seq uint64, t vType, ok bool) {
|
||||
if len(p) < 8 {
|
||||
return
|
||||
}
|
||||
num := binary.LittleEndian.Uint64(p[len(p)-8:])
|
||||
seq, t = uint64(num>>8), vType(num&0xff)
|
||||
if t > tVal {
|
||||
return
|
||||
}
|
||||
ukey = p[:len(p)-8]
|
||||
ok = true
|
||||
return
|
||||
}
|
||||
|
||||
func validIkey(p []byte) bool {
|
||||
_, _, _, ok := parseIkey(p)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (p iKey) assert() {
|
||||
if p == nil {
|
||||
panic("nil iKey")
|
||||
}
|
||||
if len(p) < 8 {
|
||||
panic(fmt.Sprintf("invalid iKey %q, len=%d", []byte(p), len(p)))
|
||||
}
|
||||
}
|
||||
|
||||
func (p iKey) ok() bool {
|
||||
if len(p) < 8 {
|
||||
return false
|
||||
}
|
||||
_, _, ok := p.parseNum()
|
||||
return ok
|
||||
}
|
||||
|
||||
func (p iKey) ukey() []byte {
|
||||
p.assert()
|
||||
return p[:len(p)-8]
|
||||
}
|
||||
|
||||
func (p iKey) num() uint64 {
|
||||
p.assert()
|
||||
return binary.LittleEndian.Uint64(p[len(p)-8:])
|
||||
}
|
||||
|
||||
func (p iKey) parseNum() (seq uint64, t vType, ok bool) {
|
||||
if p == nil {
|
||||
panic("nil iKey")
|
||||
}
|
||||
if len(p) < 8 {
|
||||
return
|
||||
}
|
||||
num := p.num()
|
||||
seq, t = uint64(num>>8), vType(num&0xff)
|
||||
if t > tVal {
|
||||
return 0, 0, false
|
||||
}
|
||||
ok = true
|
||||
return
|
||||
}
|
||||
|
||||
func (p iKey) String() string {
|
||||
if len(p) == 0 {
|
||||
return "<nil>"
|
||||
}
|
||||
if seq, t, ok := p.parseNum(); ok {
|
||||
return fmt.Sprintf("%s,%s%d", shorten(string(p.ukey())), t, seq)
|
||||
}
|
||||
return "<invalid>"
|
||||
}
|
||||
-123
@@ -1,123 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
)
|
||||
|
||||
var defaultIComparer = &iComparer{comparer.DefaultComparer}
|
||||
|
||||
func ikey(key string, seq uint64, t vType) iKey {
|
||||
return newIKey([]byte(key), uint64(seq), t)
|
||||
}
|
||||
|
||||
func shortSep(a, b []byte) []byte {
|
||||
dst := make([]byte, len(a))
|
||||
dst = defaultIComparer.Separator(dst[:0], a, b)
|
||||
if dst == nil {
|
||||
return a
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func shortSuccessor(b []byte) []byte {
|
||||
dst := make([]byte, len(b))
|
||||
dst = defaultIComparer.Successor(dst[:0], b)
|
||||
if dst == nil {
|
||||
return b
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func testSingleKey(t *testing.T, key string, seq uint64, vt vType) {
|
||||
ik := ikey(key, seq, vt)
|
||||
|
||||
if !bytes.Equal(ik.ukey(), []byte(key)) {
|
||||
t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
|
||||
}
|
||||
|
||||
if rseq, rt, ok := ik.parseNum(); ok {
|
||||
if rseq != seq {
|
||||
t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
|
||||
}
|
||||
|
||||
if rt != vt {
|
||||
t.Errorf("type does not equal, got %v, want %v", rt, vt)
|
||||
}
|
||||
} else {
|
||||
t.Error("cannot parse seq and type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIKey_EncodeDecode(t *testing.T) {
|
||||
keys := []string{"", "k", "hello", "longggggggggggggggggggggg"}
|
||||
seqs := []uint64{
|
||||
1, 2, 3,
|
||||
(1 << 8) - 1, 1 << 8, (1 << 8) + 1,
|
||||
(1 << 16) - 1, 1 << 16, (1 << 16) + 1,
|
||||
(1 << 32) - 1, 1 << 32, (1 << 32) + 1,
|
||||
}
|
||||
for _, key := range keys {
|
||||
for _, seq := range seqs {
|
||||
testSingleKey(t, key, seq, tVal)
|
||||
testSingleKey(t, "hello", 1, tDel)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertBytes(t *testing.T, want, got []byte) {
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("assert failed, got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIKeyShortSeparator(t *testing.T) {
|
||||
// When user keys are same
|
||||
assertBytes(t, ikey("foo", 100, tVal),
|
||||
shortSep(ikey("foo", 100, tVal),
|
||||
ikey("foo", 99, tVal)))
|
||||
assertBytes(t, ikey("foo", 100, tVal),
|
||||
shortSep(ikey("foo", 100, tVal),
|
||||
ikey("foo", 101, tVal)))
|
||||
assertBytes(t, ikey("foo", 100, tVal),
|
||||
shortSep(ikey("foo", 100, tVal),
|
||||
ikey("foo", 100, tVal)))
|
||||
assertBytes(t, ikey("foo", 100, tVal),
|
||||
shortSep(ikey("foo", 100, tVal),
|
||||
ikey("foo", 100, tDel)))
|
||||
|
||||
// When user keys are misordered
|
||||
assertBytes(t, ikey("foo", 100, tVal),
|
||||
shortSep(ikey("foo", 100, tVal),
|
||||
ikey("bar", 99, tVal)))
|
||||
|
||||
// When user keys are different, but correctly ordered
|
||||
assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
|
||||
shortSep(ikey("foo", 100, tVal),
|
||||
ikey("hello", 200, tVal)))
|
||||
|
||||
// When start user key is prefix of limit user key
|
||||
assertBytes(t, ikey("foo", 100, tVal),
|
||||
shortSep(ikey("foo", 100, tVal),
|
||||
ikey("foobar", 200, tVal)))
|
||||
|
||||
// When limit user key is prefix of start user key
|
||||
assertBytes(t, ikey("foobar", 100, tVal),
|
||||
shortSep(ikey("foobar", 100, tVal),
|
||||
ikey("foo", 200, tVal)))
|
||||
}
|
||||
|
||||
func TestIKeyShortestSuccessor(t *testing.T) {
|
||||
assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
|
||||
shortSuccessor(ikey("foo", 100, tVal)))
|
||||
assertBytes(t, ikey("\xff\xff", 100, tVal),
|
||||
shortSuccessor(ikey("\xff\xff", 100, tVal)))
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/testutil"
|
||||
)
|
||||
|
||||
func TestLeveldb(t *testing.T) {
|
||||
testutil.RunDefer()
|
||||
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Leveldb Suite")
|
||||
|
||||
RegisterTestingT(t)
|
||||
testutil.RunDefer("teardown")
|
||||
}
|
||||
Generated
Vendored
-17
@@ -1,17 +0,0 @@
|
||||
package memdb
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/testutil"
|
||||
)
|
||||
|
||||
func TestMemdb(t *testing.T) {
|
||||
testutil.RunDefer()
|
||||
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Memdb Suite")
|
||||
}
|
||||
-318
@@ -1,318 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package opt provides sets of options used by LevelDB.
|
||||
package opt
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/cache"
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
)
|
||||
|
||||
const (
|
||||
KiB = 1024
|
||||
MiB = KiB * 1024
|
||||
GiB = MiB * 1024
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultBlockCacheSize = 8 * MiB
|
||||
DefaultBlockRestartInterval = 16
|
||||
DefaultBlockSize = 4 * KiB
|
||||
DefaultCompressionType = SnappyCompression
|
||||
DefaultMaxOpenFiles = 1000
|
||||
DefaultWriteBuffer = 4 * MiB
|
||||
)
|
||||
|
||||
type noCache struct{}
|
||||
|
||||
func (noCache) SetCapacity(capacity int) {}
|
||||
func (noCache) GetNamespace(id uint64) cache.Namespace { return nil }
|
||||
func (noCache) Purge(fin cache.PurgeFin) {}
|
||||
func (noCache) Zap(closed bool) {}
|
||||
|
||||
var NoCache cache.Cache = noCache{}
|
||||
|
||||
// Compression is the per-block compression algorithm to use.
|
||||
type Compression uint
|
||||
|
||||
func (c Compression) String() string {
|
||||
switch c {
|
||||
case DefaultCompression:
|
||||
return "default"
|
||||
case NoCompression:
|
||||
return "none"
|
||||
case SnappyCompression:
|
||||
return "snappy"
|
||||
}
|
||||
return "invalid"
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultCompression Compression = iota
|
||||
NoCompression
|
||||
SnappyCompression
|
||||
nCompression
|
||||
)
|
||||
|
||||
// Strict is the DB strict level.
|
||||
type Strict uint
|
||||
|
||||
const (
|
||||
// If present then a corrupted or invalid chunk or block in manifest
|
||||
// journal will cause an error istead of being dropped.
|
||||
StrictManifest Strict = 1 << iota
|
||||
|
||||
// If present then a corrupted or invalid chunk or block in journal
|
||||
// will cause an error istead of being dropped.
|
||||
StrictJournal
|
||||
|
||||
// If present then journal chunk checksum will be verified.
|
||||
StrictJournalChecksum
|
||||
|
||||
// If present then an invalid key/value pair will cause an error
|
||||
// instead of being skipped.
|
||||
StrictIterator
|
||||
|
||||
// If present then 'sorted table' block checksum will be verified.
|
||||
StrictBlockChecksum
|
||||
|
||||
// StrictAll enables all strict flags.
|
||||
StrictAll = StrictManifest | StrictJournal | StrictJournalChecksum | StrictIterator | StrictBlockChecksum
|
||||
|
||||
// DefaultStrict is the default strict flags. Specify any strict flags
|
||||
// will override default strict flags as whole (i.e. not OR'ed).
|
||||
DefaultStrict = StrictJournalChecksum | StrictBlockChecksum
|
||||
|
||||
// NoStrict disables all strict flags. Override default strict flags.
|
||||
NoStrict = ^StrictAll
|
||||
)
|
||||
|
||||
// Options holds the optional parameters for the DB at large.
|
||||
type Options struct {
|
||||
// AltFilters defines one or more 'alternative filters'.
|
||||
// 'alternative filters' will be used during reads if a filter block
|
||||
// does not match with the 'effective filter'.
|
||||
//
|
||||
// The default value is nil
|
||||
AltFilters []filter.Filter
|
||||
|
||||
// BlockCache provides per-block caching for LevelDB. Specify NoCache to
|
||||
// disable block caching.
|
||||
//
|
||||
// By default LevelDB will create LRU-cache with capacity of 8MiB.
|
||||
BlockCache cache.Cache
|
||||
|
||||
// BlockRestartInterval is the number of keys between restart points for
|
||||
// delta encoding of keys.
|
||||
//
|
||||
// The default value is 16.
|
||||
BlockRestartInterval int
|
||||
|
||||
// BlockSize is the minimum uncompressed size in bytes of each 'sorted table'
|
||||
// block.
|
||||
//
|
||||
// The default value is 4KiB.
|
||||
BlockSize int
|
||||
|
||||
// Comparer defines a total ordering over the space of []byte keys: a 'less
|
||||
// than' relationship. The same comparison algorithm must be used for reads
|
||||
// and writes over the lifetime of the DB.
|
||||
//
|
||||
// The default value uses the same ordering as bytes.Compare.
|
||||
Comparer comparer.Comparer
|
||||
|
||||
// Compression defines the per-block compression to use.
|
||||
//
|
||||
// The default value (DefaultCompression) uses snappy compression.
|
||||
Compression Compression
|
||||
|
||||
// ErrorIfExist defines whether an error should returned if the DB already
|
||||
// exist.
|
||||
//
|
||||
// The default value is false.
|
||||
ErrorIfExist bool
|
||||
|
||||
// ErrorIfMissing defines whether an error should returned if the DB is
|
||||
// missing. If false then the database will be created if missing, otherwise
|
||||
// an error will be returned.
|
||||
//
|
||||
// The default value is false.
|
||||
ErrorIfMissing bool
|
||||
|
||||
// Filter defines an 'effective filter' to use. An 'effective filter'
|
||||
// if defined will be used to generate per-table filter block.
|
||||
// The filter name will be stored on disk.
|
||||
// During reads LevelDB will try to find matching filter from
|
||||
// 'effective filter' and 'alternative filters'.
|
||||
//
|
||||
// Filter can be changed after a DB has been created. It is recommended
|
||||
// to put old filter to the 'alternative filters' to mitigate lack of
|
||||
// filter during transition period.
|
||||
//
|
||||
// A filter is used to reduce disk reads when looking for a specific key.
|
||||
//
|
||||
// The default value is nil.
|
||||
Filter filter.Filter
|
||||
|
||||
// MaxOpenFiles defines maximum number of open files to kept around
|
||||
// (cached). This is not an hard limit, actual open files may exceed
|
||||
// the defined value.
|
||||
//
|
||||
// The default value is 1000.
|
||||
MaxOpenFiles int
|
||||
|
||||
// Strict defines the DB strict level.
|
||||
Strict Strict
|
||||
|
||||
// WriteBuffer defines maximum size of a 'memdb' before flushed to
|
||||
// 'sorted table'. 'memdb' is an in-memory DB backed by an on-disk
|
||||
// unsorted journal.
|
||||
//
|
||||
// LevelDB may held up to two 'memdb' at the same time.
|
||||
//
|
||||
// The default value is 4MiB.
|
||||
WriteBuffer int
|
||||
}
|
||||
|
||||
func (o *Options) GetAltFilters() []filter.Filter {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
return o.AltFilters
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockCache() cache.Cache {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
return o.BlockCache
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockRestartInterval() int {
|
||||
if o == nil || o.BlockRestartInterval <= 0 {
|
||||
return DefaultBlockRestartInterval
|
||||
}
|
||||
return o.BlockRestartInterval
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockSize() int {
|
||||
if o == nil || o.BlockSize <= 0 {
|
||||
return DefaultBlockSize
|
||||
}
|
||||
return o.BlockSize
|
||||
}
|
||||
|
||||
func (o *Options) GetComparer() comparer.Comparer {
|
||||
if o == nil || o.Comparer == nil {
|
||||
return comparer.DefaultComparer
|
||||
}
|
||||
return o.Comparer
|
||||
}
|
||||
|
||||
func (o *Options) GetCompression() Compression {
|
||||
if o == nil || o.Compression <= DefaultCompression || o.Compression >= nCompression {
|
||||
return DefaultCompressionType
|
||||
}
|
||||
return o.Compression
|
||||
}
|
||||
|
||||
func (o *Options) GetErrorIfExist() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.ErrorIfExist
|
||||
}
|
||||
|
||||
func (o *Options) GetErrorIfMissing() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.ErrorIfMissing
|
||||
}
|
||||
|
||||
func (o *Options) GetFilter() filter.Filter {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
return o.Filter
|
||||
}
|
||||
|
||||
func (o *Options) GetMaxOpenFiles() int {
|
||||
if o == nil || o.MaxOpenFiles <= 0 {
|
||||
return DefaultMaxOpenFiles
|
||||
}
|
||||
return o.MaxOpenFiles
|
||||
}
|
||||
|
||||
func (o *Options) GetStrict(strict Strict) bool {
|
||||
if o == nil || o.Strict == 0 {
|
||||
return DefaultStrict&strict != 0
|
||||
}
|
||||
return o.Strict&strict != 0
|
||||
}
|
||||
|
||||
func (o *Options) GetWriteBuffer() int {
|
||||
if o == nil || o.WriteBuffer <= 0 {
|
||||
return DefaultWriteBuffer
|
||||
}
|
||||
return o.WriteBuffer
|
||||
}
|
||||
|
||||
// ReadOptions holds the optional parameters for 'read operation'. The
|
||||
// 'read operation' includes Get, Find and NewIterator.
|
||||
type ReadOptions struct {
|
||||
// DontFillCache defines whether block reads for this 'read operation'
|
||||
// should be cached. If false then the block will be cached. This does
|
||||
// not affects already cached block.
|
||||
//
|
||||
// The default value is false.
|
||||
DontFillCache bool
|
||||
|
||||
// Strict overrides global DB strict level. Only StrictIterator and
|
||||
// StrictBlockChecksum that does have effects here.
|
||||
Strict Strict
|
||||
}
|
||||
|
||||
func (ro *ReadOptions) GetDontFillCache() bool {
|
||||
if ro == nil {
|
||||
return false
|
||||
}
|
||||
return ro.DontFillCache
|
||||
}
|
||||
|
||||
func (ro *ReadOptions) GetStrict(strict Strict) bool {
|
||||
if ro == nil {
|
||||
return false
|
||||
}
|
||||
return ro.Strict&strict != 0
|
||||
}
|
||||
|
||||
// WriteOptions holds the optional parameters for 'write operation'. The
|
||||
// 'write operation' includes Write, Put and Delete.
|
||||
type WriteOptions struct {
|
||||
// Sync is whether to sync underlying writes from the OS buffer cache
|
||||
// through to actual disk, if applicable. Setting Sync can result in
|
||||
// slower writes.
|
||||
//
|
||||
// If false, and the machine crashes, then some recent writes may be lost.
|
||||
// Note that if it is just the process that crashes (and the machine does
|
||||
// not) then no writes will be lost.
|
||||
//
|
||||
// In other words, Sync being false has the same semantics as a write
|
||||
// system call. Sync being true means write followed by fsync.
|
||||
//
|
||||
// The default value is false.
|
||||
Sync bool
|
||||
}
|
||||
|
||||
func (wo *WriteOptions) GetSync() bool {
|
||||
if wo == nil {
|
||||
return false
|
||||
}
|
||||
return wo.Sync
|
||||
}
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/cache"
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
func (s *session) setOptions(o *opt.Options) {
|
||||
s.o = &opt.Options{}
|
||||
if o != nil {
|
||||
*s.o = *o
|
||||
}
|
||||
// Alternative filters.
|
||||
if filters := o.GetAltFilters(); len(filters) > 0 {
|
||||
s.o.AltFilters = make([]filter.Filter, len(filters))
|
||||
for i, filter := range filters {
|
||||
s.o.AltFilters[i] = &iFilter{filter}
|
||||
}
|
||||
}
|
||||
// Block cache.
|
||||
switch o.GetBlockCache() {
|
||||
case nil:
|
||||
s.o.BlockCache = cache.NewLRUCache(opt.DefaultBlockCacheSize)
|
||||
case opt.NoCache:
|
||||
s.o.BlockCache = nil
|
||||
}
|
||||
// Comparer.
|
||||
s.icmp = &iComparer{o.GetComparer()}
|
||||
s.o.Comparer = s.icmp
|
||||
// Filter.
|
||||
if filter := o.GetFilter(); filter != nil {
|
||||
s.o.Filter = &iFilter{filter}
|
||||
}
|
||||
}
|
||||
-403
@@ -1,403 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/journal"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// session represent a persistent database session.
|
||||
type session struct {
|
||||
// Need 64-bit alignment.
|
||||
stFileNum uint64 // current unused file number
|
||||
stJournalNum uint64 // current journal file number; need external synchronization
|
||||
stPrevJournalNum uint64 // prev journal file number; no longer used; for compatibility with older version of leveldb
|
||||
stSeq uint64 // last mem compacted seq; need external synchronization
|
||||
stTempFileNum uint64
|
||||
|
||||
stor storage.Storage
|
||||
storLock util.Releaser
|
||||
o *opt.Options
|
||||
icmp *iComparer
|
||||
tops *tOps
|
||||
|
||||
manifest *journal.Writer
|
||||
manifestWriter storage.Writer
|
||||
manifestFile storage.File
|
||||
|
||||
stCPtrs [kNumLevels]iKey // compact pointers; need external synchronization
|
||||
stVersion *version // current version
|
||||
vmu sync.Mutex
|
||||
}
|
||||
|
||||
func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
|
||||
if stor == nil {
|
||||
return nil, os.ErrInvalid
|
||||
}
|
||||
storLock, err := stor.Lock()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s = &session{
|
||||
stor: stor,
|
||||
storLock: storLock,
|
||||
}
|
||||
s.setOptions(o)
|
||||
s.tops = newTableOps(s, s.o.GetMaxOpenFiles())
|
||||
s.setVersion(&version{s: s})
|
||||
s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock D·DeletedEntry L·Level Q·SeqNum T·TimeElapsed")
|
||||
return
|
||||
}
|
||||
|
||||
// Close session.
|
||||
func (s *session) close() {
|
||||
s.tops.close()
|
||||
if bc := s.o.GetBlockCache(); bc != nil {
|
||||
bc.Purge(nil)
|
||||
}
|
||||
if s.manifest != nil {
|
||||
s.manifest.Close()
|
||||
}
|
||||
if s.manifestWriter != nil {
|
||||
s.manifestWriter.Close()
|
||||
}
|
||||
s.manifest = nil
|
||||
s.manifestWriter = nil
|
||||
s.manifestFile = nil
|
||||
s.stVersion = nil
|
||||
}
|
||||
|
||||
func (s *session) release() {
|
||||
s.storLock.Release()
|
||||
}
|
||||
|
||||
// Create a new database session; need external synchronization.
|
||||
func (s *session) create() error {
|
||||
// create manifest
|
||||
return s.newManifest(nil, nil)
|
||||
}
|
||||
|
||||
// Recover a database session; need external synchronization.
|
||||
func (s *session) recover() (err error) {
|
||||
defer func() {
|
||||
if os.IsNotExist(err) {
|
||||
// Don't return os.ErrNotExist if the underlying storage contains
|
||||
// other files that belong to LevelDB. So the DB won't get trashed.
|
||||
if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 {
|
||||
err = ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest file missing")}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
file, err := s.stor.GetManifest()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
strict := s.o.GetStrict(opt.StrictManifest)
|
||||
jr := journal.NewReader(reader, dropper{s, file}, strict, true)
|
||||
|
||||
staging := s.version_NB().newStaging()
|
||||
rec := &sessionRecord{}
|
||||
for {
|
||||
var r io.Reader
|
||||
r, err = jr.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
break
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
err = rec.decode(r)
|
||||
if err == nil {
|
||||
// save compact pointers
|
||||
for _, rp := range rec.compactionPointers {
|
||||
s.stCPtrs[rp.level] = iKey(rp.key)
|
||||
}
|
||||
// commit record to version staging
|
||||
staging.commit(rec)
|
||||
} else if strict {
|
||||
return ErrCorrupted{Type: CorruptedManifest, Err: err}
|
||||
} else {
|
||||
s.logf("manifest error: %v (skipped)", err)
|
||||
}
|
||||
rec.resetCompactionPointers()
|
||||
rec.resetAddedTables()
|
||||
rec.resetDeletedTables()
|
||||
}
|
||||
|
||||
switch {
|
||||
case !rec.has(recComparer):
|
||||
return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing comparer name")}
|
||||
case rec.comparer != s.icmp.uName():
|
||||
return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: comparer mismatch, " + "want '" + s.icmp.uName() + "', " + "got '" + rec.comparer + "'")}
|
||||
case !rec.has(recNextNum):
|
||||
return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing next file number")}
|
||||
case !rec.has(recJournalNum):
|
||||
return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing journal file number")}
|
||||
case !rec.has(recSeq):
|
||||
return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing seq number")}
|
||||
}
|
||||
|
||||
s.manifestFile = file
|
||||
s.setVersion(staging.finish())
|
||||
s.setFileNum(rec.nextNum)
|
||||
s.recordCommited(rec)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Commit session; need external synchronization.
|
||||
func (s *session) commit(r *sessionRecord) (err error) {
|
||||
// spawn new version based on current version
|
||||
nv := s.version_NB().spawn(r)
|
||||
|
||||
if s.manifest == nil {
|
||||
// manifest journal writer not yet created, create one
|
||||
err = s.newManifest(r, nv)
|
||||
} else {
|
||||
err = s.flushManifest(r)
|
||||
}
|
||||
|
||||
// finally, apply new version if no error rise
|
||||
if err == nil {
|
||||
s.setVersion(nv)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Pick a compaction based on current state; need external synchronization.
|
||||
func (s *session) pickCompaction() *compaction {
|
||||
v := s.version_NB()
|
||||
|
||||
var level int
|
||||
var t0 tFiles
|
||||
if v.cScore >= 1 {
|
||||
level = v.cLevel
|
||||
cp := s.stCPtrs[level]
|
||||
tt := v.tables[level]
|
||||
for _, t := range tt {
|
||||
if cp == nil || s.icmp.Compare(t.max, cp) > 0 {
|
||||
t0 = append(t0, t)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(t0) == 0 {
|
||||
t0 = append(t0, tt[0])
|
||||
}
|
||||
} else {
|
||||
if p := atomic.LoadPointer(&v.cSeek); p != nil {
|
||||
ts := (*tSet)(p)
|
||||
level = ts.level
|
||||
t0 = append(t0, ts.table)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
c := &compaction{s: s, version: v, level: level}
|
||||
if level == 0 {
|
||||
min, max := t0.getRange(s.icmp)
|
||||
t0 = nil
|
||||
v.tables[0].getOverlaps(min.ukey(), max.ukey(), &t0, false, s.icmp.ucmp)
|
||||
}
|
||||
|
||||
c.tables[0] = t0
|
||||
c.expand()
|
||||
return c
|
||||
}
|
||||
|
||||
// Create compaction from given level and range; need external synchronization.
|
||||
func (s *session) getCompactionRange(level int, min, max []byte) *compaction {
|
||||
v := s.version_NB()
|
||||
|
||||
var t0 tFiles
|
||||
v.tables[level].getOverlaps(min, max, &t0, level != 0, s.icmp.ucmp)
|
||||
if len(t0) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Avoid compacting too much in one shot in case the range is large.
|
||||
// But we cannot do this for level-0 since level-0 files can overlap
|
||||
// and we must not pick one file and drop another older file if the
|
||||
// two files overlap.
|
||||
if level > 0 {
|
||||
limit := uint64(kMaxTableSize)
|
||||
total := uint64(0)
|
||||
for i, t := range t0 {
|
||||
total += t.size
|
||||
if total >= limit {
|
||||
s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1)
|
||||
t0 = t0[:i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
c := &compaction{s: s, version: v, level: level}
|
||||
c.tables[0] = t0
|
||||
c.expand()
|
||||
return c
|
||||
}
|
||||
|
||||
// compaction represent a compaction state
|
||||
type compaction struct {
|
||||
s *session
|
||||
version *version
|
||||
|
||||
level int
|
||||
tables [2]tFiles
|
||||
|
||||
gp tFiles
|
||||
gpidx int
|
||||
seenKey bool
|
||||
overlappedBytes uint64
|
||||
min, max iKey
|
||||
|
||||
tPtrs [kNumLevels]int
|
||||
}
|
||||
|
||||
// Expand compacted tables; need external synchronization.
|
||||
func (c *compaction) expand() {
|
||||
s := c.s
|
||||
v := c.version
|
||||
|
||||
level := c.level
|
||||
vt0, vt1 := v.tables[level], v.tables[level+1]
|
||||
|
||||
t0, t1 := c.tables[0], c.tables[1]
|
||||
min, max := t0.getRange(s.icmp)
|
||||
vt1.getOverlaps(min.ukey(), max.ukey(), &t1, true, s.icmp.ucmp)
|
||||
|
||||
// Get entire range covered by compaction
|
||||
amin, amax := append(t0, t1...).getRange(s.icmp)
|
||||
|
||||
// See if we can grow the number of inputs in "level" without
|
||||
// changing the number of "level+1" files we pick up.
|
||||
if len(t1) > 0 {
|
||||
var exp0 tFiles
|
||||
vt0.getOverlaps(amin.ukey(), amax.ukey(), &exp0, level != 0, s.icmp.ucmp)
|
||||
if len(exp0) > len(t0) && t1.size()+exp0.size() < kExpCompactionMaxBytes {
|
||||
var exp1 tFiles
|
||||
xmin, xmax := exp0.getRange(s.icmp)
|
||||
vt1.getOverlaps(xmin.ukey(), xmax.ukey(), &exp1, true, s.icmp.ucmp)
|
||||
if len(exp1) == len(t1) {
|
||||
s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
|
||||
level, level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
|
||||
len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
|
||||
min, max = xmin, xmax
|
||||
t0, t1 = exp0, exp1
|
||||
amin, amax = append(t0, t1...).getRange(s.icmp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute the set of grandparent files that overlap this compaction
|
||||
// (parent == level+1; grandparent == level+2)
|
||||
if level+2 < kNumLevels {
|
||||
v.tables[level+2].getOverlaps(amin.ukey(), amax.ukey(), &c.gp, true, s.icmp.ucmp)
|
||||
}
|
||||
|
||||
c.tables[0], c.tables[1] = t0, t1
|
||||
c.min, c.max = min, max
|
||||
}
|
||||
|
||||
// Check whether compaction is trivial.
|
||||
func (c *compaction) trivial() bool {
|
||||
return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= kMaxGrandParentOverlapBytes
|
||||
}
|
||||
|
||||
func (c *compaction) isBaseLevelForKey(key []byte) bool {
|
||||
s := c.s
|
||||
v := c.version
|
||||
|
||||
for level, tt := range v.tables[c.level+2:] {
|
||||
for c.tPtrs[level] < len(tt) {
|
||||
t := tt[c.tPtrs[level]]
|
||||
if s.icmp.uCompare(key, t.max.ukey()) <= 0 {
|
||||
// We've advanced far enough
|
||||
if s.icmp.uCompare(key, t.min.ukey()) >= 0 {
|
||||
// Key falls in this file's range, so definitely not base level
|
||||
return false
|
||||
}
|
||||
break
|
||||
}
|
||||
c.tPtrs[level]++
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *compaction) shouldStopBefore(key iKey) bool {
|
||||
for ; c.gpidx < len(c.gp); c.gpidx++ {
|
||||
gp := c.gp[c.gpidx]
|
||||
if c.s.icmp.Compare(key, gp.max) <= 0 {
|
||||
break
|
||||
}
|
||||
if c.seenKey {
|
||||
c.overlappedBytes += gp.size
|
||||
}
|
||||
}
|
||||
c.seenKey = true
|
||||
|
||||
if c.overlappedBytes > kMaxGrandParentOverlapBytes {
|
||||
// Too much overlap for current output; start new output
|
||||
c.overlappedBytes = 0
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *compaction) newIterator() iterator.Iterator {
|
||||
s := c.s
|
||||
|
||||
level := c.level
|
||||
icap := 2
|
||||
if c.level == 0 {
|
||||
icap = len(c.tables[0]) + 1
|
||||
}
|
||||
its := make([]iterator.Iterator, 0, icap)
|
||||
|
||||
ro := &opt.ReadOptions{
|
||||
DontFillCache: true,
|
||||
}
|
||||
strict := s.o.GetStrict(opt.StrictIterator)
|
||||
|
||||
for i, tt := range c.tables {
|
||||
if len(tt) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if level+i == 0 {
|
||||
for _, t := range tt {
|
||||
its = append(its, s.tops.newIterator(t, nil, ro))
|
||||
}
|
||||
} else {
|
||||
it := iterator.NewIndexedIterator(tt.newIndexIterator(s.tops, s.icmp, nil, ro), strict, true)
|
||||
its = append(its, it)
|
||||
}
|
||||
}
|
||||
|
||||
return iterator.NewMergedIterator(its, s.icmp, true)
|
||||
}
|
||||
-308
@@ -1,308 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
var errCorruptManifest = errors.New("leveldb: corrupt manifest")
|
||||
|
||||
type byteReader interface {
|
||||
io.Reader
|
||||
io.ByteReader
|
||||
}
|
||||
|
||||
// These numbers are written to disk and should not be changed.
|
||||
const (
|
||||
recComparer = 1
|
||||
recJournalNum = 2
|
||||
recNextNum = 3
|
||||
recSeq = 4
|
||||
recCompactionPointer = 5
|
||||
recDeletedTable = 6
|
||||
recNewTable = 7
|
||||
// 8 was used for large value refs
|
||||
recPrevJournalNum = 9
|
||||
)
|
||||
|
||||
type cpRecord struct {
|
||||
level int
|
||||
key iKey
|
||||
}
|
||||
|
||||
type ntRecord struct {
|
||||
level int
|
||||
num uint64
|
||||
size uint64
|
||||
min iKey
|
||||
max iKey
|
||||
}
|
||||
|
||||
func (r ntRecord) makeFile(s *session) *tFile {
|
||||
return newTFile(s.getTableFile(r.num), r.size, r.min, r.max)
|
||||
}
|
||||
|
||||
type dtRecord struct {
|
||||
level int
|
||||
num uint64
|
||||
}
|
||||
|
||||
type sessionRecord struct {
|
||||
hasRec int
|
||||
comparer string
|
||||
journalNum uint64
|
||||
prevJournalNum uint64
|
||||
nextNum uint64
|
||||
seq uint64
|
||||
compactionPointers []cpRecord
|
||||
addedTables []ntRecord
|
||||
deletedTables []dtRecord
|
||||
scratch [binary.MaxVarintLen64]byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *sessionRecord) has(rec int) bool {
|
||||
return p.hasRec&(1<<uint(rec)) != 0
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setComparer(name string) {
|
||||
p.hasRec |= 1 << recComparer
|
||||
p.comparer = name
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setJournalNum(num uint64) {
|
||||
p.hasRec |= 1 << recJournalNum
|
||||
p.journalNum = num
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setPrevJournalNum(num uint64) {
|
||||
p.hasRec |= 1 << recPrevJournalNum
|
||||
p.prevJournalNum = num
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setNextNum(num uint64) {
|
||||
p.hasRec |= 1 << recNextNum
|
||||
p.nextNum = num
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setSeq(seq uint64) {
|
||||
p.hasRec |= 1 << recSeq
|
||||
p.seq = seq
|
||||
}
|
||||
|
||||
func (p *sessionRecord) addCompactionPointer(level int, key iKey) {
|
||||
p.hasRec |= 1 << recCompactionPointer
|
||||
p.compactionPointers = append(p.compactionPointers, cpRecord{level, key})
|
||||
}
|
||||
|
||||
func (p *sessionRecord) resetCompactionPointers() {
|
||||
p.hasRec &= ^(1 << recCompactionPointer)
|
||||
p.compactionPointers = p.compactionPointers[:0]
|
||||
}
|
||||
|
||||
func (p *sessionRecord) addTable(level int, num, size uint64, min, max iKey) {
|
||||
p.hasRec |= 1 << recNewTable
|
||||
p.addedTables = append(p.addedTables, ntRecord{level, num, size, min, max})
|
||||
}
|
||||
|
||||
func (p *sessionRecord) addTableFile(level int, t *tFile) {
|
||||
p.addTable(level, t.file.Num(), t.size, t.min, t.max)
|
||||
}
|
||||
|
||||
func (p *sessionRecord) resetAddedTables() {
|
||||
p.hasRec &= ^(1 << recNewTable)
|
||||
p.addedTables = p.addedTables[:0]
|
||||
}
|
||||
|
||||
func (p *sessionRecord) deleteTable(level int, num uint64) {
|
||||
p.hasRec |= 1 << recDeletedTable
|
||||
p.deletedTables = append(p.deletedTables, dtRecord{level, num})
|
||||
}
|
||||
|
||||
func (p *sessionRecord) resetDeletedTables() {
|
||||
p.hasRec &= ^(1 << recDeletedTable)
|
||||
p.deletedTables = p.deletedTables[:0]
|
||||
}
|
||||
|
||||
func (p *sessionRecord) putUvarint(w io.Writer, x uint64) {
|
||||
if p.err != nil {
|
||||
return
|
||||
}
|
||||
n := binary.PutUvarint(p.scratch[:], x)
|
||||
_, p.err = w.Write(p.scratch[:n])
|
||||
}
|
||||
|
||||
func (p *sessionRecord) putBytes(w io.Writer, x []byte) {
|
||||
if p.err != nil {
|
||||
return
|
||||
}
|
||||
p.putUvarint(w, uint64(len(x)))
|
||||
if p.err != nil {
|
||||
return
|
||||
}
|
||||
_, p.err = w.Write(x)
|
||||
}
|
||||
|
||||
func (p *sessionRecord) encode(w io.Writer) error {
|
||||
p.err = nil
|
||||
if p.has(recComparer) {
|
||||
p.putUvarint(w, recComparer)
|
||||
p.putBytes(w, []byte(p.comparer))
|
||||
}
|
||||
if p.has(recJournalNum) {
|
||||
p.putUvarint(w, recJournalNum)
|
||||
p.putUvarint(w, p.journalNum)
|
||||
}
|
||||
if p.has(recNextNum) {
|
||||
p.putUvarint(w, recNextNum)
|
||||
p.putUvarint(w, p.nextNum)
|
||||
}
|
||||
if p.has(recSeq) {
|
||||
p.putUvarint(w, recSeq)
|
||||
p.putUvarint(w, p.seq)
|
||||
}
|
||||
for _, cp := range p.compactionPointers {
|
||||
p.putUvarint(w, recCompactionPointer)
|
||||
p.putUvarint(w, uint64(cp.level))
|
||||
p.putBytes(w, cp.key)
|
||||
}
|
||||
for _, t := range p.deletedTables {
|
||||
p.putUvarint(w, recDeletedTable)
|
||||
p.putUvarint(w, uint64(t.level))
|
||||
p.putUvarint(w, t.num)
|
||||
}
|
||||
for _, t := range p.addedTables {
|
||||
p.putUvarint(w, recNewTable)
|
||||
p.putUvarint(w, uint64(t.level))
|
||||
p.putUvarint(w, t.num)
|
||||
p.putUvarint(w, t.size)
|
||||
p.putBytes(w, t.min)
|
||||
p.putBytes(w, t.max)
|
||||
}
|
||||
return p.err
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 {
|
||||
if p.err != nil {
|
||||
return 0
|
||||
}
|
||||
x, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
p.err = errCorruptManifest
|
||||
} else {
|
||||
p.err = err
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readBytes(r byteReader) []byte {
|
||||
if p.err != nil {
|
||||
return nil
|
||||
}
|
||||
n := p.readUvarint(r)
|
||||
if p.err != nil {
|
||||
return nil
|
||||
}
|
||||
x := make([]byte, n)
|
||||
_, p.err = io.ReadFull(r, x)
|
||||
if p.err != nil {
|
||||
if p.err == io.EOF {
|
||||
p.err = errCorruptManifest
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readLevel(r io.ByteReader) int {
|
||||
if p.err != nil {
|
||||
return 0
|
||||
}
|
||||
x := p.readUvarint(r)
|
||||
if p.err != nil {
|
||||
return 0
|
||||
}
|
||||
if x >= kNumLevels {
|
||||
p.err = errCorruptManifest
|
||||
return 0
|
||||
}
|
||||
return int(x)
|
||||
}
|
||||
|
||||
func (p *sessionRecord) decode(r io.Reader) error {
|
||||
br, ok := r.(byteReader)
|
||||
if !ok {
|
||||
br = bufio.NewReader(r)
|
||||
}
|
||||
p.err = nil
|
||||
for p.err == nil {
|
||||
rec, err := binary.ReadUvarint(br)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
switch rec {
|
||||
case recComparer:
|
||||
x := p.readBytes(br)
|
||||
if p.err == nil {
|
||||
p.setComparer(string(x))
|
||||
}
|
||||
case recJournalNum:
|
||||
x := p.readUvarint(br)
|
||||
if p.err == nil {
|
||||
p.setJournalNum(x)
|
||||
}
|
||||
case recPrevJournalNum:
|
||||
x := p.readUvarint(br)
|
||||
if p.err == nil {
|
||||
p.setPrevJournalNum(x)
|
||||
}
|
||||
case recNextNum:
|
||||
x := p.readUvarint(br)
|
||||
if p.err == nil {
|
||||
p.setNextNum(x)
|
||||
}
|
||||
case recSeq:
|
||||
x := p.readUvarint(br)
|
||||
if p.err == nil {
|
||||
p.setSeq(x)
|
||||
}
|
||||
case recCompactionPointer:
|
||||
level := p.readLevel(br)
|
||||
key := p.readBytes(br)
|
||||
if p.err == nil {
|
||||
p.addCompactionPointer(level, iKey(key))
|
||||
}
|
||||
case recNewTable:
|
||||
level := p.readLevel(br)
|
||||
num := p.readUvarint(br)
|
||||
size := p.readUvarint(br)
|
||||
min := p.readBytes(br)
|
||||
max := p.readBytes(br)
|
||||
if p.err == nil {
|
||||
p.addTable(level, num, size, min, max)
|
||||
}
|
||||
case recDeletedTable:
|
||||
level := p.readLevel(br)
|
||||
num := p.readUvarint(br)
|
||||
if p.err == nil {
|
||||
p.deleteTable(level, num)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return p.err
|
||||
}
|
||||
-62
@@ -1,62 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func decodeEncode(v *sessionRecord) (res bool, err error) {
|
||||
b := new(bytes.Buffer)
|
||||
err = v.encode(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
v2 := new(sessionRecord)
|
||||
err = v.decode(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b2 := new(bytes.Buffer)
|
||||
err = v2.encode(b2)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return bytes.Equal(b.Bytes(), b2.Bytes()), nil
|
||||
}
|
||||
|
||||
func TestSessionRecord_EncodeDecode(t *testing.T) {
|
||||
big := uint64(1) << 50
|
||||
v := new(sessionRecord)
|
||||
i := uint64(0)
|
||||
test := func() {
|
||||
res, err := decodeEncode(v)
|
||||
if err != nil {
|
||||
t.Fatalf("error when testing encode/decode sessionRecord: %v", err)
|
||||
}
|
||||
if !res {
|
||||
t.Error("encode/decode test failed at iteration:", i)
|
||||
}
|
||||
}
|
||||
|
||||
for ; i < 4; i++ {
|
||||
test()
|
||||
v.addTable(3, big+300+i, big+400+i,
|
||||
newIKey([]byte("foo"), big+500+1, tVal),
|
||||
newIKey([]byte("zoo"), big+600+1, tDel))
|
||||
v.deleteTable(4, big+700+i)
|
||||
v.addCompactionPointer(int(i), newIKey([]byte("x"), big+900+1, tVal))
|
||||
}
|
||||
|
||||
v.setComparer("foo")
|
||||
v.setJournalNum(big + 100)
|
||||
v.setPrevJournalNum(big + 99)
|
||||
v.setNextNum(big + 200)
|
||||
v.setSeq(big + 1000)
|
||||
test()
|
||||
}
|
||||
-253
@@ -1,253 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/journal"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
// logging
|
||||
|
||||
type dropper struct {
|
||||
s *session
|
||||
file storage.File
|
||||
}
|
||||
|
||||
func (d dropper) Drop(err error) {
|
||||
if e, ok := err.(journal.DroppedError); ok {
|
||||
d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason)
|
||||
} else {
|
||||
d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *session) log(v ...interface{}) {
|
||||
s.stor.Log(fmt.Sprint(v...))
|
||||
}
|
||||
|
||||
func (s *session) logf(format string, v ...interface{}) {
|
||||
s.stor.Log(fmt.Sprintf(format, v...))
|
||||
}
|
||||
|
||||
// file utils
|
||||
|
||||
func (s *session) getJournalFile(num uint64) storage.File {
|
||||
return s.stor.GetFile(num, storage.TypeJournal)
|
||||
}
|
||||
|
||||
func (s *session) getTableFile(num uint64) storage.File {
|
||||
return s.stor.GetFile(num, storage.TypeTable)
|
||||
}
|
||||
|
||||
func (s *session) getFiles(t storage.FileType) ([]storage.File, error) {
|
||||
return s.stor.GetFiles(t)
|
||||
}
|
||||
|
||||
func (s *session) newTemp() storage.File {
|
||||
num := atomic.AddUint64(&s.stTempFileNum, 1) - 1
|
||||
return s.stor.GetFile(num, storage.TypeTemp)
|
||||
}
|
||||
|
||||
// session state
|
||||
|
||||
// Get current version.
|
||||
func (s *session) version() *version {
|
||||
s.vmu.Lock()
|
||||
defer s.vmu.Unlock()
|
||||
s.stVersion.ref++
|
||||
return s.stVersion
|
||||
}
|
||||
|
||||
// Get current version; no barrier.
|
||||
func (s *session) version_NB() *version {
|
||||
return s.stVersion
|
||||
}
|
||||
|
||||
// Set current version to v.
|
||||
func (s *session) setVersion(v *version) {
|
||||
s.vmu.Lock()
|
||||
v.ref = 1
|
||||
if old := s.stVersion; old != nil {
|
||||
v.ref++
|
||||
old.next = v
|
||||
old.release_NB()
|
||||
}
|
||||
s.stVersion = v
|
||||
s.vmu.Unlock()
|
||||
}
|
||||
|
||||
// Get current unused file number.
|
||||
func (s *session) fileNum() uint64 {
|
||||
return atomic.LoadUint64(&s.stFileNum)
|
||||
}
|
||||
|
||||
// Get current unused file number to num.
|
||||
func (s *session) setFileNum(num uint64) {
|
||||
atomic.StoreUint64(&s.stFileNum, num)
|
||||
}
|
||||
|
||||
// Mark file number as used.
|
||||
func (s *session) markFileNum(num uint64) {
|
||||
num += 1
|
||||
for {
|
||||
old, x := s.stFileNum, num
|
||||
if old > x {
|
||||
x = old
|
||||
}
|
||||
if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate a file number.
|
||||
func (s *session) allocFileNum() (num uint64) {
|
||||
return atomic.AddUint64(&s.stFileNum, 1) - 1
|
||||
}
|
||||
|
||||
// Reuse given file number.
|
||||
func (s *session) reuseFileNum(num uint64) {
|
||||
for {
|
||||
old, x := s.stFileNum, num
|
||||
if old != x+1 {
|
||||
x = old
|
||||
}
|
||||
if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// manifest related utils
|
||||
|
||||
// Fill given session record obj with current states; need external
|
||||
// synchronization.
|
||||
func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
|
||||
r.setNextNum(s.fileNum())
|
||||
|
||||
if snapshot {
|
||||
if !r.has(recJournalNum) {
|
||||
r.setJournalNum(s.stJournalNum)
|
||||
}
|
||||
|
||||
if !r.has(recSeq) {
|
||||
r.setSeq(s.stSeq)
|
||||
}
|
||||
|
||||
for level, ik := range s.stCPtrs {
|
||||
if ik != nil {
|
||||
r.addCompactionPointer(level, ik)
|
||||
}
|
||||
}
|
||||
|
||||
r.setComparer(s.icmp.uName())
|
||||
}
|
||||
}
|
||||
|
||||
// Mark if record has been commited, this will update session state;
|
||||
// need external synchronization.
|
||||
func (s *session) recordCommited(r *sessionRecord) {
|
||||
if r.has(recJournalNum) {
|
||||
s.stJournalNum = r.journalNum
|
||||
}
|
||||
|
||||
if r.has(recPrevJournalNum) {
|
||||
s.stPrevJournalNum = r.prevJournalNum
|
||||
}
|
||||
|
||||
if r.has(recSeq) {
|
||||
s.stSeq = r.seq
|
||||
}
|
||||
|
||||
for _, p := range r.compactionPointers {
|
||||
s.stCPtrs[p.level] = iKey(p.key)
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new manifest file; need external synchronization.
|
||||
func (s *session) newManifest(rec *sessionRecord, v *version) (err error) {
|
||||
num := s.allocFileNum()
|
||||
file := s.stor.GetFile(num, storage.TypeManifest)
|
||||
writer, err := file.Create()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
jw := journal.NewWriter(writer)
|
||||
|
||||
if v == nil {
|
||||
v = s.version_NB()
|
||||
}
|
||||
if rec == nil {
|
||||
rec = new(sessionRecord)
|
||||
}
|
||||
s.fillRecord(rec, true)
|
||||
v.fillRecord(rec)
|
||||
|
||||
defer func() {
|
||||
if err == nil {
|
||||
s.recordCommited(rec)
|
||||
if s.manifest != nil {
|
||||
s.manifest.Close()
|
||||
}
|
||||
if s.manifestWriter != nil {
|
||||
s.manifestWriter.Close()
|
||||
}
|
||||
if s.manifestFile != nil {
|
||||
s.manifestFile.Remove()
|
||||
}
|
||||
s.manifestFile = file
|
||||
s.manifestWriter = writer
|
||||
s.manifest = jw
|
||||
} else {
|
||||
writer.Close()
|
||||
file.Remove()
|
||||
s.reuseFileNum(num)
|
||||
}
|
||||
}()
|
||||
|
||||
w, err := jw.Next()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = rec.encode(w)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = jw.Flush()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = s.stor.SetManifest(file)
|
||||
return
|
||||
}
|
||||
|
||||
// Flush record to disk.
|
||||
func (s *session) flushManifest(rec *sessionRecord) (err error) {
|
||||
s.fillRecord(rec, false)
|
||||
w, err := s.manifest.Next()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = rec.encode(w)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = s.manifest.Flush()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = s.manifestWriter.Sync()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.recordCommited(rec)
|
||||
return
|
||||
}
|
||||
Generated
Vendored
-536
@@ -1,536 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reservefs.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var errFileOpen = errors.New("leveldb/storage: file still open")
|
||||
|
||||
type fileLock interface {
|
||||
release() error
|
||||
}
|
||||
|
||||
type fileStorageLock struct {
|
||||
fs *fileStorage
|
||||
}
|
||||
|
||||
func (lock *fileStorageLock) Release() {
|
||||
fs := lock.fs
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.slock == lock {
|
||||
fs.slock = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// fileStorage is a file-system backed storage.
|
||||
type fileStorage struct {
|
||||
path string
|
||||
|
||||
mu sync.Mutex
|
||||
flock fileLock
|
||||
slock *fileStorageLock
|
||||
logw *os.File
|
||||
buf []byte
|
||||
// Opened file counter; if open < 0 means closed.
|
||||
open int
|
||||
day int
|
||||
}
|
||||
|
||||
// OpenFile returns a new filesytem-backed storage implementation with the given
|
||||
// path. This also hold a file lock, so any subsequent attempt to open the same
|
||||
// path will fail.
|
||||
//
|
||||
// The storage must be closed after use, by calling Close method.
|
||||
func OpenFile(path string) (Storage, error) {
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
flock, err := newFileLock(filepath.Join(path, "LOCK"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
flock.release()
|
||||
}
|
||||
}()
|
||||
|
||||
rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
|
||||
logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fs := &fileStorage{path: path, flock: flock, logw: logw}
|
||||
runtime.SetFinalizer(fs, (*fileStorage).Close)
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Lock() (util.Releaser, error) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
if fs.slock != nil {
|
||||
return nil, ErrLocked
|
||||
}
|
||||
fs.slock = &fileStorageLock{fs: fs}
|
||||
return fs.slock, nil
|
||||
}
|
||||
|
||||
func itoa(buf []byte, i int, wid int) []byte {
|
||||
var u uint = uint(i)
|
||||
if u == 0 && wid <= 1 {
|
||||
return append(buf, '0')
|
||||
}
|
||||
|
||||
// Assemble decimal in reverse order.
|
||||
var b [32]byte
|
||||
bp := len(b)
|
||||
for ; u > 0 || wid > 0; u /= 10 {
|
||||
bp--
|
||||
wid--
|
||||
b[bp] = byte(u%10) + '0'
|
||||
}
|
||||
return append(buf, b[bp:]...)
|
||||
}
|
||||
|
||||
func (fs *fileStorage) printDay(t time.Time) {
|
||||
if fs.day == t.Day() {
|
||||
return
|
||||
}
|
||||
fs.day = t.Day()
|
||||
fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
|
||||
}
|
||||
|
||||
func (fs *fileStorage) doLog(t time.Time, str string) {
|
||||
fs.printDay(t)
|
||||
hour, min, sec := t.Clock()
|
||||
msec := t.Nanosecond() / 1e3
|
||||
// time
|
||||
fs.buf = itoa(fs.buf[:0], hour, 2)
|
||||
fs.buf = append(fs.buf, ':')
|
||||
fs.buf = itoa(fs.buf, min, 2)
|
||||
fs.buf = append(fs.buf, ':')
|
||||
fs.buf = itoa(fs.buf, sec, 2)
|
||||
fs.buf = append(fs.buf, '.')
|
||||
fs.buf = itoa(fs.buf, msec, 6)
|
||||
fs.buf = append(fs.buf, ' ')
|
||||
// write
|
||||
fs.buf = append(fs.buf, []byte(str)...)
|
||||
fs.buf = append(fs.buf, '\n')
|
||||
fs.logw.Write(fs.buf)
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Log(str string) {
|
||||
t := time.Now()
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return
|
||||
}
|
||||
fs.doLog(t, str)
|
||||
}
|
||||
|
||||
func (fs *fileStorage) log(str string) {
|
||||
fs.doLog(time.Now(), str)
|
||||
}
|
||||
|
||||
func (fs *fileStorage) GetFile(num uint64, t FileType) File {
|
||||
return &file{fs: fs, num: num, t: t}
|
||||
}
|
||||
|
||||
func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
dir, err := os.Open(fs.path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fnn, err := dir.Readdirnames(0)
|
||||
// Close the dir first before checking for Readdirnames error.
|
||||
if err := dir.Close(); err != nil {
|
||||
fs.log(fmt.Sprintf("close dir: %v", err))
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
f := &file{fs: fs}
|
||||
for _, fn := range fnn {
|
||||
if f.parse(fn) && (f.t&t) != 0 {
|
||||
ff = append(ff, f)
|
||||
f = &file{fs: fs}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *fileStorage) GetManifest() (f File, err error) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
dir, err := os.Open(fs.path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fnn, err := dir.Readdirnames(0)
|
||||
// Close the dir first before checking for Readdirnames error.
|
||||
if err := dir.Close(); err != nil {
|
||||
fs.log(fmt.Sprintf("close dir: %v", err))
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Find latest CURRENT file.
|
||||
var rem []string
|
||||
var pend bool
|
||||
var cerr error
|
||||
for _, fn := range fnn {
|
||||
if strings.HasPrefix(fn, "CURRENT") {
|
||||
pend1 := len(fn) > 7
|
||||
// Make sure it is valid name for a CURRENT file, otherwise skip it.
|
||||
if pend1 {
|
||||
if fn[7] != '.' || len(fn) < 9 {
|
||||
fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
|
||||
continue
|
||||
}
|
||||
if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
|
||||
fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
|
||||
continue
|
||||
}
|
||||
}
|
||||
path := filepath.Join(fs.path, fn)
|
||||
r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
|
||||
if e1 != nil {
|
||||
return nil, e1
|
||||
}
|
||||
b, e1 := ioutil.ReadAll(r)
|
||||
if e1 != nil {
|
||||
r.Close()
|
||||
return nil, e1
|
||||
}
|
||||
f1 := &file{fs: fs}
|
||||
if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
|
||||
fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
|
||||
if pend1 {
|
||||
rem = append(rem, fn)
|
||||
}
|
||||
if !pend1 || cerr == nil {
|
||||
cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
|
||||
}
|
||||
} else if f != nil && f1.Num() < f.Num() {
|
||||
fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
|
||||
if pend1 {
|
||||
rem = append(rem, fn)
|
||||
}
|
||||
} else {
|
||||
f = f1
|
||||
pend = pend1
|
||||
}
|
||||
if err := r.Close(); err != nil {
|
||||
fs.log(fmt.Sprintf("close %s: %v", fn, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
// Don't remove any files if there is no valid CURRENT file.
|
||||
if f == nil {
|
||||
if cerr != nil {
|
||||
err = cerr
|
||||
} else {
|
||||
err = os.ErrNotExist
|
||||
}
|
||||
return
|
||||
}
|
||||
// Rename pending CURRENT file to an effective CURRENT.
|
||||
if pend {
|
||||
path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
|
||||
if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
|
||||
fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", f.Num(), err))
|
||||
}
|
||||
}
|
||||
// Remove obsolete or incomplete pending CURRENT files.
|
||||
for _, fn := range rem {
|
||||
path := filepath.Join(fs.path, fn)
|
||||
if err := os.Remove(path); err != nil {
|
||||
fs.log(fmt.Sprintf("remove %s: %v", fn, err))
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *fileStorage) SetManifest(f File) (err error) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
f2, ok := f.(*file)
|
||||
if !ok || f2.t != TypeManifest {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
fs.log(fmt.Sprintf("CURRENT: %v", err))
|
||||
}
|
||||
}()
|
||||
path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
|
||||
w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintln(w, f2.name())
|
||||
// Close the file first.
|
||||
if err := w.Close(); err != nil {
|
||||
fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return rename(path, filepath.Join(fs.path, "CURRENT"))
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Close() error {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
// Clear the finalizer.
|
||||
runtime.SetFinalizer(fs, nil)
|
||||
|
||||
if fs.open > 0 {
|
||||
fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
|
||||
return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
|
||||
}
|
||||
fs.open = -1
|
||||
e1 := fs.logw.Close()
|
||||
err := fs.flock.release()
|
||||
if err == nil {
|
||||
err = e1
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type fileWrap struct {
|
||||
*os.File
|
||||
f *file
|
||||
}
|
||||
|
||||
func (fw fileWrap) Sync() error {
|
||||
if fw.f.Type() == TypeManifest {
|
||||
// Also sync parent directory if file type is manifest.
|
||||
// See: https://code.google.com/p/leveldb/issues/detail?id=190.
|
||||
f, err := os.Open(fw.f.fs.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if err := f.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fw.File.Sync()
|
||||
}
|
||||
|
||||
func (fw fileWrap) Close() error {
|
||||
f := fw.f
|
||||
f.fs.mu.Lock()
|
||||
defer f.fs.mu.Unlock()
|
||||
if !f.open {
|
||||
return ErrClosed
|
||||
}
|
||||
f.open = false
|
||||
f.fs.open--
|
||||
err := fw.File.Close()
|
||||
if err != nil {
|
||||
f.fs.log(fmt.Sprintf("close %s.%d: %v", f.Type(), f.Num(), err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type file struct {
|
||||
fs *fileStorage
|
||||
num uint64
|
||||
t FileType
|
||||
open bool
|
||||
}
|
||||
|
||||
func (f *file) Open() (Reader, error) {
|
||||
f.fs.mu.Lock()
|
||||
defer f.fs.mu.Unlock()
|
||||
if f.fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
if f.open {
|
||||
return nil, errFileOpen
|
||||
}
|
||||
of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
if f.hasOldName() && os.IsNotExist(err) {
|
||||
of, err = os.OpenFile(f.oldPath(), os.O_RDONLY, 0)
|
||||
if err == nil {
|
||||
goto ok
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
ok:
|
||||
f.open = true
|
||||
f.fs.open++
|
||||
return fileWrap{of, f}, nil
|
||||
}
|
||||
|
||||
func (f *file) Create() (Writer, error) {
|
||||
f.fs.mu.Lock()
|
||||
defer f.fs.mu.Unlock()
|
||||
if f.fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
if f.open {
|
||||
return nil, errFileOpen
|
||||
}
|
||||
of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.open = true
|
||||
f.fs.open++
|
||||
return fileWrap{of, f}, nil
|
||||
}
|
||||
|
||||
func (f *file) Replace(newfile File) error {
|
||||
f.fs.mu.Lock()
|
||||
defer f.fs.mu.Unlock()
|
||||
if f.fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
newfile2, ok := newfile.(*file)
|
||||
if !ok {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
if f.open || newfile2.open {
|
||||
return errFileOpen
|
||||
}
|
||||
return rename(newfile2.path(), f.path())
|
||||
}
|
||||
|
||||
func (f *file) Type() FileType {
|
||||
return f.t
|
||||
}
|
||||
|
||||
func (f *file) Num() uint64 {
|
||||
return f.num
|
||||
}
|
||||
|
||||
func (f *file) Remove() error {
|
||||
f.fs.mu.Lock()
|
||||
defer f.fs.mu.Unlock()
|
||||
if f.fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
if f.open {
|
||||
return errFileOpen
|
||||
}
|
||||
err := os.Remove(f.path())
|
||||
if err != nil {
|
||||
f.fs.log(fmt.Sprintf("remove %s.%d: %v", f.Type(), f.Num(), err))
|
||||
}
|
||||
// Also try remove file with old name, just in case.
|
||||
if f.hasOldName() {
|
||||
if e1 := os.Remove(f.oldPath()); !os.IsNotExist(e1) {
|
||||
f.fs.log(fmt.Sprintf("remove %s.%d: %v (old name)", f.Type(), f.Num(), err))
|
||||
err = e1
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *file) hasOldName() bool {
|
||||
return f.t == TypeTable
|
||||
}
|
||||
|
||||
func (f *file) oldName() string {
|
||||
switch f.t {
|
||||
case TypeTable:
|
||||
return fmt.Sprintf("%06d.sst", f.num)
|
||||
}
|
||||
return f.name()
|
||||
}
|
||||
|
||||
func (f *file) oldPath() string {
|
||||
return filepath.Join(f.fs.path, f.oldName())
|
||||
}
|
||||
|
||||
func (f *file) name() string {
|
||||
switch f.t {
|
||||
case TypeManifest:
|
||||
return fmt.Sprintf("MANIFEST-%06d", f.num)
|
||||
case TypeJournal:
|
||||
return fmt.Sprintf("%06d.log", f.num)
|
||||
case TypeTable:
|
||||
return fmt.Sprintf("%06d.ldb", f.num)
|
||||
case TypeTemp:
|
||||
return fmt.Sprintf("%06d.tmp", f.num)
|
||||
default:
|
||||
panic("invalid file type")
|
||||
}
|
||||
}
|
||||
|
||||
func (f *file) path() string {
|
||||
return filepath.Join(f.fs.path, f.name())
|
||||
}
|
||||
|
||||
func (f *file) parse(name string) bool {
|
||||
var num uint64
|
||||
var tail string
|
||||
_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
|
||||
if err == nil {
|
||||
switch tail {
|
||||
case "log":
|
||||
f.t = TypeJournal
|
||||
case "ldb", "sst":
|
||||
f.t = TypeTable
|
||||
case "tmp":
|
||||
f.t = TypeTemp
|
||||
default:
|
||||
return false
|
||||
}
|
||||
f.num = num
|
||||
return true
|
||||
}
|
||||
n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
|
||||
if n == 1 {
|
||||
f.t = TypeManifest
|
||||
f.num = num
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
Generated
Vendored
-40
@@ -1,40 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type plan9FileLock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (fl *plan9FileLock) release() error {
|
||||
return fl.f.Close()
|
||||
}
|
||||
|
||||
func newFileLock(path string) (fl fileLock, err error) {
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fl = &plan9FileLock{f: f}
|
||||
return
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
if _, err := os.Stat(newpath); err == nil {
|
||||
if err := os.Remove(newpath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, fname := filepath.Split(newpath)
|
||||
return os.Rename(oldpath, fname)
|
||||
}
|
||||
Generated
Vendored
-142
@@ -1,142 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var cases = []struct {
|
||||
oldName []string
|
||||
name string
|
||||
ftype FileType
|
||||
num uint64
|
||||
}{
|
||||
{nil, "000100.log", TypeJournal, 100},
|
||||
{nil, "000000.log", TypeJournal, 0},
|
||||
{[]string{"000000.sst"}, "000000.ldb", TypeTable, 0},
|
||||
{nil, "MANIFEST-000002", TypeManifest, 2},
|
||||
{nil, "MANIFEST-000007", TypeManifest, 7},
|
||||
{nil, "18446744073709551615.log", TypeJournal, 18446744073709551615},
|
||||
{nil, "000100.tmp", TypeTemp, 100},
|
||||
}
|
||||
|
||||
var invalidCases = []string{
|
||||
"",
|
||||
"foo",
|
||||
"foo-dx-100.log",
|
||||
".log",
|
||||
"",
|
||||
"manifest",
|
||||
"CURREN",
|
||||
"CURRENTX",
|
||||
"MANIFES",
|
||||
"MANIFEST",
|
||||
"MANIFEST-",
|
||||
"XMANIFEST-3",
|
||||
"MANIFEST-3x",
|
||||
"LOC",
|
||||
"LOCKx",
|
||||
"LO",
|
||||
"LOGx",
|
||||
"18446744073709551616.log",
|
||||
"184467440737095516150.log",
|
||||
"100",
|
||||
"100.",
|
||||
"100.lop",
|
||||
}
|
||||
|
||||
func TestFileStorage_CreateFileName(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
f := &file{num: c.num, t: c.ftype}
|
||||
if f.name() != c.name {
|
||||
t.Errorf("invalid filename got '%s', want '%s'", f.name(), c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStorage_ParseFileName(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
for _, name := range append([]string{c.name}, c.oldName...) {
|
||||
f := new(file)
|
||||
if !f.parse(name) {
|
||||
t.Errorf("cannot parse filename '%s'", name)
|
||||
continue
|
||||
}
|
||||
if f.Type() != c.ftype {
|
||||
t.Errorf("filename '%s' invalid type got '%d', want '%d'", name, f.Type(), c.ftype)
|
||||
}
|
||||
if f.Num() != c.num {
|
||||
t.Errorf("filename '%s' invalid number got '%d', want '%d'", name, f.Num(), c.num)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStorage_InvalidFileName(t *testing.T) {
|
||||
for _, name := range invalidCases {
|
||||
f := new(file)
|
||||
if f.parse(name) {
|
||||
t.Errorf("filename '%s' should be invalid", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStorage_Locking(t *testing.T) {
|
||||
path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestfd-%d", os.Getuid()))
|
||||
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
err = os.RemoveAll(path)
|
||||
if err != nil {
|
||||
t.Fatal("RemoveAll: got error: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
p1, err := OpenFile(path)
|
||||
if err != nil {
|
||||
t.Fatal("OpenFile(1): got error: ", err)
|
||||
}
|
||||
|
||||
defer os.RemoveAll(path)
|
||||
|
||||
p2, err := OpenFile(path)
|
||||
if err != nil {
|
||||
t.Logf("OpenFile(2): got error: %s (expected)", err)
|
||||
} else {
|
||||
p2.Close()
|
||||
p1.Close()
|
||||
t.Fatal("OpenFile(2): expect error")
|
||||
}
|
||||
|
||||
p1.Close()
|
||||
|
||||
p3, err := OpenFile(path)
|
||||
if err != nil {
|
||||
t.Fatal("OpenFile(3): got error: ", err)
|
||||
}
|
||||
defer p3.Close()
|
||||
|
||||
l, err := p3.Lock()
|
||||
if err != nil {
|
||||
t.Fatal("storage lock failed(1): ", err)
|
||||
}
|
||||
_, err = p3.Lock()
|
||||
if err == nil {
|
||||
t.Fatal("expect error for second storage lock attempt")
|
||||
} else {
|
||||
t.Logf("storage lock got error: %s (expected)", err)
|
||||
}
|
||||
l.Release()
|
||||
_, err = p3.Lock()
|
||||
if err != nil {
|
||||
t.Fatal("storage lock failed(2): ", err)
|
||||
}
|
||||
}
|
||||
Generated
Vendored
-51
@@ -1,51 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// +build darwin freebsd linux netbsd openbsd
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type unixFileLock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (fl *unixFileLock) release() error {
|
||||
if err := setFileLock(fl.f, false); err != nil {
|
||||
return err
|
||||
}
|
||||
return fl.f.Close()
|
||||
}
|
||||
|
||||
func newFileLock(path string) (fl fileLock, err error) {
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = setFileLock(f, true)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return
|
||||
}
|
||||
fl = &unixFileLock{f: f}
|
||||
return
|
||||
}
|
||||
|
||||
func setFileLock(f *os.File, lock bool) error {
|
||||
how := syscall.LOCK_UN
|
||||
if lock {
|
||||
how = syscall.LOCK_EX
|
||||
}
|
||||
return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
return os.Rename(oldpath, newpath)
|
||||
}
|
||||
Generated
Vendored
-67
@@ -1,67 +0,0 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var (
|
||||
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
|
||||
|
||||
procMoveFileExW = modkernel32.NewProc("MoveFileExW")
|
||||
)
|
||||
|
||||
const (
|
||||
_MOVEFILE_REPLACE_EXISTING = 1
|
||||
)
|
||||
|
||||
type windowsFileLock struct {
|
||||
fd syscall.Handle
|
||||
}
|
||||
|
||||
func (fl *windowsFileLock) release() error {
|
||||
return syscall.Close(fl.fd)
|
||||
}
|
||||
|
||||
func newFileLock(path string) (fl fileLock, err error) {
|
||||
pathp, err := syscall.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fd, err := syscall.CreateFile(pathp, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.CREATE_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fl = &windowsFileLock{fd: fd}
|
||||
return
|
||||
}
|
||||
|
||||
func moveFileEx(from *uint16, to *uint16, flags uint32) error {
|
||||
r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags))
|
||||
if r1 == 0 {
|
||||
if e1 != 0 {
|
||||
return error(e1)
|
||||
} else {
|
||||
return syscall.EINVAL
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
from, err := syscall.UTF16PtrFromString(oldpath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
to, err := syscall.UTF16PtrFromString(newpath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING)
|
||||
}
|
||||
-203
@@ -1,203 +0,0 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
const typeShift = 3
|
||||
|
||||
type memStorageLock struct {
|
||||
ms *memStorage
|
||||
}
|
||||
|
||||
func (lock *memStorageLock) Release() {
|
||||
ms := lock.ms
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if ms.slock == lock {
|
||||
ms.slock = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// memStorage is a memory-backed storage.
|
||||
type memStorage struct {
|
||||
mu sync.Mutex
|
||||
slock *memStorageLock
|
||||
files map[uint64]*memFile
|
||||
manifest *memFilePtr
|
||||
}
|
||||
|
||||
// NewMemStorage returns a new memory-backed storage implementation.
|
||||
func NewMemStorage() Storage {
|
||||
return &memStorage{
|
||||
files: make(map[uint64]*memFile),
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *memStorage) Lock() (util.Releaser, error) {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if ms.slock != nil {
|
||||
return nil, ErrLocked
|
||||
}
|
||||
ms.slock = &memStorageLock{ms: ms}
|
||||
return ms.slock, nil
|
||||
}
|
||||
|
||||
func (*memStorage) Log(str string) {}
|
||||
|
||||
func (ms *memStorage) GetFile(num uint64, t FileType) File {
|
||||
return &memFilePtr{ms: ms, num: num, t: t}
|
||||
}
|
||||
|
||||
func (ms *memStorage) GetFiles(t FileType) ([]File, error) {
|
||||
ms.mu.Lock()
|
||||
var ff []File
|
||||
for x, _ := range ms.files {
|
||||
num, mt := x>>typeShift, FileType(x)&TypeAll
|
||||
if mt&t == 0 {
|
||||
continue
|
||||
}
|
||||
ff = append(ff, &memFilePtr{ms: ms, num: num, t: mt})
|
||||
}
|
||||
ms.mu.Unlock()
|
||||
return ff, nil
|
||||
}
|
||||
|
||||
func (ms *memStorage) GetManifest() (File, error) {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if ms.manifest == nil {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return ms.manifest, nil
|
||||
}
|
||||
|
||||
func (ms *memStorage) SetManifest(f File) error {
|
||||
fm, ok := f.(*memFilePtr)
|
||||
if !ok || fm.t != TypeManifest {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
ms.mu.Lock()
|
||||
ms.manifest = fm
|
||||
ms.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*memStorage) Close() error { return nil }
|
||||
|
||||
type memReader struct {
|
||||
*bytes.Reader
|
||||
m *memFile
|
||||
}
|
||||
|
||||
func (mr *memReader) Close() error {
|
||||
return mr.m.Close()
|
||||
}
|
||||
|
||||
type memFile struct {
|
||||
bytes.Buffer
|
||||
ms *memStorage
|
||||
open bool
|
||||
}
|
||||
|
||||
func (*memFile) Sync() error { return nil }
|
||||
func (m *memFile) Close() error {
|
||||
m.ms.mu.Lock()
|
||||
m.open = false
|
||||
m.ms.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
type memFilePtr struct {
|
||||
ms *memStorage
|
||||
num uint64
|
||||
t FileType
|
||||
}
|
||||
|
||||
func (p *memFilePtr) x() uint64 {
|
||||
return p.Num()<<typeShift | uint64(p.Type())
|
||||
}
|
||||
|
||||
func (p *memFilePtr) Open() (Reader, error) {
|
||||
ms := p.ms
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if m, exist := ms.files[p.x()]; exist {
|
||||
if m.open {
|
||||
return nil, errFileOpen
|
||||
}
|
||||
m.open = true
|
||||
return &memReader{Reader: bytes.NewReader(m.Bytes()), m: m}, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func (p *memFilePtr) Create() (Writer, error) {
|
||||
ms := p.ms
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
m, exist := ms.files[p.x()]
|
||||
if exist {
|
||||
if m.open {
|
||||
return nil, errFileOpen
|
||||
}
|
||||
m.Reset()
|
||||
} else {
|
||||
m = &memFile{ms: ms}
|
||||
ms.files[p.x()] = m
|
||||
}
|
||||
m.open = true
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (p *memFilePtr) Replace(newfile File) error {
|
||||
p1, ok := newfile.(*memFilePtr)
|
||||
if !ok {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
ms := p.ms
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
m1, exist := ms.files[p1.x()]
|
||||
if !exist {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
m0, exist := ms.files[p.x()]
|
||||
if (exist && m0.open) || m1.open {
|
||||
return errFileOpen
|
||||
}
|
||||
delete(ms.files, p1.x())
|
||||
ms.files[p.x()] = m1
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *memFilePtr) Type() FileType {
|
||||
return p.t
|
||||
}
|
||||
|
||||
func (p *memFilePtr) Num() uint64 {
|
||||
return p.num
|
||||
}
|
||||
|
||||
func (p *memFilePtr) Remove() error {
|
||||
ms := p.ms
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if _, exist := ms.files[p.x()]; exist {
|
||||
delete(ms.files, p.x())
|
||||
return nil
|
||||
}
|
||||
return os.ErrNotExist
|
||||
}
|
||||
Generated
Vendored
-66
@@ -1,66 +0,0 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMemStorage(t *testing.T) {
|
||||
m := NewMemStorage()
|
||||
|
||||
l, err := m.Lock()
|
||||
if err != nil {
|
||||
t.Fatal("storage lock failed(1): ", err)
|
||||
}
|
||||
_, err = m.Lock()
|
||||
if err == nil {
|
||||
t.Fatal("expect error for second storage lock attempt")
|
||||
} else {
|
||||
t.Logf("storage lock got error: %s (expected)", err)
|
||||
}
|
||||
l.Release()
|
||||
_, err = m.Lock()
|
||||
if err != nil {
|
||||
t.Fatal("storage lock failed(2): ", err)
|
||||
}
|
||||
|
||||
f := m.GetFile(1, TypeTable)
|
||||
if f.Num() != 1 && f.Type() != TypeTable {
|
||||
t.Fatal("invalid file number and type")
|
||||
}
|
||||
w, _ := f.Create()
|
||||
w.Write([]byte("abc"))
|
||||
w.Close()
|
||||
if ff, _ := m.GetFiles(TypeAll); len(ff) != 1 {
|
||||
t.Fatal("invalid GetFiles len")
|
||||
}
|
||||
buf := new(bytes.Buffer)
|
||||
r, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatal("Open: got error: ", err)
|
||||
}
|
||||
buf.ReadFrom(r)
|
||||
r.Close()
|
||||
if got := buf.String(); got != "abc" {
|
||||
t.Fatalf("Read: invalid value, want=abc got=%s", got)
|
||||
}
|
||||
if _, err := f.Open(); err != nil {
|
||||
t.Fatal("Open: got error: ", err)
|
||||
}
|
||||
if _, err := m.GetFile(1, TypeTable).Open(); err == nil {
|
||||
t.Fatal("expecting error")
|
||||
}
|
||||
f.Remove()
|
||||
if ff, _ := m.GetFiles(TypeAll); len(ff) != 0 {
|
||||
t.Fatal("invalid GetFiles len", len(ff))
|
||||
}
|
||||
if _, err := f.Open(); err == nil {
|
||||
t.Fatal("expecting error")
|
||||
}
|
||||
}
|
||||
-127
@@ -1,127 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package storage provides storage abstraction for LevelDB.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type FileType uint32
|
||||
|
||||
const (
|
||||
TypeManifest FileType = 1 << iota
|
||||
TypeJournal
|
||||
TypeTable
|
||||
TypeTemp
|
||||
|
||||
TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp
|
||||
)
|
||||
|
||||
func (t FileType) String() string {
|
||||
switch t {
|
||||
case TypeManifest:
|
||||
return "manifest"
|
||||
case TypeJournal:
|
||||
return "journal"
|
||||
case TypeTable:
|
||||
return "table"
|
||||
case TypeTemp:
|
||||
return "temp"
|
||||
}
|
||||
return fmt.Sprintf("<unknown:%d>", t)
|
||||
}
|
||||
|
||||
var (
|
||||
ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument")
|
||||
ErrLocked = errors.New("leveldb/storage: already locked")
|
||||
ErrClosed = errors.New("leveldb/storage: closed")
|
||||
)
|
||||
|
||||
// Syncer is the interface that wraps basic Sync method.
|
||||
type Syncer interface {
|
||||
// Sync commits the current contents of the file to stable storage.
|
||||
Sync() error
|
||||
}
|
||||
|
||||
// Reader is the interface that groups the basic Read, Seek, ReadAt and Close
|
||||
// methods.
|
||||
type Reader interface {
|
||||
io.ReadSeeker
|
||||
io.ReaderAt
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Writer is the interface that groups the basic Write, Sync and Close
|
||||
// methods.
|
||||
type Writer interface {
|
||||
io.WriteCloser
|
||||
Syncer
|
||||
}
|
||||
|
||||
// File is the file.
|
||||
type File interface {
|
||||
// Open opens the file for read. Returns os.ErrNotExist error
|
||||
// if the file does not exist.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Open() (r Reader, err error)
|
||||
|
||||
// Create creates the file for writting. Truncate the file if
|
||||
// already exist.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Create() (w Writer, err error)
|
||||
|
||||
// Replace replaces file with newfile.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Replace(newfile File) error
|
||||
|
||||
// Type returns the file type
|
||||
Type() FileType
|
||||
|
||||
// Num returns the file number.
|
||||
Num() uint64
|
||||
|
||||
// Remove removes the file.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Remove() error
|
||||
}
|
||||
|
||||
// Storage is the storage.
|
||||
type Storage interface {
|
||||
// Lock locks the storage. Any subsequent attempt to call Lock will fail
|
||||
// until the last lock released.
|
||||
// After use the caller should call the Release method.
|
||||
Lock() (l util.Releaser, err error)
|
||||
|
||||
// Log logs a string. This is used for logging. An implementation
|
||||
// may write to a file, stdout or simply do nothing.
|
||||
Log(str string)
|
||||
|
||||
// GetFile returns a file for the given number and type. GetFile will never
|
||||
// returns nil, even if the underlying storage is closed.
|
||||
GetFile(num uint64, t FileType) File
|
||||
|
||||
// GetFiles returns a slice of files that match the given file types.
|
||||
// The file types may be OR'ed together.
|
||||
GetFiles(t FileType) ([]File, error)
|
||||
|
||||
// GetManifest returns a manifest file. Returns os.ErrNotExist if manifest
|
||||
// file does not exist.
|
||||
GetManifest() (File, error)
|
||||
|
||||
// SetManifest sets the given file as manifest file. The given file should
|
||||
// be a manifest file type or error will be returned.
|
||||
SetManifest(f File) error
|
||||
|
||||
// Close closes the storage. It is valid to call Close multiple times.
|
||||
// Other methods should not be called after the storage has been closed.
|
||||
Close() error
|
||||
}
|
||||
-459
@@ -1,459 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
const typeShift = 4
|
||||
|
||||
var (
|
||||
tsErrInvalidFile = errors.New("leveldb.testStorage: invalid file for argument")
|
||||
tsErrFileOpen = errors.New("leveldb.testStorage: file still open")
|
||||
)
|
||||
|
||||
var (
|
||||
tsFSEnv = os.Getenv("GOLEVELDB_USEFS")
|
||||
tsKeepFS = tsFSEnv == "2"
|
||||
tsFS = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
|
||||
tsMU = &sync.Mutex{}
|
||||
tsNum = 0
|
||||
)
|
||||
|
||||
type tsLock struct {
|
||||
ts *testStorage
|
||||
r util.Releaser
|
||||
}
|
||||
|
||||
func (l tsLock) Release() {
|
||||
l.r.Release()
|
||||
l.ts.t.Log("I: storage lock released")
|
||||
}
|
||||
|
||||
type tsReader struct {
|
||||
tf tsFile
|
||||
storage.Reader
|
||||
}
|
||||
|
||||
func (tr tsReader) Read(b []byte) (n int, err error) {
|
||||
ts := tr.tf.ts
|
||||
ts.countRead(tr.tf.Type())
|
||||
n, err = tr.Reader.Read(b)
|
||||
if err != nil && err != io.EOF {
|
||||
ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (tr tsReader) ReadAt(b []byte, off int64) (n int, err error) {
|
||||
ts := tr.tf.ts
|
||||
ts.countRead(tr.tf.Type())
|
||||
n, err = tr.Reader.ReadAt(b, off)
|
||||
if err != nil && err != io.EOF {
|
||||
ts.t.Errorf("E: readAt error, num=%d type=%v off=%d n=%d: %v", tr.tf.Num(), tr.tf.Type(), off, n, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (tr tsReader) Close() (err error) {
|
||||
err = tr.Reader.Close()
|
||||
tr.tf.close("reader", err)
|
||||
return
|
||||
}
|
||||
|
||||
type tsWriter struct {
|
||||
tf tsFile
|
||||
storage.Writer
|
||||
}
|
||||
|
||||
func (tw tsWriter) Write(b []byte) (n int, err error) {
|
||||
ts := tw.tf.ts
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
if ts.emuWriteErr&tw.tf.Type() != 0 {
|
||||
return 0, errors.New("leveldb.testStorage: emulated write error")
|
||||
}
|
||||
n, err = tw.Writer.Write(b)
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (tw tsWriter) Sync() (err error) {
|
||||
ts := tw.tf.ts
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
for ts.emuDelaySync&tw.tf.Type() != 0 {
|
||||
ts.cond.Wait()
|
||||
}
|
||||
if ts.emuSyncErr&tw.tf.Type() != 0 {
|
||||
return errors.New("leveldb.testStorage: emulated sync error")
|
||||
}
|
||||
err = tw.Writer.Sync()
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (tw tsWriter) Close() (err error) {
|
||||
err = tw.Writer.Close()
|
||||
tw.tf.close("reader", err)
|
||||
return
|
||||
}
|
||||
|
||||
type tsFile struct {
|
||||
ts *testStorage
|
||||
storage.File
|
||||
}
|
||||
|
||||
func (tf tsFile) x() uint64 {
|
||||
return tf.Num()<<typeShift | uint64(tf.Type())
|
||||
}
|
||||
|
||||
func (tf tsFile) checkOpen(m string) error {
|
||||
ts := tf.ts
|
||||
if writer, ok := ts.opens[tf.x()]; ok {
|
||||
if writer {
|
||||
ts.t.Errorf("E: cannot %s file, num=%d type=%v: a writer still open", m, tf.Num(), tf.Type())
|
||||
} else {
|
||||
ts.t.Errorf("E: cannot %s file, num=%d type=%v: a reader still open", m, tf.Num(), tf.Type())
|
||||
}
|
||||
return tsErrFileOpen
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tf tsFile) close(m string, err error) {
|
||||
ts := tf.ts
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
if _, ok := ts.opens[tf.x()]; !ok {
|
||||
ts.t.Errorf("E: %s: redudant file closing, num=%d type=%v", m, tf.Num(), tf.Type())
|
||||
} else if err == nil {
|
||||
ts.t.Logf("I: %s: file closed, num=%d type=%v", m, tf.Num(), tf.Type())
|
||||
}
|
||||
delete(ts.opens, tf.x())
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: %s: cannot close file, num=%d type=%v: %v", m, tf.Num(), tf.Type(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func (tf tsFile) Open() (r storage.Reader, err error) {
|
||||
ts := tf.ts
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
err = tf.checkOpen("open")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if ts.emuOpenErr&tf.Type() != 0 {
|
||||
err = errors.New("leveldb.testStorage: emulated open error")
|
||||
return
|
||||
}
|
||||
r, err = tf.File.Open()
|
||||
if err != nil {
|
||||
if ts.ignoreOpenErr&tf.Type() != 0 {
|
||||
ts.t.Logf("I: cannot open file, num=%d type=%v: %v (ignored)", tf.Num(), tf.Type(), err)
|
||||
} else {
|
||||
ts.t.Errorf("E: cannot open file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
|
||||
}
|
||||
} else {
|
||||
ts.t.Logf("I: file opened, num=%d type=%v", tf.Num(), tf.Type())
|
||||
ts.opens[tf.x()] = false
|
||||
r = tsReader{tf, r}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (tf tsFile) Create() (w storage.Writer, err error) {
|
||||
ts := tf.ts
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
err = tf.checkOpen("create")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if ts.emuCreateErr&tf.Type() != 0 {
|
||||
err = errors.New("leveldb.testStorage: emulated create error")
|
||||
return
|
||||
}
|
||||
w, err = tf.File.Create()
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: cannot create file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
|
||||
} else {
|
||||
ts.t.Logf("I: file created, num=%d type=%v", tf.Num(), tf.Type())
|
||||
ts.opens[tf.x()] = true
|
||||
w = tsWriter{tf, w}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (tf tsFile) Remove() (err error) {
|
||||
ts := tf.ts
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
err = tf.checkOpen("remove")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = tf.File.Remove()
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
|
||||
} else {
|
||||
ts.t.Logf("I: file removed, num=%d type=%v", tf.Num(), tf.Type())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type testStorage struct {
|
||||
t *testing.T
|
||||
storage.Storage
|
||||
closeFn func() error
|
||||
|
||||
mu sync.Mutex
|
||||
cond sync.Cond
|
||||
// Open files, true=writer, false=reader
|
||||
opens map[uint64]bool
|
||||
emuOpenErr storage.FileType
|
||||
emuCreateErr storage.FileType
|
||||
emuDelaySync storage.FileType
|
||||
emuWriteErr storage.FileType
|
||||
emuSyncErr storage.FileType
|
||||
ignoreOpenErr storage.FileType
|
||||
readCnt uint64
|
||||
readCntEn storage.FileType
|
||||
}
|
||||
|
||||
func (ts *testStorage) SetOpenErr(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
ts.emuOpenErr = t
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) SetCreateErr(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
ts.emuCreateErr = t
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) DelaySync(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
ts.emuDelaySync |= t
|
||||
ts.cond.Broadcast()
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) ReleaseSync(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
ts.emuDelaySync &= ^t
|
||||
ts.cond.Broadcast()
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) SetWriteErr(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
ts.emuWriteErr = t
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) SetSyncErr(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
ts.emuSyncErr = t
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) ReadCounter() uint64 {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
return ts.readCnt
|
||||
}
|
||||
|
||||
func (ts *testStorage) ResetReadCounter() {
|
||||
ts.mu.Lock()
|
||||
ts.readCnt = 0
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) SetReadCounter(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
ts.readCntEn = t
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) countRead(t storage.FileType) {
|
||||
ts.mu.Lock()
|
||||
if ts.readCntEn&t != 0 {
|
||||
ts.readCnt++
|
||||
}
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ts *testStorage) SetIgnoreOpenErr(t storage.FileType) {
|
||||
ts.ignoreOpenErr = t
|
||||
}
|
||||
|
||||
func (ts *testStorage) Lock() (r util.Releaser, err error) {
|
||||
r, err = ts.Storage.Lock()
|
||||
if err != nil {
|
||||
ts.t.Logf("W: storage locking failed: %v", err)
|
||||
} else {
|
||||
ts.t.Log("I: storage locked")
|
||||
r = tsLock{ts, r}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (ts *testStorage) Log(str string) {
|
||||
ts.t.Log("L: " + str)
|
||||
ts.Storage.Log(str)
|
||||
}
|
||||
|
||||
func (ts *testStorage) GetFile(num uint64, t storage.FileType) storage.File {
|
||||
return tsFile{ts, ts.Storage.GetFile(num, t)}
|
||||
}
|
||||
|
||||
func (ts *testStorage) GetFiles(t storage.FileType) (ff []storage.File, err error) {
|
||||
ff0, err := ts.Storage.GetFiles(t)
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: get files failed: %v", err)
|
||||
return
|
||||
}
|
||||
ff = make([]storage.File, len(ff0))
|
||||
for i, f := range ff0 {
|
||||
ff[i] = tsFile{ts, f}
|
||||
}
|
||||
ts.t.Logf("I: get files, type=0x%x count=%d", int(t), len(ff))
|
||||
return
|
||||
}
|
||||
|
||||
func (ts *testStorage) GetManifest() (f storage.File, err error) {
|
||||
f0, err := ts.Storage.GetManifest()
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
ts.t.Errorf("E: get manifest failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
f = tsFile{ts, f0}
|
||||
ts.t.Logf("I: get manifest, num=%d", f.Num())
|
||||
return
|
||||
}
|
||||
|
||||
func (ts *testStorage) SetManifest(f storage.File) error {
|
||||
tf, ok := f.(tsFile)
|
||||
if !ok {
|
||||
ts.t.Error("E: set manifest failed: type assertion failed")
|
||||
return tsErrInvalidFile
|
||||
} else if tf.Type() != storage.TypeManifest {
|
||||
ts.t.Errorf("E: set manifest failed: invalid file type: %s", tf.Type())
|
||||
return tsErrInvalidFile
|
||||
}
|
||||
err := ts.Storage.SetManifest(tf.File)
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: set manifest failed: %v", err)
|
||||
} else {
|
||||
ts.t.Logf("I: set manifest, num=%d", tf.Num())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (ts *testStorage) Close() error {
|
||||
ts.CloseCheck()
|
||||
err := ts.Storage.Close()
|
||||
if err != nil {
|
||||
ts.t.Errorf("E: closing storage failed: %v", err)
|
||||
} else {
|
||||
ts.t.Log("I: storage closed")
|
||||
}
|
||||
if ts.closeFn != nil {
|
||||
if err := ts.closeFn(); err != nil {
|
||||
ts.t.Errorf("E: close function: %v", err)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (ts *testStorage) CloseCheck() {
|
||||
ts.mu.Lock()
|
||||
if len(ts.opens) == 0 {
|
||||
ts.t.Log("I: all files are closed")
|
||||
} else {
|
||||
ts.t.Errorf("E: %d files still open", len(ts.opens))
|
||||
for x, writer := range ts.opens {
|
||||
num, tt := x>>typeShift, storage.FileType(x)&storage.TypeAll
|
||||
ts.t.Errorf("E: * num=%d type=%v writer=%v", num, tt, writer)
|
||||
}
|
||||
}
|
||||
ts.mu.Unlock()
|
||||
}
|
||||
|
||||
func newTestStorage(t *testing.T) *testStorage {
|
||||
var stor storage.Storage
|
||||
var closeFn func() error
|
||||
if tsFS {
|
||||
for {
|
||||
tsMU.Lock()
|
||||
num := tsNum
|
||||
tsNum++
|
||||
tsMU.Unlock()
|
||||
path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
stor, err = storage.OpenFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("F: cannot create storage: %v", err)
|
||||
}
|
||||
t.Logf("I: storage created: %s", path)
|
||||
closeFn = func() error {
|
||||
for _, name := range []string{"LOG.old", "LOG"} {
|
||||
f, err := os.Open(filepath.Join(path, name))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if log, err := ioutil.ReadAll(f); err != nil {
|
||||
t.Logf("---------------------- %s ----------------------", name)
|
||||
t.Logf("cannot read log: %v", err)
|
||||
t.Logf("---------------------- %s ----------------------", name)
|
||||
} else if len(log) > 0 {
|
||||
t.Logf("---------------------- %s ----------------------\n%s", name, string(log))
|
||||
t.Logf("---------------------- %s ----------------------", name)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
if tsKeepFS {
|
||||
return nil
|
||||
}
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stor = storage.NewMemStorage()
|
||||
}
|
||||
ts := &testStorage{
|
||||
t: t,
|
||||
Storage: stor,
|
||||
closeFn: closeFn,
|
||||
opens: make(map[uint64]bool),
|
||||
}
|
||||
ts.cond.L = &ts.mu
|
||||
return ts
|
||||
}
|
||||
-424
@@ -1,424 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/cache"
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/table"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// table file
|
||||
type tFile struct {
|
||||
file storage.File
|
||||
seekLeft int32
|
||||
size uint64
|
||||
min, max iKey
|
||||
}
|
||||
|
||||
// test if key is after t
|
||||
func (t *tFile) isAfter(key []byte, ucmp comparer.BasicComparer) bool {
|
||||
return key != nil && ucmp.Compare(key, t.max.ukey()) > 0
|
||||
}
|
||||
|
||||
// test if key is before t
|
||||
func (t *tFile) isBefore(key []byte, ucmp comparer.BasicComparer) bool {
|
||||
return key != nil && ucmp.Compare(key, t.min.ukey()) < 0
|
||||
}
|
||||
|
||||
func (t *tFile) incrSeek() int32 {
|
||||
return atomic.AddInt32(&t.seekLeft, -1)
|
||||
}
|
||||
|
||||
func newTFile(file storage.File, size uint64, min, max iKey) *tFile {
|
||||
f := &tFile{
|
||||
file: file,
|
||||
size: size,
|
||||
min: min,
|
||||
max: max,
|
||||
}
|
||||
|
||||
// We arrange to automatically compact this file after
|
||||
// a certain number of seeks. Let's assume:
|
||||
// (1) One seek costs 10ms
|
||||
// (2) Writing or reading 1MB costs 10ms (100MB/s)
|
||||
// (3) A compaction of 1MB does 25MB of IO:
|
||||
// 1MB read from this level
|
||||
// 10-12MB read from next level (boundaries may be misaligned)
|
||||
// 10-12MB written to next level
|
||||
// This implies that 25 seeks cost the same as the compaction
|
||||
// of 1MB of data. I.e., one seek costs approximately the
|
||||
// same as the compaction of 40KB of data. We are a little
|
||||
// conservative and allow approximately one seek for every 16KB
|
||||
// of data before triggering a compaction.
|
||||
f.seekLeft = int32(size / 16384)
|
||||
if f.seekLeft < 100 {
|
||||
f.seekLeft = 100
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
// table files
|
||||
type tFiles []*tFile
|
||||
|
||||
func (tf tFiles) Len() int { return len(tf) }
|
||||
func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] }
|
||||
|
||||
func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool {
|
||||
a, b := tf[i], tf[j]
|
||||
n := icmp.Compare(a.min, b.min)
|
||||
if n == 0 {
|
||||
return a.file.Num() < b.file.Num()
|
||||
}
|
||||
return n < 0
|
||||
}
|
||||
|
||||
func (tf tFiles) lessByNum(i, j int) bool {
|
||||
return tf[i].file.Num() > tf[j].file.Num()
|
||||
}
|
||||
|
||||
func (tf tFiles) sortByKey(icmp *iComparer) {
|
||||
sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp})
|
||||
}
|
||||
|
||||
func (tf tFiles) sortByNum() {
|
||||
sort.Sort(&tFilesSortByNum{tFiles: tf})
|
||||
}
|
||||
|
||||
func (tf tFiles) size() (sum uint64) {
|
||||
for _, t := range tf {
|
||||
sum += t.size
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
func (tf tFiles) searchMin(key iKey, icmp *iComparer) int {
|
||||
return sort.Search(len(tf), func(i int) bool {
|
||||
return icmp.Compare(tf[i].min, key) >= 0
|
||||
})
|
||||
}
|
||||
|
||||
func (tf tFiles) searchMax(key iKey, icmp *iComparer) int {
|
||||
return sort.Search(len(tf), func(i int) bool {
|
||||
return icmp.Compare(tf[i].max, key) >= 0
|
||||
})
|
||||
}
|
||||
|
||||
func (tf tFiles) isOverlaps(min, max []byte, disjSorted bool, icmp *iComparer) bool {
|
||||
if !disjSorted {
|
||||
// Need to check against all files
|
||||
for _, t := range tf {
|
||||
if !t.isAfter(min, icmp.ucmp) && !t.isBefore(max, icmp.ucmp) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var idx int
|
||||
if len(min) > 0 {
|
||||
// Find the earliest possible internal key for min
|
||||
idx = tf.searchMax(newIKey(min, kMaxSeq, tSeek), icmp)
|
||||
}
|
||||
|
||||
if idx >= len(tf) {
|
||||
// beginning of range is after all files, so no overlap
|
||||
return false
|
||||
}
|
||||
return !tf[idx].isBefore(max, icmp.ucmp)
|
||||
}
|
||||
|
||||
func (tf tFiles) getOverlaps(min, max []byte, r *tFiles, disjSorted bool, ucmp comparer.BasicComparer) {
|
||||
for i := 0; i < len(tf); {
|
||||
t := tf[i]
|
||||
i++
|
||||
if t.isAfter(min, ucmp) || t.isBefore(max, ucmp) {
|
||||
continue
|
||||
}
|
||||
|
||||
*r = append(*r, t)
|
||||
if !disjSorted {
|
||||
// Level-0 files may overlap each other. So check if the newly
|
||||
// added file has expanded the range. If so, restart search.
|
||||
if min != nil && ucmp.Compare(t.min.ukey(), min) < 0 {
|
||||
min = t.min.ukey()
|
||||
*r = nil
|
||||
i = 0
|
||||
} else if max != nil && ucmp.Compare(t.max.ukey(), max) > 0 {
|
||||
max = t.max.ukey()
|
||||
*r = nil
|
||||
i = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (tf tFiles) getRange(icmp *iComparer) (min, max iKey) {
|
||||
for i, t := range tf {
|
||||
if i == 0 {
|
||||
min, max = t.min, t.max
|
||||
continue
|
||||
}
|
||||
if icmp.Compare(t.min, min) < 0 {
|
||||
min = t.min
|
||||
}
|
||||
if icmp.Compare(t.max, max) > 0 {
|
||||
max = t.max
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range, ro *opt.ReadOptions) iterator.IteratorIndexer {
|
||||
if slice != nil {
|
||||
var start, limit int
|
||||
if slice.Start != nil {
|
||||
start = tf.searchMax(iKey(slice.Start), icmp)
|
||||
}
|
||||
if slice.Limit != nil {
|
||||
limit = tf.searchMin(iKey(slice.Limit), icmp)
|
||||
} else {
|
||||
limit = tf.Len()
|
||||
}
|
||||
tf = tf[start:limit]
|
||||
}
|
||||
return iterator.NewArrayIndexer(&tFilesArrayIndexer{
|
||||
tFiles: tf,
|
||||
tops: tops,
|
||||
icmp: icmp,
|
||||
slice: slice,
|
||||
ro: ro,
|
||||
})
|
||||
}
|
||||
|
||||
type tFilesArrayIndexer struct {
|
||||
tFiles
|
||||
tops *tOps
|
||||
icmp *iComparer
|
||||
slice *util.Range
|
||||
ro *opt.ReadOptions
|
||||
}
|
||||
|
||||
func (a *tFilesArrayIndexer) Search(key []byte) int {
|
||||
return a.searchMax(iKey(key), a.icmp)
|
||||
}
|
||||
|
||||
func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator {
|
||||
if i == 0 || i == a.Len()-1 {
|
||||
return a.tops.newIterator(a.tFiles[i], a.slice, a.ro)
|
||||
}
|
||||
return a.tops.newIterator(a.tFiles[i], nil, a.ro)
|
||||
}
|
||||
|
||||
type tFilesSortByKey struct {
|
||||
tFiles
|
||||
icmp *iComparer
|
||||
}
|
||||
|
||||
func (x *tFilesSortByKey) Less(i, j int) bool {
|
||||
return x.lessByKey(x.icmp, i, j)
|
||||
}
|
||||
|
||||
type tFilesSortByNum struct {
|
||||
tFiles
|
||||
}
|
||||
|
||||
func (x *tFilesSortByNum) Less(i, j int) bool {
|
||||
return x.lessByNum(i, j)
|
||||
}
|
||||
|
||||
// table operations
|
||||
type tOps struct {
|
||||
s *session
|
||||
cache cache.Cache
|
||||
cacheNS cache.Namespace
|
||||
}
|
||||
|
||||
func newTableOps(s *session, cacheCap int) *tOps {
|
||||
c := cache.NewLRUCache(cacheCap)
|
||||
ns := c.GetNamespace(0)
|
||||
return &tOps{s, c, ns}
|
||||
}
|
||||
|
||||
func (t *tOps) create() (*tWriter, error) {
|
||||
file := t.s.getTableFile(t.s.allocFileNum())
|
||||
fw, err := file.Create()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &tWriter{
|
||||
t: t,
|
||||
file: file,
|
||||
w: fw,
|
||||
tw: table.NewWriter(fw, t.s.o),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
|
||||
w, err := t.create()
|
||||
if err != nil {
|
||||
return f, n, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
w.drop()
|
||||
}
|
||||
}()
|
||||
|
||||
for src.Next() {
|
||||
err = w.add(src.Key(), src.Value())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
err = src.Error()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
n = w.tw.EntriesLen()
|
||||
f, err = w.finish()
|
||||
return
|
||||
}
|
||||
|
||||
func (t *tOps) lookup(f *tFile) (c cache.Object, err error) {
|
||||
num := f.file.Num()
|
||||
c, ok := t.cacheNS.Get(num, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
|
||||
var r storage.Reader
|
||||
r, err = f.file.Open()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
o := t.s.o
|
||||
|
||||
var cacheNS cache.Namespace
|
||||
if bc := o.GetBlockCache(); bc != nil {
|
||||
cacheNS = bc.GetNamespace(num)
|
||||
}
|
||||
|
||||
ok = true
|
||||
value = table.NewReader(r, int64(f.size), cacheNS, o)
|
||||
charge = 1
|
||||
fin = func() {
|
||||
r.Close()
|
||||
}
|
||||
return
|
||||
})
|
||||
if !ok && err == nil {
|
||||
err = ErrClosed
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (t *tOps) get(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
|
||||
c, err := t.lookup(f)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer c.Release()
|
||||
return c.Value().(*table.Reader).Find(key, ro)
|
||||
}
|
||||
|
||||
func (t *tOps) offsetOf(f *tFile, key []byte) (offset uint64, err error) {
|
||||
c, err := t.lookup(f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_offset, err := c.Value().(*table.Reader).OffsetOf(key)
|
||||
offset = uint64(_offset)
|
||||
c.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
c, err := t.lookup(f)
|
||||
if err != nil {
|
||||
return iterator.NewEmptyIterator(err)
|
||||
}
|
||||
iter := c.Value().(*table.Reader).NewIterator(slice, ro)
|
||||
iter.SetReleaser(c)
|
||||
return iter
|
||||
}
|
||||
|
||||
func (t *tOps) remove(f *tFile) {
|
||||
num := f.file.Num()
|
||||
t.cacheNS.Delete(num, func(exist bool) {
|
||||
if err := f.file.Remove(); err != nil {
|
||||
t.s.logf("table@remove removing @%d %q", num, err)
|
||||
} else {
|
||||
t.s.logf("table@remove removed @%d", num)
|
||||
}
|
||||
if bc := t.s.o.GetBlockCache(); bc != nil {
|
||||
bc.GetNamespace(num).Zap(false)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *tOps) close() {
|
||||
t.cache.Zap(true)
|
||||
}
|
||||
|
||||
type tWriter struct {
|
||||
t *tOps
|
||||
|
||||
file storage.File
|
||||
w storage.Writer
|
||||
tw *table.Writer
|
||||
|
||||
first, last []byte
|
||||
}
|
||||
|
||||
func (w *tWriter) add(key, value []byte) error {
|
||||
if w.first == nil {
|
||||
w.first = append([]byte{}, key...)
|
||||
}
|
||||
w.last = append(w.last[:0], key...)
|
||||
return w.tw.Append(key, value)
|
||||
}
|
||||
|
||||
func (w *tWriter) empty() bool {
|
||||
return w.first == nil
|
||||
}
|
||||
|
||||
func (w *tWriter) finish() (f *tFile, err error) {
|
||||
err = w.tw.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = w.w.Sync()
|
||||
if err != nil {
|
||||
w.w.Close()
|
||||
return
|
||||
}
|
||||
w.w.Close()
|
||||
f = newTFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
|
||||
return
|
||||
}
|
||||
|
||||
func (w *tWriter) drop() {
|
||||
w.w.Close()
|
||||
w.file.Remove()
|
||||
w.t.s.reuseFileNum(w.file.Num())
|
||||
w.w = nil
|
||||
w.file = nil
|
||||
w.tw = nil
|
||||
w.first = nil
|
||||
w.last = nil
|
||||
}
|
||||
-848
@@ -1,848 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package table
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"code.google.com/p/snappy-go/snappy"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/cache"
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = util.ErrNotFound
|
||||
ErrIterReleased = errors.New("leveldb/table: iterator released")
|
||||
)
|
||||
|
||||
func max(x, y int) int {
|
||||
if x > y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
||||
type block struct {
|
||||
cmp comparer.BasicComparer
|
||||
data []byte
|
||||
restartsLen int
|
||||
restartsOffset int
|
||||
// Whether checksum is verified and valid.
|
||||
checksum bool
|
||||
}
|
||||
|
||||
func (b *block) seek(rstart, rlimit int, key []byte) (index, offset int, err error) {
|
||||
n := b.restartsOffset
|
||||
data := b.data
|
||||
cmp := b.cmp
|
||||
|
||||
index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
|
||||
offset := int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):]))
|
||||
offset += 1 // shared always zero, since this is a restart point
|
||||
v1, n1 := binary.Uvarint(data[offset:]) // key length
|
||||
_, n2 := binary.Uvarint(data[offset+n1:]) // value length
|
||||
m := offset + n1 + n2
|
||||
return cmp.Compare(data[m:m+int(v1)], key) > 0
|
||||
}) + rstart - 1
|
||||
if index < rstart {
|
||||
// The smallest key is greater-than key sought.
|
||||
index = rstart
|
||||
}
|
||||
offset = int(binary.LittleEndian.Uint32(data[n+4*index:]))
|
||||
return
|
||||
}
|
||||
|
||||
func (b *block) restartIndex(rstart, rlimit, offset int) int {
|
||||
n := b.restartsOffset
|
||||
data := b.data
|
||||
return sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
|
||||
return int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):])) > offset
|
||||
}) + rstart - 1
|
||||
}
|
||||
|
||||
func (b *block) restartOffset(index int) int {
|
||||
return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:]))
|
||||
}
|
||||
|
||||
func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) {
|
||||
if offset >= b.restartsOffset {
|
||||
if offset != b.restartsOffset {
|
||||
err = errors.New("leveldb/table: Reader: BlockEntry: invalid block (block entries offset not aligned)")
|
||||
}
|
||||
return
|
||||
}
|
||||
v0, n0 := binary.Uvarint(b.data[offset:]) // Shared prefix length
|
||||
v1, n1 := binary.Uvarint(b.data[offset+n0:]) // Key length
|
||||
v2, n2 := binary.Uvarint(b.data[offset+n0+n1:]) // Value length
|
||||
m := n0 + n1 + n2
|
||||
n = m + int(v1) + int(v2)
|
||||
if n0 <= 0 || n1 <= 0 || n2 <= 0 || offset+n > b.restartsOffset {
|
||||
err = errors.New("leveldb/table: Reader: invalid block (block entries corrupted)")
|
||||
return
|
||||
}
|
||||
key = b.data[offset+m : offset+m+int(v1)]
|
||||
value = b.data[offset+m+int(v1) : offset+n]
|
||||
nShared = int(v0)
|
||||
return
|
||||
}
|
||||
|
||||
func (b *block) newIterator(slice *util.Range, inclLimit bool, cache util.Releaser) *blockIter {
|
||||
bi := &blockIter{
|
||||
block: b,
|
||||
cache: cache,
|
||||
// Valid key should never be nil.
|
||||
key: make([]byte, 0),
|
||||
dir: dirSOI,
|
||||
riStart: 0,
|
||||
riLimit: b.restartsLen,
|
||||
offsetStart: 0,
|
||||
offsetRealStart: 0,
|
||||
offsetLimit: b.restartsOffset,
|
||||
}
|
||||
if slice != nil {
|
||||
if slice.Start != nil {
|
||||
if bi.Seek(slice.Start) {
|
||||
bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset)
|
||||
bi.offsetStart = b.restartOffset(bi.riStart)
|
||||
bi.offsetRealStart = bi.prevOffset
|
||||
} else {
|
||||
bi.riStart = b.restartsLen
|
||||
bi.offsetStart = b.restartsOffset
|
||||
bi.offsetRealStart = b.restartsOffset
|
||||
}
|
||||
}
|
||||
if slice.Limit != nil {
|
||||
if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) {
|
||||
bi.offsetLimit = bi.prevOffset
|
||||
bi.riLimit = bi.restartIndex + 1
|
||||
}
|
||||
}
|
||||
bi.reset()
|
||||
if bi.offsetStart > bi.offsetLimit {
|
||||
bi.sErr(errors.New("leveldb/table: Reader: invalid slice range"))
|
||||
}
|
||||
}
|
||||
return bi
|
||||
}
|
||||
|
||||
type dir int
|
||||
|
||||
const (
|
||||
dirReleased dir = iota - 1
|
||||
dirSOI
|
||||
dirEOI
|
||||
dirBackward
|
||||
dirForward
|
||||
)
|
||||
|
||||
type blockIter struct {
|
||||
block *block
|
||||
cache, releaser util.Releaser
|
||||
key, value []byte
|
||||
offset int
|
||||
// Previous offset, only filled by Next.
|
||||
prevOffset int
|
||||
prevNode []int
|
||||
prevKeys []byte
|
||||
restartIndex int
|
||||
// Iterator direction.
|
||||
dir dir
|
||||
// Restart index slice range.
|
||||
riStart int
|
||||
riLimit int
|
||||
// Offset slice range.
|
||||
offsetStart int
|
||||
offsetRealStart int
|
||||
offsetLimit int
|
||||
// Error.
|
||||
err error
|
||||
}
|
||||
|
||||
func (i *blockIter) sErr(err error) {
|
||||
i.err = err
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
i.prevNode = nil
|
||||
i.prevKeys = nil
|
||||
}
|
||||
|
||||
func (i *blockIter) reset() {
|
||||
if i.dir == dirBackward {
|
||||
i.prevNode = i.prevNode[:0]
|
||||
i.prevKeys = i.prevKeys[:0]
|
||||
}
|
||||
i.restartIndex = i.riStart
|
||||
i.offset = i.offsetStart
|
||||
i.dir = dirSOI
|
||||
i.key = i.key[:0]
|
||||
i.value = nil
|
||||
}
|
||||
|
||||
func (i *blockIter) isFirst() bool {
|
||||
switch i.dir {
|
||||
case dirForward:
|
||||
return i.prevOffset == i.offsetRealStart
|
||||
case dirBackward:
|
||||
return len(i.prevNode) == 1 && i.restartIndex == i.riStart
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *blockIter) isLast() bool {
|
||||
switch i.dir {
|
||||
case dirForward, dirBackward:
|
||||
return i.offset == i.offsetLimit
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *blockIter) First() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.dir == dirBackward {
|
||||
i.prevNode = i.prevNode[:0]
|
||||
i.prevKeys = i.prevKeys[:0]
|
||||
}
|
||||
i.dir = dirSOI
|
||||
return i.Next()
|
||||
}
|
||||
|
||||
func (i *blockIter) Last() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.dir == dirBackward {
|
||||
i.prevNode = i.prevNode[:0]
|
||||
i.prevKeys = i.prevKeys[:0]
|
||||
}
|
||||
i.dir = dirEOI
|
||||
return i.Prev()
|
||||
}
|
||||
|
||||
func (i *blockIter) Seek(key []byte) bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
ri, offset, err := i.block.seek(i.riStart, i.riLimit, key)
|
||||
if err != nil {
|
||||
i.sErr(err)
|
||||
return false
|
||||
}
|
||||
i.restartIndex = ri
|
||||
i.offset = max(i.offsetStart, offset)
|
||||
if i.dir == dirSOI || i.dir == dirEOI {
|
||||
i.dir = dirForward
|
||||
}
|
||||
for i.Next() {
|
||||
if i.block.cmp.Compare(i.key, key) >= 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *blockIter) Next() bool {
|
||||
if i.dir == dirEOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.dir == dirSOI {
|
||||
i.restartIndex = i.riStart
|
||||
i.offset = i.offsetStart
|
||||
} else if i.dir == dirBackward {
|
||||
i.prevNode = i.prevNode[:0]
|
||||
i.prevKeys = i.prevKeys[:0]
|
||||
}
|
||||
for i.offset < i.offsetRealStart {
|
||||
key, value, nShared, n, err := i.block.entry(i.offset)
|
||||
if err != nil {
|
||||
i.sErr(err)
|
||||
return false
|
||||
}
|
||||
if n == 0 {
|
||||
i.dir = dirEOI
|
||||
return false
|
||||
}
|
||||
i.key = append(i.key[:nShared], key...)
|
||||
i.value = value
|
||||
i.offset += n
|
||||
}
|
||||
if i.offset >= i.offsetLimit {
|
||||
i.dir = dirEOI
|
||||
if i.offset != i.offsetLimit {
|
||||
i.sErr(errors.New("leveldb/table: Reader: Next: invalid block (block entries offset not aligned)"))
|
||||
}
|
||||
return false
|
||||
}
|
||||
key, value, nShared, n, err := i.block.entry(i.offset)
|
||||
if err != nil {
|
||||
i.sErr(err)
|
||||
return false
|
||||
}
|
||||
if n == 0 {
|
||||
i.dir = dirEOI
|
||||
return false
|
||||
}
|
||||
i.key = append(i.key[:nShared], key...)
|
||||
i.value = value
|
||||
i.prevOffset = i.offset
|
||||
i.offset += n
|
||||
i.dir = dirForward
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *blockIter) Prev() bool {
|
||||
if i.dir == dirSOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
var ri int
|
||||
if i.dir == dirForward {
|
||||
// Change direction.
|
||||
i.offset = i.prevOffset
|
||||
if i.offset == i.offsetRealStart {
|
||||
i.dir = dirSOI
|
||||
return false
|
||||
}
|
||||
ri = i.block.restartIndex(i.restartIndex, i.riLimit, i.offset)
|
||||
i.dir = dirBackward
|
||||
} else if i.dir == dirEOI {
|
||||
// At the end of iterator.
|
||||
i.restartIndex = i.riLimit
|
||||
i.offset = i.offsetLimit
|
||||
if i.offset == i.offsetRealStart {
|
||||
i.dir = dirSOI
|
||||
return false
|
||||
}
|
||||
ri = i.riLimit - 1
|
||||
i.dir = dirBackward
|
||||
} else if len(i.prevNode) == 1 {
|
||||
// This is the end of a restart range.
|
||||
i.offset = i.prevNode[0]
|
||||
i.prevNode = i.prevNode[:0]
|
||||
if i.restartIndex == i.riStart {
|
||||
i.dir = dirSOI
|
||||
return false
|
||||
}
|
||||
i.restartIndex--
|
||||
ri = i.restartIndex
|
||||
} else {
|
||||
// In the middle of restart range, get from cache.
|
||||
n := len(i.prevNode) - 3
|
||||
node := i.prevNode[n:]
|
||||
i.prevNode = i.prevNode[:n]
|
||||
// Get the key.
|
||||
ko := node[0]
|
||||
i.key = append(i.key[:0], i.prevKeys[ko:]...)
|
||||
i.prevKeys = i.prevKeys[:ko]
|
||||
// Get the value.
|
||||
vo := node[1]
|
||||
vl := vo + node[2]
|
||||
i.value = i.block.data[vo:vl]
|
||||
i.offset = vl
|
||||
return true
|
||||
}
|
||||
// Build entries cache.
|
||||
i.key = i.key[:0]
|
||||
i.value = nil
|
||||
offset := i.block.restartOffset(ri)
|
||||
if offset == i.offset {
|
||||
ri -= 1
|
||||
if ri < 0 {
|
||||
i.dir = dirSOI
|
||||
return false
|
||||
}
|
||||
offset = i.block.restartOffset(ri)
|
||||
}
|
||||
i.prevNode = append(i.prevNode, offset)
|
||||
for {
|
||||
key, value, nShared, n, err := i.block.entry(offset)
|
||||
if err != nil {
|
||||
i.sErr(err)
|
||||
return false
|
||||
}
|
||||
if offset >= i.offsetRealStart {
|
||||
if i.value != nil {
|
||||
// Appends 3 variables:
|
||||
// 1. Previous keys offset
|
||||
// 2. Value offset in the data block
|
||||
// 3. Value length
|
||||
i.prevNode = append(i.prevNode, len(i.prevKeys), offset-len(i.value), len(i.value))
|
||||
i.prevKeys = append(i.prevKeys, i.key...)
|
||||
}
|
||||
i.value = value
|
||||
}
|
||||
i.key = append(i.key[:nShared], key...)
|
||||
offset += n
|
||||
// Stop if target offset reached.
|
||||
if offset >= i.offset {
|
||||
if offset != i.offset {
|
||||
i.sErr(errors.New("leveldb/table: Reader: Prev: invalid block (block entries offset not aligned)"))
|
||||
return false
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
i.restartIndex = ri
|
||||
i.offset = offset
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *blockIter) Key() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.key
|
||||
}
|
||||
|
||||
func (i *blockIter) Value() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.value
|
||||
}
|
||||
|
||||
func (i *blockIter) Release() {
|
||||
i.prevNode = nil
|
||||
i.prevKeys = nil
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
i.dir = dirReleased
|
||||
if i.cache != nil {
|
||||
i.cache.Release()
|
||||
i.cache = nil
|
||||
}
|
||||
if i.releaser != nil {
|
||||
i.releaser.Release()
|
||||
i.releaser = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i *blockIter) SetReleaser(releaser util.Releaser) {
|
||||
if i.dir > dirReleased {
|
||||
i.releaser = releaser
|
||||
}
|
||||
}
|
||||
|
||||
func (i *blockIter) Valid() bool {
|
||||
return i.err == nil && (i.dir == dirBackward || i.dir == dirForward)
|
||||
}
|
||||
|
||||
func (i *blockIter) Error() error {
|
||||
return i.err
|
||||
}
|
||||
|
||||
type filterBlock struct {
|
||||
filter filter.Filter
|
||||
data []byte
|
||||
oOffset int
|
||||
baseLg uint
|
||||
filtersNum int
|
||||
}
|
||||
|
||||
func (b *filterBlock) contains(offset uint64, key []byte) bool {
|
||||
i := int(offset >> b.baseLg)
|
||||
if i < b.filtersNum {
|
||||
o := b.data[b.oOffset+i*4:]
|
||||
n := int(binary.LittleEndian.Uint32(o))
|
||||
m := int(binary.LittleEndian.Uint32(o[4:]))
|
||||
if n < m && m <= b.oOffset {
|
||||
return b.filter.Contains(b.data[n:m], key)
|
||||
} else if n == m {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type indexIter struct {
|
||||
blockIter
|
||||
tableReader *Reader
|
||||
slice *util.Range
|
||||
// Options
|
||||
checksum bool
|
||||
fillCache bool
|
||||
}
|
||||
|
||||
func (i *indexIter) Get() iterator.Iterator {
|
||||
value := i.Value()
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
dataBH, n := decodeBlockHandle(value)
|
||||
if n == 0 {
|
||||
return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid table (bad data block handle)"))
|
||||
}
|
||||
var slice *util.Range
|
||||
if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) {
|
||||
slice = i.slice
|
||||
}
|
||||
return i.tableReader.getDataIter(dataBH, slice, i.checksum, i.fillCache)
|
||||
}
|
||||
|
||||
// Reader is a table reader.
|
||||
type Reader struct {
|
||||
reader io.ReaderAt
|
||||
cache cache.Namespace
|
||||
err error
|
||||
// Options
|
||||
cmp comparer.Comparer
|
||||
filter filter.Filter
|
||||
checksum bool
|
||||
strictIter bool
|
||||
|
||||
dataEnd int64
|
||||
indexBlock *block
|
||||
filterBlock *filterBlock
|
||||
}
|
||||
|
||||
func verifyChecksum(data []byte) bool {
|
||||
n := len(data) - 4
|
||||
checksum0 := binary.LittleEndian.Uint32(data[n:])
|
||||
checksum1 := util.NewCRC(data[:n]).Value()
|
||||
return checksum0 == checksum1
|
||||
}
|
||||
|
||||
func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
|
||||
data := make([]byte, bh.length+blockTrailerLen)
|
||||
if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
if checksum || r.checksum {
|
||||
if !verifyChecksum(data) {
|
||||
return nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")
|
||||
}
|
||||
}
|
||||
switch data[bh.length] {
|
||||
case blockTypeNoCompression:
|
||||
data = data[:bh.length]
|
||||
case blockTypeSnappyCompression:
|
||||
var err error
|
||||
data, err = snappy.Decode(nil, data[:bh.length])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("leveldb/table: Reader: unknown block compression type: %d", data[bh.length])
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (r *Reader) readBlock(bh blockHandle, checksum bool) (*block, error) {
|
||||
data, err := r.readRawBlock(bh, checksum)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
|
||||
b := &block{
|
||||
cmp: r.cmp,
|
||||
data: data,
|
||||
restartsLen: restartsLen,
|
||||
restartsOffset: len(data) - (restartsLen+1)*4,
|
||||
checksum: checksum || r.checksum,
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (r *Reader) readFilterBlock(bh blockHandle, filter filter.Filter) (*filterBlock, error) {
|
||||
data, err := r.readRawBlock(bh, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n := len(data)
|
||||
if n < 5 {
|
||||
return nil, errors.New("leveldb/table: Reader: invalid filter block (too short)")
|
||||
}
|
||||
m := n - 5
|
||||
oOffset := int(binary.LittleEndian.Uint32(data[m:]))
|
||||
if oOffset > m {
|
||||
return nil, errors.New("leveldb/table: Reader: invalid filter block (invalid offset)")
|
||||
}
|
||||
b := &filterBlock{
|
||||
filter: filter,
|
||||
data: data,
|
||||
oOffset: oOffset,
|
||||
baseLg: uint(data[n-1]),
|
||||
filtersNum: (m - oOffset) / 4,
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
|
||||
if r.cache != nil {
|
||||
// Get/set block cache.
|
||||
var err error
|
||||
cache, ok := r.cache.Get(dataBH.offset, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
|
||||
if !fillCache {
|
||||
return
|
||||
}
|
||||
var dataBlock *block
|
||||
dataBlock, err = r.readBlock(dataBH, checksum)
|
||||
if err == nil {
|
||||
ok = true
|
||||
value = dataBlock
|
||||
charge = int(dataBH.length)
|
||||
}
|
||||
return
|
||||
})
|
||||
if err != nil {
|
||||
return iterator.NewEmptyIterator(err)
|
||||
}
|
||||
if ok {
|
||||
dataBlock := cache.Value().(*block)
|
||||
if !dataBlock.checksum && (r.checksum || checksum) {
|
||||
if !verifyChecksum(dataBlock.data) {
|
||||
return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid block (checksum mismatch)"))
|
||||
}
|
||||
dataBlock.checksum = true
|
||||
}
|
||||
iter := dataBlock.newIterator(slice, false, cache)
|
||||
return iter
|
||||
}
|
||||
}
|
||||
dataBlock, err := r.readBlock(dataBH, checksum)
|
||||
if err != nil {
|
||||
return iterator.NewEmptyIterator(err)
|
||||
}
|
||||
iter := dataBlock.newIterator(slice, false, nil)
|
||||
return iter
|
||||
}
|
||||
|
||||
// NewIterator creates an iterator from the table.
|
||||
//
|
||||
// Slice allows slicing the iterator to only contains keys in the given
|
||||
// range. A nil Range.Start is treated as a key before all keys in the
|
||||
// table. And a nil Range.Limit is treated as a key after all keys in
|
||||
// the table.
|
||||
//
|
||||
// The returned iterator is not goroutine-safe and should be released
|
||||
// when not used.
|
||||
//
|
||||
// Also read Iterator documentation of the leveldb/iterator package.
|
||||
|
||||
func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
if r.err != nil {
|
||||
return iterator.NewEmptyIterator(r.err)
|
||||
}
|
||||
|
||||
index := &indexIter{
|
||||
blockIter: *r.indexBlock.newIterator(slice, true, nil),
|
||||
tableReader: r,
|
||||
slice: slice,
|
||||
checksum: ro.GetStrict(opt.StrictBlockChecksum),
|
||||
fillCache: !ro.GetDontFillCache(),
|
||||
}
|
||||
return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), false)
|
||||
}
|
||||
|
||||
// Find finds key/value pair whose key is greater than or equal to the
|
||||
// given key. It returns ErrNotFound if the table doesn't contain
|
||||
// such pair.
|
||||
//
|
||||
// The caller should not modify the contents of the returned slice, but
|
||||
// it is safe to modify the contents of the argument after Find returns.
|
||||
func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err error) {
|
||||
if r.err != nil {
|
||||
err = r.err
|
||||
return
|
||||
}
|
||||
|
||||
index := r.indexBlock.newIterator(nil, true, nil)
|
||||
defer index.Release()
|
||||
if !index.Seek(key) {
|
||||
err = index.Error()
|
||||
if err == nil {
|
||||
err = ErrNotFound
|
||||
}
|
||||
return
|
||||
}
|
||||
dataBH, n := decodeBlockHandle(index.Value())
|
||||
if n == 0 {
|
||||
err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)")
|
||||
return
|
||||
}
|
||||
if r.filterBlock != nil && !r.filterBlock.contains(dataBH.offset, key) {
|
||||
err = ErrNotFound
|
||||
return
|
||||
}
|
||||
data := r.getDataIter(dataBH, nil, ro.GetStrict(opt.StrictBlockChecksum), !ro.GetDontFillCache())
|
||||
defer data.Release()
|
||||
if !data.Seek(key) {
|
||||
err = data.Error()
|
||||
if err == nil {
|
||||
err = ErrNotFound
|
||||
}
|
||||
return
|
||||
}
|
||||
rkey = data.Key()
|
||||
value = data.Value()
|
||||
return
|
||||
}
|
||||
|
||||
// Get gets the value for the given key. It returns errors.ErrNotFound
|
||||
// if the table does not contain the key.
|
||||
//
|
||||
// The caller should not modify the contents of the returned slice, but
|
||||
// it is safe to modify the contents of the argument after Get returns.
|
||||
func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
|
||||
if r.err != nil {
|
||||
err = r.err
|
||||
return
|
||||
}
|
||||
|
||||
rkey, value, err := r.Find(key, ro)
|
||||
if err == nil && r.cmp.Compare(rkey, key) != 0 {
|
||||
value = nil
|
||||
err = ErrNotFound
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// OffsetOf returns approximate offset for the given key.
|
||||
//
|
||||
// It is safe to modify the contents of the argument after Get returns.
|
||||
func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
|
||||
if r.err != nil {
|
||||
err = r.err
|
||||
return
|
||||
}
|
||||
|
||||
index := r.indexBlock.newIterator(nil, true, nil)
|
||||
defer index.Release()
|
||||
if index.Seek(key) {
|
||||
dataBH, n := decodeBlockHandle(index.Value())
|
||||
if n == 0 {
|
||||
err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)")
|
||||
return
|
||||
}
|
||||
offset = int64(dataBH.offset)
|
||||
return
|
||||
}
|
||||
err = index.Error()
|
||||
if err == nil {
|
||||
offset = r.dataEnd
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewReader creates a new initialized table reader for the file.
|
||||
// The cache is optional and can be nil.
|
||||
func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, o *opt.Options) *Reader {
|
||||
r := &Reader{
|
||||
reader: f,
|
||||
cache: cache,
|
||||
cmp: o.GetComparer(),
|
||||
checksum: o.GetStrict(opt.StrictBlockChecksum),
|
||||
strictIter: o.GetStrict(opt.StrictIterator),
|
||||
}
|
||||
if f == nil {
|
||||
r.err = errors.New("leveldb/table: Reader: nil file")
|
||||
return r
|
||||
}
|
||||
if size < footerLen {
|
||||
r.err = errors.New("leveldb/table: Reader: invalid table (file size is too small)")
|
||||
return r
|
||||
}
|
||||
var footer [footerLen]byte
|
||||
if _, err := r.reader.ReadAt(footer[:], size-footerLen); err != nil && err != io.EOF {
|
||||
r.err = fmt.Errorf("leveldb/table: Reader: invalid table (could not read footer): %v", err)
|
||||
}
|
||||
if string(footer[footerLen-len(magic):footerLen]) != magic {
|
||||
r.err = errors.New("leveldb/table: Reader: invalid table (bad magic number)")
|
||||
return r
|
||||
}
|
||||
// Decode the metaindex block handle.
|
||||
metaBH, n := decodeBlockHandle(footer[:])
|
||||
if n == 0 {
|
||||
r.err = errors.New("leveldb/table: Reader: invalid table (bad metaindex block handle)")
|
||||
return r
|
||||
}
|
||||
// Decode the index block handle.
|
||||
indexBH, n := decodeBlockHandle(footer[n:])
|
||||
if n == 0 {
|
||||
r.err = errors.New("leveldb/table: Reader: invalid table (bad index block handle)")
|
||||
return r
|
||||
}
|
||||
// Read index block.
|
||||
r.indexBlock, r.err = r.readBlock(indexBH, true)
|
||||
if r.err != nil {
|
||||
return r
|
||||
}
|
||||
// Read metaindex block.
|
||||
metaBlock, err := r.readBlock(metaBH, true)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return r
|
||||
}
|
||||
// Set data end.
|
||||
r.dataEnd = int64(metaBH.offset)
|
||||
metaIter := metaBlock.newIterator(nil, false, nil)
|
||||
for metaIter.Next() {
|
||||
key := string(metaIter.Key())
|
||||
if !strings.HasPrefix(key, "filter.") {
|
||||
continue
|
||||
}
|
||||
fn := key[7:]
|
||||
var filter filter.Filter
|
||||
if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn {
|
||||
filter = f0
|
||||
} else {
|
||||
for _, f0 := range o.GetAltFilters() {
|
||||
if f0.Name() == fn {
|
||||
filter = f0
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if filter != nil {
|
||||
filterBH, n := decodeBlockHandle(metaIter.Value())
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
// Update data end.
|
||||
r.dataEnd = int64(filterBH.offset)
|
||||
filterBlock, err := r.readFilterBlock(filterBH, filter)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
r.filterBlock = filterBlock
|
||||
break
|
||||
}
|
||||
}
|
||||
metaIter.Release()
|
||||
return r
|
||||
}
|
||||
-177
@@ -1,177 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package table allows read and write sorted key/value.
|
||||
package table
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
/*
|
||||
Table:
|
||||
|
||||
Table is consist of one or more data blocks, an optional filter block
|
||||
a metaindex block, an index block and a table footer. Metaindex block
|
||||
is a special block used to keep parameters of the table, such as filter
|
||||
block name and its block handle. Index block is a special block used to
|
||||
keep record of data blocks offset and length, index block use one as
|
||||
restart interval. The key used by index block are the last key of preceding
|
||||
block, shorter separator of adjacent blocks or shorter successor of the
|
||||
last key of the last block. Filter block is an optional block contains
|
||||
sequence of filter data generated by a filter generator.
|
||||
|
||||
Table data structure:
|
||||
+ optional
|
||||
/
|
||||
+--------------+--------------+--------------+------+-------+-----------------+-------------+--------+
|
||||
| data block 1 | ... | data block n | filter block | metaindex block | index block | footer |
|
||||
+--------------+--------------+--------------+--------------+-----------------+-------------+--------+
|
||||
|
||||
Each block followed by a 5-bytes trailer contains compression type and checksum.
|
||||
|
||||
Table block trailer:
|
||||
|
||||
+---------------------------+-------------------+
|
||||
| compression type (1-byte) | checksum (4-byte) |
|
||||
+---------------------------+-------------------+
|
||||
|
||||
The checksum is a CRC-32 computed using Castagnoli's polynomial. Compression
|
||||
type also included in the checksum.
|
||||
|
||||
Table footer:
|
||||
|
||||
+------------------- 40-bytes -------------------+
|
||||
/ \
|
||||
+------------------------+--------------------+------+-----------------+
|
||||
| metaindex block handle / index block handle / ---- | magic (8-bytes) |
|
||||
+------------------------+--------------------+------+-----------------+
|
||||
|
||||
The magic are first 64-bit of SHA-1 sum of "http://code.google.com/p/leveldb/".
|
||||
|
||||
NOTE: All fixed-length integer are little-endian.
|
||||
*/
|
||||
|
||||
/*
|
||||
Block:
|
||||
|
||||
Block is consist of one or more key/value entries and a block trailer.
|
||||
Block entry shares key prefix with its preceding key until a restart
|
||||
point reached. A block should contains at least one restart point.
|
||||
First restart point are always zero.
|
||||
|
||||
Block data structure:
|
||||
|
||||
+ restart point + restart point (depends on restart interval)
|
||||
/ /
|
||||
+---------------+---------------+---------------+---------------+---------+
|
||||
| block entry 1 | block entry 2 | ... | block entry n | trailer |
|
||||
+---------------+---------------+---------------+---------------+---------+
|
||||
|
||||
Key/value entry:
|
||||
|
||||
+---- key len ----+
|
||||
/ \
|
||||
+-------+---------+-----------+---------+--------------------+--------------+----------------+
|
||||
| shared (varint) | not shared (varint) | value len (varint) | key (varlen) | value (varlen) |
|
||||
+-----------------+---------------------+--------------------+--------------+----------------+
|
||||
|
||||
Block entry shares key prefix with its preceding key:
|
||||
Conditions:
|
||||
restart_interval=2
|
||||
entry one : key=deck,value=v1
|
||||
entry two : key=dock,value=v2
|
||||
entry three: key=duck,value=v3
|
||||
The entries will be encoded as follow:
|
||||
|
||||
+ restart point (offset=0) + restart point (offset=16)
|
||||
/ /
|
||||
+-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
|
||||
| 0 | 4 | 2 | "deck" | "v1" | 1 | 3 | 2 | "ock" | "v2" | 0 | 4 | 2 | "duck" | "v3" |
|
||||
+-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
|
||||
\ / \ / \ /
|
||||
+----------- entry one -----------+ +----------- entry two ----------+ +---------- entry three ----------+
|
||||
|
||||
The block trailer will contains two restart points:
|
||||
|
||||
+------------+-----------+--------+
|
||||
| 0 | 16 | 2 |
|
||||
+------------+-----------+---+----+
|
||||
\ / \
|
||||
+-- restart points --+ + restart points length
|
||||
|
||||
Block trailer:
|
||||
|
||||
+-- 4-bytes --+
|
||||
/ \
|
||||
+-----------------+-----------------+-----------------+------------------------------+
|
||||
| restart point 1 | .... | restart point n | restart points len (4-bytes) |
|
||||
+-----------------+-----------------+-----------------+------------------------------+
|
||||
|
||||
|
||||
NOTE: All fixed-length integer are little-endian.
|
||||
*/
|
||||
|
||||
/*
|
||||
Filter block:
|
||||
|
||||
Filter block consist of one or more filter data and a filter block trailer.
|
||||
The trailer contains filter data offsets, a trailer offset and a 1-byte base Lg.
|
||||
|
||||
Filter block data structure:
|
||||
|
||||
+ offset 1 + offset 2 + offset n + trailer offset
|
||||
/ / / /
|
||||
+---------------+---------------+---------------+---------+
|
||||
| filter data 1 | ... | filter data n | trailer |
|
||||
+---------------+---------------+---------------+---------+
|
||||
|
||||
Filter block trailer:
|
||||
|
||||
+- 4-bytes -+
|
||||
/ \
|
||||
+---------------+---------------+---------------+-------------------------+------------------+
|
||||
| offset 1 | .... | offset n | filter offset (4-bytes) | base Lg (1-byte) |
|
||||
+-------------- +---------------+---------------+-------------------------+------------------+
|
||||
|
||||
|
||||
NOTE: All fixed-length integer are little-endian.
|
||||
*/
|
||||
|
||||
const (
|
||||
blockTrailerLen = 5
|
||||
footerLen = 48
|
||||
|
||||
magic = "\x57\xfb\x80\x8b\x24\x75\x47\xdb"
|
||||
|
||||
// The block type gives the per-block compression format.
|
||||
// These constants are part of the file format and should not be changed.
|
||||
blockTypeNoCompression = 0
|
||||
blockTypeSnappyCompression = 1
|
||||
|
||||
// Generate new filter every 2KB of data
|
||||
filterBaseLg = 11
|
||||
filterBase = 1 << filterBaseLg
|
||||
)
|
||||
|
||||
type blockHandle struct {
|
||||
offset, length uint64
|
||||
}
|
||||
|
||||
func decodeBlockHandle(src []byte) (blockHandle, int) {
|
||||
offset, n := binary.Uvarint(src)
|
||||
length, m := binary.Uvarint(src[n:])
|
||||
if n == 0 || m == 0 {
|
||||
return blockHandle{}, 0
|
||||
}
|
||||
return blockHandle{offset, length}, n + m
|
||||
}
|
||||
|
||||
func encodeBlockHandle(dst []byte, b blockHandle) int {
|
||||
n := binary.PutUvarint(dst, b.offset)
|
||||
m := binary.PutUvarint(dst[n:], b.length)
|
||||
return n + m
|
||||
}
|
||||
Generated
Vendored
-17
@@ -1,17 +0,0 @@
|
||||
package table
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/testutil"
|
||||
)
|
||||
|
||||
func TestTable(t *testing.T) {
|
||||
testutil.RunDefer()
|
||||
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Table Suite")
|
||||
}
|
||||
-119
@@ -1,119 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package table
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/testutil"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type tableWrapper struct {
|
||||
*Reader
|
||||
}
|
||||
|
||||
func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) {
|
||||
return t.Reader.Find(key, nil)
|
||||
}
|
||||
|
||||
func (t tableWrapper) TestGet(key []byte) (value []byte, err error) {
|
||||
return t.Reader.Get(key, nil)
|
||||
}
|
||||
|
||||
func (t tableWrapper) TestNewIterator(slice *util.Range) iterator.Iterator {
|
||||
return t.Reader.NewIterator(slice, nil)
|
||||
}
|
||||
|
||||
var _ = testutil.Defer(func() {
|
||||
Describe("Table", func() {
|
||||
Describe("approximate offset test", func() {
|
||||
var (
|
||||
buf = &bytes.Buffer{}
|
||||
o = &opt.Options{
|
||||
BlockSize: 1024,
|
||||
Compression: opt.NoCompression,
|
||||
}
|
||||
)
|
||||
|
||||
// Building the table.
|
||||
tw := NewWriter(buf, o)
|
||||
tw.Append([]byte("k01"), []byte("hello"))
|
||||
tw.Append([]byte("k02"), []byte("hello2"))
|
||||
tw.Append([]byte("k03"), bytes.Repeat([]byte{'x'}, 10000))
|
||||
tw.Append([]byte("k04"), bytes.Repeat([]byte{'x'}, 200000))
|
||||
tw.Append([]byte("k05"), bytes.Repeat([]byte{'x'}, 300000))
|
||||
tw.Append([]byte("k06"), []byte("hello3"))
|
||||
tw.Append([]byte("k07"), bytes.Repeat([]byte{'x'}, 100000))
|
||||
err := tw.Close()
|
||||
|
||||
It("Should be able to approximate offset of a key correctly", func() {
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
|
||||
tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
|
||||
CheckOffset := func(key string, expect, threshold int) {
|
||||
offset, err := tr.OffsetOf([]byte(key))
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
Expect(offset).Should(BeNumerically("~", expect, threshold), "Offset of key %q", key)
|
||||
}
|
||||
|
||||
CheckOffset("k0", 0, 0)
|
||||
CheckOffset("k01a", 0, 0)
|
||||
CheckOffset("k02", 0, 0)
|
||||
CheckOffset("k03", 0, 0)
|
||||
CheckOffset("k04", 10000, 1000)
|
||||
CheckOffset("k04a", 210000, 1000)
|
||||
CheckOffset("k05", 210000, 1000)
|
||||
CheckOffset("k06", 510000, 1000)
|
||||
CheckOffset("k07", 510000, 1000)
|
||||
CheckOffset("xyz", 610000, 2000)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("read test", func() {
|
||||
Build := func(kv testutil.KeyValue) testutil.DB {
|
||||
o := &opt.Options{
|
||||
BlockSize: 512,
|
||||
BlockRestartInterval: 3,
|
||||
}
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
// Building the table.
|
||||
tw := NewWriter(buf, o)
|
||||
kv.Iterate(func(i int, key, value []byte) {
|
||||
tw.Append(key, value)
|
||||
})
|
||||
tw.Close()
|
||||
|
||||
// Opening the table.
|
||||
tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
|
||||
return tableWrapper{tr}
|
||||
}
|
||||
Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() {
|
||||
return func() {
|
||||
db := Build(*kv)
|
||||
if body != nil {
|
||||
body(db.(tableWrapper).Reader)
|
||||
}
|
||||
testutil.KeyValueTesting(nil, db, *kv)
|
||||
}
|
||||
}
|
||||
|
||||
testutil.AllKeyValueTesting(nil, Build)
|
||||
Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) {
|
||||
It("should have correct blocks number", func() {
|
||||
Expect(r.indexBlock.restartsLen).Should(Equal(9))
|
||||
})
|
||||
}))
|
||||
})
|
||||
})
|
||||
})
|
||||
-379
@@ -1,379 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package table
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"code.google.com/p/snappy-go/snappy"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
func sharedPrefixLen(a, b []byte) int {
|
||||
i, n := 0, len(a)
|
||||
if n > len(b) {
|
||||
n = len(b)
|
||||
}
|
||||
for i < n && a[i] == b[i] {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
type blockWriter struct {
|
||||
restartInterval int
|
||||
buf util.Buffer
|
||||
nEntries int
|
||||
prevKey []byte
|
||||
restarts []uint32
|
||||
scratch []byte
|
||||
}
|
||||
|
||||
func (w *blockWriter) append(key, value []byte) {
|
||||
nShared := 0
|
||||
if w.nEntries%w.restartInterval == 0 {
|
||||
w.restarts = append(w.restarts, uint32(w.buf.Len()))
|
||||
} else {
|
||||
nShared = sharedPrefixLen(w.prevKey, key)
|
||||
}
|
||||
n := binary.PutUvarint(w.scratch[0:], uint64(nShared))
|
||||
n += binary.PutUvarint(w.scratch[n:], uint64(len(key)-nShared))
|
||||
n += binary.PutUvarint(w.scratch[n:], uint64(len(value)))
|
||||
w.buf.Write(w.scratch[:n])
|
||||
w.buf.Write(key[nShared:])
|
||||
w.buf.Write(value)
|
||||
w.prevKey = append(w.prevKey[:0], key...)
|
||||
w.nEntries++
|
||||
}
|
||||
|
||||
func (w *blockWriter) finish() {
|
||||
// Write restarts entry.
|
||||
if w.nEntries == 0 {
|
||||
// Must have at least one restart entry.
|
||||
w.restarts = append(w.restarts, 0)
|
||||
}
|
||||
w.restarts = append(w.restarts, uint32(len(w.restarts)))
|
||||
for _, x := range w.restarts {
|
||||
buf4 := w.buf.Alloc(4)
|
||||
binary.LittleEndian.PutUint32(buf4, x)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *blockWriter) reset() {
|
||||
w.buf.Reset()
|
||||
w.nEntries = 0
|
||||
w.restarts = w.restarts[:0]
|
||||
}
|
||||
|
||||
func (w *blockWriter) bytesLen() int {
|
||||
restartsLen := len(w.restarts)
|
||||
if restartsLen == 0 {
|
||||
restartsLen = 1
|
||||
}
|
||||
return w.buf.Len() + 4*restartsLen + 4
|
||||
}
|
||||
|
||||
type filterWriter struct {
|
||||
generator filter.FilterGenerator
|
||||
buf util.Buffer
|
||||
nKeys int
|
||||
offsets []uint32
|
||||
}
|
||||
|
||||
func (w *filterWriter) add(key []byte) {
|
||||
if w.generator == nil {
|
||||
return
|
||||
}
|
||||
w.generator.Add(key)
|
||||
w.nKeys++
|
||||
}
|
||||
|
||||
func (w *filterWriter) flush(offset uint64) {
|
||||
if w.generator == nil {
|
||||
return
|
||||
}
|
||||
for x := int(offset / filterBase); x > len(w.offsets); {
|
||||
w.generate()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *filterWriter) finish() {
|
||||
if w.generator == nil {
|
||||
return
|
||||
}
|
||||
// Generate last keys.
|
||||
|
||||
if w.nKeys > 0 {
|
||||
w.generate()
|
||||
}
|
||||
w.offsets = append(w.offsets, uint32(w.buf.Len()))
|
||||
for _, x := range w.offsets {
|
||||
buf4 := w.buf.Alloc(4)
|
||||
binary.LittleEndian.PutUint32(buf4, x)
|
||||
}
|
||||
w.buf.WriteByte(filterBaseLg)
|
||||
}
|
||||
|
||||
func (w *filterWriter) generate() {
|
||||
// Record offset.
|
||||
w.offsets = append(w.offsets, uint32(w.buf.Len()))
|
||||
// Generate filters.
|
||||
if w.nKeys > 0 {
|
||||
w.generator.Generate(&w.buf)
|
||||
w.nKeys = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Writer is a table writer.
|
||||
type Writer struct {
|
||||
writer io.Writer
|
||||
err error
|
||||
// Options
|
||||
cmp comparer.Comparer
|
||||
filter filter.Filter
|
||||
compression opt.Compression
|
||||
blockSize int
|
||||
|
||||
dataBlock blockWriter
|
||||
indexBlock blockWriter
|
||||
filterBlock filterWriter
|
||||
pendingBH blockHandle
|
||||
offset uint64
|
||||
nEntries int
|
||||
// Scratch allocated enough for 5 uvarint. Block writer should not use
|
||||
// first 20-bytes since it will be used to encode block handle, which
|
||||
// then passed to the block writer itself.
|
||||
scratch [50]byte
|
||||
comparerScratch []byte
|
||||
compressionScratch []byte
|
||||
}
|
||||
|
||||
func (w *Writer) writeBlock(buf *util.Buffer, compression opt.Compression) (bh blockHandle, err error) {
|
||||
// Compress the buffer if necessary.
|
||||
var b []byte
|
||||
if compression == opt.SnappyCompression {
|
||||
// Allocate scratch enough for compression and block trailer.
|
||||
if n := snappy.MaxEncodedLen(buf.Len()) + blockTrailerLen; len(w.compressionScratch) < n {
|
||||
w.compressionScratch = make([]byte, n)
|
||||
}
|
||||
var compressed []byte
|
||||
compressed, err = snappy.Encode(w.compressionScratch, buf.Bytes())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
n := len(compressed)
|
||||
b = compressed[:n+blockTrailerLen]
|
||||
b[n] = blockTypeSnappyCompression
|
||||
} else {
|
||||
tmp := buf.Alloc(blockTrailerLen)
|
||||
tmp[0] = blockTypeNoCompression
|
||||
b = buf.Bytes()
|
||||
}
|
||||
|
||||
// Calculate the checksum.
|
||||
n := len(b) - 4
|
||||
checksum := util.NewCRC(b[:n]).Value()
|
||||
binary.LittleEndian.PutUint32(b[n:], checksum)
|
||||
|
||||
// Write the buffer to the file.
|
||||
_, err = w.writer.Write(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
bh = blockHandle{w.offset, uint64(len(b) - blockTrailerLen)}
|
||||
w.offset += uint64(len(b))
|
||||
return
|
||||
}
|
||||
|
||||
func (w *Writer) flushPendingBH(key []byte) {
|
||||
if w.pendingBH.length == 0 {
|
||||
return
|
||||
}
|
||||
var separator []byte
|
||||
if len(key) == 0 {
|
||||
separator = w.cmp.Successor(w.comparerScratch[:0], w.dataBlock.prevKey)
|
||||
} else {
|
||||
separator = w.cmp.Separator(w.comparerScratch[:0], w.dataBlock.prevKey, key)
|
||||
}
|
||||
if separator == nil {
|
||||
separator = w.dataBlock.prevKey
|
||||
} else {
|
||||
w.comparerScratch = separator
|
||||
}
|
||||
n := encodeBlockHandle(w.scratch[:20], w.pendingBH)
|
||||
// Append the block handle to the index block.
|
||||
w.indexBlock.append(separator, w.scratch[:n])
|
||||
// Reset prev key of the data block.
|
||||
w.dataBlock.prevKey = w.dataBlock.prevKey[:0]
|
||||
// Clear pending block handle.
|
||||
w.pendingBH = blockHandle{}
|
||||
}
|
||||
|
||||
func (w *Writer) finishBlock() error {
|
||||
w.dataBlock.finish()
|
||||
bh, err := w.writeBlock(&w.dataBlock.buf, w.compression)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.pendingBH = bh
|
||||
// Reset the data block.
|
||||
w.dataBlock.reset()
|
||||
// Flush the filter block.
|
||||
w.filterBlock.flush(w.offset)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Append appends key/value pair to the table. The keys passed must
|
||||
// be in increasing order.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Append returns.
|
||||
func (w *Writer) Append(key, value []byte) error {
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
if w.nEntries > 0 && w.cmp.Compare(w.dataBlock.prevKey, key) >= 0 {
|
||||
w.err = fmt.Errorf("leveldb/table: Writer: keys are not in increasing order: %q, %q", w.dataBlock.prevKey, key)
|
||||
return w.err
|
||||
}
|
||||
|
||||
w.flushPendingBH(key)
|
||||
// Append key/value pair to the data block.
|
||||
w.dataBlock.append(key, value)
|
||||
// Add key to the filter block.
|
||||
w.filterBlock.add(key)
|
||||
|
||||
// Finish the data block if block size target reached.
|
||||
if w.dataBlock.bytesLen() >= w.blockSize {
|
||||
if err := w.finishBlock(); err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
}
|
||||
w.nEntries++
|
||||
return nil
|
||||
}
|
||||
|
||||
// BlocksLen returns number of blocks written so far.
|
||||
func (w *Writer) BlocksLen() int {
|
||||
n := w.indexBlock.nEntries
|
||||
if w.pendingBH.length > 0 {
|
||||
// Includes the pending block.
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// EntriesLen returns number of entries added so far.
|
||||
func (w *Writer) EntriesLen() int {
|
||||
return w.nEntries
|
||||
}
|
||||
|
||||
// BytesLen returns number of bytes written so far.
|
||||
func (w *Writer) BytesLen() int {
|
||||
return int(w.offset)
|
||||
}
|
||||
|
||||
// Close will finalize the table. Calling Append is not possible
|
||||
// after Close, but calling BlocksLen, EntriesLen and BytesLen
|
||||
// is still possible.
|
||||
func (w *Writer) Close() error {
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
|
||||
// Write the last data block. Or empty data block if there
|
||||
// aren't any data blocks at all.
|
||||
if w.dataBlock.nEntries > 0 || w.nEntries == 0 {
|
||||
if err := w.finishBlock(); err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
}
|
||||
w.flushPendingBH(nil)
|
||||
|
||||
// Write the filter block.
|
||||
var filterBH blockHandle
|
||||
w.filterBlock.finish()
|
||||
if buf := &w.filterBlock.buf; buf.Len() > 0 {
|
||||
filterBH, w.err = w.writeBlock(buf, opt.NoCompression)
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
}
|
||||
|
||||
// Write the metaindex block.
|
||||
if filterBH.length > 0 {
|
||||
key := []byte("filter." + w.filter.Name())
|
||||
n := encodeBlockHandle(w.scratch[:20], filterBH)
|
||||
w.dataBlock.append(key, w.scratch[:n])
|
||||
}
|
||||
w.dataBlock.finish()
|
||||
metaindexBH, err := w.writeBlock(&w.dataBlock.buf, w.compression)
|
||||
if err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
|
||||
// Write the index block.
|
||||
w.indexBlock.finish()
|
||||
indexBH, err := w.writeBlock(&w.indexBlock.buf, w.compression)
|
||||
if err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
|
||||
// Write the table footer.
|
||||
footer := w.scratch[:footerLen]
|
||||
for i := range footer {
|
||||
footer[i] = 0
|
||||
}
|
||||
n := encodeBlockHandle(footer, metaindexBH)
|
||||
encodeBlockHandle(footer[n:], indexBH)
|
||||
copy(footer[footerLen-len(magic):], magic)
|
||||
if _, err := w.writer.Write(footer); err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
w.offset += footerLen
|
||||
|
||||
w.err = errors.New("leveldb/table: writer is closed")
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewWriter creates a new initialized table writer for the file.
|
||||
//
|
||||
// Table writer is not goroutine-safe.
|
||||
func NewWriter(f io.Writer, o *opt.Options) *Writer {
|
||||
w := &Writer{
|
||||
writer: f,
|
||||
cmp: o.GetComparer(),
|
||||
filter: o.GetFilter(),
|
||||
compression: o.GetCompression(),
|
||||
blockSize: o.GetBlockSize(),
|
||||
comparerScratch: make([]byte, 0),
|
||||
}
|
||||
// data block
|
||||
w.dataBlock.restartInterval = o.GetBlockRestartInterval()
|
||||
// The first 20-bytes are used for encoding block handle.
|
||||
w.dataBlock.scratch = w.scratch[20:]
|
||||
// index block
|
||||
w.indexBlock.restartInterval = 1
|
||||
w.indexBlock.scratch = w.scratch[20:]
|
||||
// filter block
|
||||
if w.filter != nil {
|
||||
w.filterBlock.generator = w.filter.NewGenerator()
|
||||
w.filterBlock.flush(0)
|
||||
}
|
||||
return w
|
||||
}
|
||||
-216
@@ -1,216 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type DB interface{}
|
||||
|
||||
type Put interface {
|
||||
TestPut(key []byte, value []byte) error
|
||||
}
|
||||
|
||||
type Delete interface {
|
||||
TestDelete(key []byte) error
|
||||
}
|
||||
|
||||
type Find interface {
|
||||
TestFind(key []byte) (rkey, rvalue []byte, err error)
|
||||
}
|
||||
|
||||
type Get interface {
|
||||
TestGet(key []byte) (value []byte, err error)
|
||||
}
|
||||
|
||||
type NewIterator interface {
|
||||
TestNewIterator(slice *util.Range) iterator.Iterator
|
||||
}
|
||||
|
||||
type DBAct int
|
||||
|
||||
func (a DBAct) String() string {
|
||||
switch a {
|
||||
case DBNone:
|
||||
return "none"
|
||||
case DBPut:
|
||||
return "put"
|
||||
case DBOverwrite:
|
||||
return "overwrite"
|
||||
case DBDelete:
|
||||
return "delete"
|
||||
case DBDeleteNA:
|
||||
return "delete_na"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
const (
|
||||
DBNone DBAct = iota
|
||||
DBPut
|
||||
DBOverwrite
|
||||
DBDelete
|
||||
DBDeleteNA
|
||||
)
|
||||
|
||||
type DBTesting struct {
|
||||
Rand *rand.Rand
|
||||
DB interface {
|
||||
Get
|
||||
Put
|
||||
Delete
|
||||
}
|
||||
PostFn func(t *DBTesting)
|
||||
Deleted, Present KeyValue
|
||||
Act, LastAct DBAct
|
||||
ActKey, LastActKey []byte
|
||||
}
|
||||
|
||||
func (t *DBTesting) post() {
|
||||
if t.PostFn != nil {
|
||||
t.PostFn(t)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *DBTesting) setAct(act DBAct, key []byte) {
|
||||
t.LastAct, t.Act = t.Act, act
|
||||
t.LastActKey, t.ActKey = t.ActKey, key
|
||||
}
|
||||
|
||||
func (t *DBTesting) text() string {
|
||||
return fmt.Sprintf("last action was <%v> %q, <%v> %q", t.LastAct, t.LastActKey, t.Act, t.ActKey)
|
||||
}
|
||||
|
||||
func (t *DBTesting) Text() string {
|
||||
return "DBTesting " + t.text()
|
||||
}
|
||||
|
||||
func (t *DBTesting) TestPresentKV(key, value []byte) {
|
||||
rvalue, err := t.DB.TestGet(key)
|
||||
Expect(err).ShouldNot(HaveOccurred(), "Get on key %q, %s", key, t.text())
|
||||
Expect(rvalue).Should(Equal(value), "Value for key %q, %s", key, t.text())
|
||||
}
|
||||
|
||||
func (t *DBTesting) TestAllPresent() {
|
||||
t.Present.IterateShuffled(t.Rand, func(i int, key, value []byte) {
|
||||
t.TestPresentKV(key, value)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *DBTesting) TestDeletedKey(key []byte) {
|
||||
_, err := t.DB.TestGet(key)
|
||||
Expect(err).Should(Equal(util.ErrNotFound), "Get on deleted key %q, %s", key, t.text())
|
||||
}
|
||||
|
||||
func (t *DBTesting) TestAllDeleted() {
|
||||
t.Deleted.IterateShuffled(t.Rand, func(i int, key, value []byte) {
|
||||
t.TestDeletedKey(key)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *DBTesting) TestAll() {
|
||||
dn := t.Deleted.Len()
|
||||
pn := t.Present.Len()
|
||||
ShuffledIndex(t.Rand, dn+pn, 1, func(i int) {
|
||||
if i >= dn {
|
||||
key, value := t.Present.Index(i - dn)
|
||||
t.TestPresentKV(key, value)
|
||||
} else {
|
||||
t.TestDeletedKey(t.Deleted.KeyAt(i))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *DBTesting) Put(key, value []byte) {
|
||||
if new := t.Present.PutU(key, value); new {
|
||||
t.setAct(DBPut, key)
|
||||
} else {
|
||||
t.setAct(DBOverwrite, key)
|
||||
}
|
||||
t.Deleted.Delete(key)
|
||||
err := t.DB.TestPut(key, value)
|
||||
Expect(err).ShouldNot(HaveOccurred(), t.Text())
|
||||
t.TestPresentKV(key, value)
|
||||
t.post()
|
||||
}
|
||||
|
||||
func (t *DBTesting) PutRandom() bool {
|
||||
if t.Deleted.Len() > 0 {
|
||||
i := t.Rand.Intn(t.Deleted.Len())
|
||||
key, value := t.Deleted.Index(i)
|
||||
t.Put(key, value)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *DBTesting) Delete(key []byte) {
|
||||
if exist, value := t.Present.Delete(key); exist {
|
||||
t.setAct(DBDelete, key)
|
||||
t.Deleted.PutU(key, value)
|
||||
} else {
|
||||
t.setAct(DBDeleteNA, key)
|
||||
}
|
||||
err := t.DB.TestDelete(key)
|
||||
Expect(err).ShouldNot(HaveOccurred(), t.Text())
|
||||
t.TestDeletedKey(key)
|
||||
t.post()
|
||||
}
|
||||
|
||||
func (t *DBTesting) DeleteRandom() bool {
|
||||
if t.Present.Len() > 0 {
|
||||
i := t.Rand.Intn(t.Present.Len())
|
||||
t.Delete(t.Present.KeyAt(i))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *DBTesting) RandomAct(round int) {
|
||||
for i := 0; i < round; i++ {
|
||||
if t.Rand.Int()%2 == 0 {
|
||||
t.PutRandom()
|
||||
} else {
|
||||
t.DeleteRandom()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func DoDBTesting(t *DBTesting) {
|
||||
if t.Rand == nil {
|
||||
t.Rand = NewRand()
|
||||
}
|
||||
|
||||
t.DeleteRandom()
|
||||
t.PutRandom()
|
||||
t.DeleteRandom()
|
||||
t.DeleteRandom()
|
||||
for i := t.Deleted.Len() / 2; i >= 0; i-- {
|
||||
t.PutRandom()
|
||||
}
|
||||
t.RandomAct((t.Deleted.Len() + t.Present.Len()) * 10)
|
||||
|
||||
// Additional iterator testing
|
||||
if db, ok := t.DB.(NewIterator); ok {
|
||||
iter := db.TestNewIterator(nil)
|
||||
Expect(iter.Error()).NotTo(HaveOccurred())
|
||||
|
||||
it := IteratorTesting{
|
||||
KeyValue: t.Present,
|
||||
Iter: iter,
|
||||
}
|
||||
|
||||
DoIteratorTesting(&it)
|
||||
}
|
||||
}
|
||||
-136
@@ -1,136 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
|
||||
if rnd == nil {
|
||||
rnd = NewRand()
|
||||
}
|
||||
|
||||
if db, ok := p.(Find); ok {
|
||||
It("Should find all keys with Find", func() {
|
||||
ShuffledIndex(nil, kv.Len(), 1, func(i int) {
|
||||
key_, key, value := kv.IndexInexact(i)
|
||||
|
||||
// Using exact key.
|
||||
rkey, rvalue, err := db.TestFind(key)
|
||||
Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
|
||||
Expect(rkey).Should(Equal(key), "Key")
|
||||
Expect(rvalue).Should(Equal(value), "Value for key %q", key)
|
||||
|
||||
// Using inexact key.
|
||||
rkey, rvalue, err = db.TestFind(key_)
|
||||
Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q)", key_, key)
|
||||
Expect(rkey).Should(Equal(key))
|
||||
Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
|
||||
})
|
||||
})
|
||||
|
||||
It("Should return error if the key is not present", func() {
|
||||
var key []byte
|
||||
if kv.Len() > 0 {
|
||||
key_, _ := kv.Index(kv.Len() - 1)
|
||||
key = BytesAfter(key_)
|
||||
}
|
||||
rkey, _, err := db.TestFind(key)
|
||||
Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
|
||||
Expect(err).Should(Equal(util.ErrNotFound))
|
||||
})
|
||||
}
|
||||
|
||||
if db, ok := p.(Get); ok {
|
||||
It("Should only find exact key with Get", func() {
|
||||
ShuffledIndex(nil, kv.Len(), 1, func(i int) {
|
||||
key_, key, value := kv.IndexInexact(i)
|
||||
|
||||
// Using exact key.
|
||||
rvalue, err := db.TestGet(key)
|
||||
Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
|
||||
Expect(rvalue).Should(Equal(value), "Value for key %q", key)
|
||||
|
||||
// Using inexact key.
|
||||
if len(key_) > 0 {
|
||||
_, err = db.TestGet(key_)
|
||||
Expect(err).Should(HaveOccurred(), "Error for key %q", key_)
|
||||
Expect(err).Should(Equal(util.ErrNotFound))
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
if db, ok := p.(NewIterator); ok {
|
||||
TestIter := func(r *util.Range, _kv KeyValue) {
|
||||
iter := db.TestNewIterator(r)
|
||||
Expect(iter.Error()).ShouldNot(HaveOccurred())
|
||||
|
||||
t := IteratorTesting{
|
||||
KeyValue: _kv,
|
||||
Iter: iter,
|
||||
}
|
||||
|
||||
DoIteratorTesting(&t)
|
||||
}
|
||||
|
||||
It("Should iterates and seeks correctly", func(done Done) {
|
||||
TestIter(nil, kv.Clone())
|
||||
done <- true
|
||||
}, 3.0)
|
||||
|
||||
RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
|
||||
type slice struct {
|
||||
r *util.Range
|
||||
start, limit int
|
||||
}
|
||||
|
||||
key_, _, _ := kv.IndexInexact(i)
|
||||
for _, x := range []slice{
|
||||
{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
|
||||
{&util.Range{Start: nil, Limit: key_}, 0, i},
|
||||
} {
|
||||
It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
|
||||
TestIter(x.r, kv.Slice(x.start, x.limit))
|
||||
done <- true
|
||||
}, 3.0)
|
||||
}
|
||||
})
|
||||
|
||||
RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
|
||||
It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
|
||||
r := kv.Range(start, limit)
|
||||
TestIter(&r, kv.Slice(start, limit))
|
||||
done <- true
|
||||
}, 3.0)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
|
||||
Test := func(kv *KeyValue) func() {
|
||||
return func() {
|
||||
db := body(*kv)
|
||||
KeyValueTesting(rnd, db, *kv)
|
||||
}
|
||||
}
|
||||
|
||||
Describe("with no key/value (empty)", Test(&KeyValue{}))
|
||||
Describe("with empty key", Test(KeyValue_EmptyKey()))
|
||||
Describe("with empty value", Test(KeyValue_EmptyValue()))
|
||||
Describe("with one key/value", Test(KeyValue_OneKeyValue()))
|
||||
Describe("with big value", Test(KeyValue_BigValue()))
|
||||
Describe("with special key", Test(KeyValue_SpecialKey()))
|
||||
Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue()))
|
||||
}
|
||||
-585
@@ -1,585 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var (
|
||||
storageMu sync.Mutex
|
||||
storageUseFS bool = true
|
||||
storageKeepFS bool = false
|
||||
storageNum int
|
||||
)
|
||||
|
||||
type StorageMode int
|
||||
|
||||
const (
|
||||
ModeOpen StorageMode = 1 << iota
|
||||
ModeCreate
|
||||
ModeRemove
|
||||
ModeRead
|
||||
ModeWrite
|
||||
ModeSync
|
||||
ModeClose
|
||||
)
|
||||
|
||||
const (
|
||||
modeOpen = iota
|
||||
modeCreate
|
||||
modeRemove
|
||||
modeRead
|
||||
modeWrite
|
||||
modeSync
|
||||
modeClose
|
||||
|
||||
modeCount
|
||||
)
|
||||
|
||||
const (
|
||||
typeManifest = iota
|
||||
typeJournal
|
||||
typeTable
|
||||
typeTemp
|
||||
|
||||
typeCount
|
||||
)
|
||||
|
||||
const flattenCount = modeCount * typeCount
|
||||
|
||||
func flattenType(m StorageMode, t storage.FileType) int {
|
||||
var x int
|
||||
switch m {
|
||||
case ModeOpen:
|
||||
x = modeOpen
|
||||
case ModeCreate:
|
||||
x = modeCreate
|
||||
case ModeRemove:
|
||||
x = modeRemove
|
||||
case ModeRead:
|
||||
x = modeRead
|
||||
case ModeWrite:
|
||||
x = modeWrite
|
||||
case ModeSync:
|
||||
x = modeSync
|
||||
case ModeClose:
|
||||
x = modeClose
|
||||
default:
|
||||
panic("invalid storage mode")
|
||||
}
|
||||
x *= typeCount
|
||||
switch t {
|
||||
case storage.TypeManifest:
|
||||
return x + typeManifest
|
||||
case storage.TypeJournal:
|
||||
return x + typeJournal
|
||||
case storage.TypeTable:
|
||||
return x + typeTable
|
||||
case storage.TypeTemp:
|
||||
return x + typeTemp
|
||||
default:
|
||||
panic("invalid file type")
|
||||
}
|
||||
}
|
||||
|
||||
func listFlattenType(m StorageMode, t storage.FileType) []int {
|
||||
ret := make([]int, 0, flattenCount)
|
||||
add := func(x int) {
|
||||
x *= typeCount
|
||||
switch {
|
||||
case t&storage.TypeManifest != 0:
|
||||
ret = append(ret, x+typeManifest)
|
||||
case t&storage.TypeJournal != 0:
|
||||
ret = append(ret, x+typeJournal)
|
||||
case t&storage.TypeTable != 0:
|
||||
ret = append(ret, x+typeTable)
|
||||
case t&storage.TypeTemp != 0:
|
||||
ret = append(ret, x+typeTemp)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case m&ModeOpen != 0:
|
||||
add(modeOpen)
|
||||
case m&ModeCreate != 0:
|
||||
add(modeCreate)
|
||||
case m&ModeRemove != 0:
|
||||
add(modeRemove)
|
||||
case m&ModeRead != 0:
|
||||
add(modeRead)
|
||||
case m&ModeWrite != 0:
|
||||
add(modeWrite)
|
||||
case m&ModeSync != 0:
|
||||
add(modeSync)
|
||||
case m&ModeClose != 0:
|
||||
add(modeClose)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func packFile(num uint64, t storage.FileType) uint64 {
|
||||
if num>>(64-typeCount) != 0 {
|
||||
panic("overflow")
|
||||
}
|
||||
return num<<typeCount | uint64(t)
|
||||
}
|
||||
|
||||
func unpackFile(x uint64) (uint64, storage.FileType) {
|
||||
return x >> typeCount, storage.FileType(x) & storage.TypeAll
|
||||
}
|
||||
|
||||
type emulatedError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (err emulatedError) Error() string {
|
||||
return fmt.Sprintf("emulated storage error: %v", err.err)
|
||||
}
|
||||
|
||||
type storageLock struct {
|
||||
s *Storage
|
||||
r util.Releaser
|
||||
}
|
||||
|
||||
func (l storageLock) Release() {
|
||||
l.r.Release()
|
||||
l.s.logI("storage lock released")
|
||||
}
|
||||
|
||||
type reader struct {
|
||||
f *file
|
||||
storage.Reader
|
||||
}
|
||||
|
||||
func (r *reader) Read(p []byte) (n int, err error) {
|
||||
err = r.f.s.emulateError(ModeRead, r.f.Type())
|
||||
if err == nil {
|
||||
r.f.s.stall(ModeRead, r.f.Type())
|
||||
n, err = r.Reader.Read(p)
|
||||
}
|
||||
r.f.s.count(ModeRead, r.f.Type(), n)
|
||||
if err != nil && err != io.EOF {
|
||||
r.f.s.logI("read error, num=%d type=%v n=%d err=%v", r.f.Num(), r.f.Type(), n, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (r *reader) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
err = r.f.s.emulateError(ModeRead, r.f.Type())
|
||||
if err == nil {
|
||||
r.f.s.stall(ModeRead, r.f.Type())
|
||||
n, err = r.Reader.ReadAt(p, off)
|
||||
}
|
||||
r.f.s.count(ModeRead, r.f.Type(), n)
|
||||
if err != nil && err != io.EOF {
|
||||
r.f.s.logI("readAt error, num=%d type=%v offset=%d n=%d err=%v", r.f.Num(), r.f.Type(), off, n, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (r *reader) Close() (err error) {
|
||||
return r.f.doClose(r.Reader)
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
f *file
|
||||
storage.Writer
|
||||
}
|
||||
|
||||
func (w *writer) Write(p []byte) (n int, err error) {
|
||||
err = w.f.s.emulateError(ModeWrite, w.f.Type())
|
||||
if err == nil {
|
||||
w.f.s.stall(ModeWrite, w.f.Type())
|
||||
n, err = w.Writer.Write(p)
|
||||
}
|
||||
w.f.s.count(ModeWrite, w.f.Type(), n)
|
||||
if err != nil && err != io.EOF {
|
||||
w.f.s.logI("write error, num=%d type=%v n=%d err=%v", w.f.Num(), w.f.Type(), n, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (w *writer) Sync() (err error) {
|
||||
err = w.f.s.emulateError(ModeSync, w.f.Type())
|
||||
if err == nil {
|
||||
w.f.s.stall(ModeSync, w.f.Type())
|
||||
err = w.Writer.Sync()
|
||||
}
|
||||
w.f.s.count(ModeSync, w.f.Type(), 0)
|
||||
if err != nil {
|
||||
w.f.s.logI("sync error, num=%d type=%v err=%v", w.f.Num(), w.f.Type(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (w *writer) Close() (err error) {
|
||||
return w.f.doClose(w.Writer)
|
||||
}
|
||||
|
||||
type file struct {
|
||||
s *Storage
|
||||
storage.File
|
||||
}
|
||||
|
||||
func (f *file) pack() uint64 {
|
||||
return packFile(f.Num(), f.Type())
|
||||
}
|
||||
|
||||
func (f *file) assertOpen() {
|
||||
ExpectWithOffset(2, f.s.opens).NotTo(HaveKey(f.pack()), "File open, num=%d type=%v writer=%v", f.Num(), f.Type(), f.s.opens[f.pack()])
|
||||
}
|
||||
|
||||
func (f *file) doClose(closer io.Closer) (err error) {
|
||||
err = f.s.emulateError(ModeClose, f.Type())
|
||||
if err == nil {
|
||||
f.s.stall(ModeClose, f.Type())
|
||||
}
|
||||
f.s.mu.Lock()
|
||||
defer f.s.mu.Unlock()
|
||||
if err == nil {
|
||||
ExpectWithOffset(2, f.s.opens).To(HaveKey(f.pack()), "File closed, num=%d type=%v", f.Num(), f.Type())
|
||||
err = closer.Close()
|
||||
}
|
||||
f.s.countNB(ModeClose, f.Type(), 0)
|
||||
writer := f.s.opens[f.pack()]
|
||||
if err != nil {
|
||||
f.s.logISkip(1, "file close failed, num=%d type=%v writer=%v err=%v", f.Num(), f.Type(), writer, err)
|
||||
} else {
|
||||
f.s.logISkip(1, "file closed, num=%d type=%v writer=%v", f.Num(), f.Type(), writer)
|
||||
delete(f.s.opens, f.pack())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (f *file) Open() (r storage.Reader, err error) {
|
||||
err = f.s.emulateError(ModeOpen, f.Type())
|
||||
if err == nil {
|
||||
f.s.stall(ModeOpen, f.Type())
|
||||
}
|
||||
f.s.mu.Lock()
|
||||
defer f.s.mu.Unlock()
|
||||
if err == nil {
|
||||
f.assertOpen()
|
||||
f.s.countNB(ModeOpen, f.Type(), 0)
|
||||
r, err = f.File.Open()
|
||||
}
|
||||
if err != nil {
|
||||
f.s.logI("file open failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
|
||||
} else {
|
||||
f.s.logI("file opened, num=%d type=%v", f.Num(), f.Type())
|
||||
f.s.opens[f.pack()] = false
|
||||
r = &reader{f, r}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (f *file) Create() (w storage.Writer, err error) {
|
||||
err = f.s.emulateError(ModeCreate, f.Type())
|
||||
if err == nil {
|
||||
f.s.stall(ModeCreate, f.Type())
|
||||
}
|
||||
f.s.mu.Lock()
|
||||
defer f.s.mu.Unlock()
|
||||
if err == nil {
|
||||
f.assertOpen()
|
||||
f.s.countNB(ModeCreate, f.Type(), 0)
|
||||
w, err = f.File.Create()
|
||||
}
|
||||
if err != nil {
|
||||
f.s.logI("file create failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
|
||||
} else {
|
||||
f.s.logI("file created, num=%d type=%v", f.Num(), f.Type())
|
||||
f.s.opens[f.pack()] = true
|
||||
w = &writer{f, w}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (f *file) Remove() (err error) {
|
||||
err = f.s.emulateError(ModeRemove, f.Type())
|
||||
if err == nil {
|
||||
f.s.stall(ModeRemove, f.Type())
|
||||
}
|
||||
f.s.mu.Lock()
|
||||
defer f.s.mu.Unlock()
|
||||
if err == nil {
|
||||
f.assertOpen()
|
||||
f.s.countNB(ModeRemove, f.Type(), 0)
|
||||
err = f.File.Remove()
|
||||
}
|
||||
if err != nil {
|
||||
f.s.logI("file remove failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
|
||||
} else {
|
||||
f.s.logI("file removed, num=%d type=%v", f.Num(), f.Type())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type Storage struct {
|
||||
storage.Storage
|
||||
closeFn func() error
|
||||
|
||||
lmu sync.Mutex
|
||||
lb bytes.Buffer
|
||||
|
||||
mu sync.Mutex
|
||||
// Open files, true=writer, false=reader
|
||||
opens map[uint64]bool
|
||||
counters [flattenCount]int
|
||||
bytesCounter [flattenCount]int64
|
||||
emulatedError [flattenCount]error
|
||||
stallCond sync.Cond
|
||||
stalled [flattenCount]bool
|
||||
}
|
||||
|
||||
func (s *Storage) log(skip int, str string) {
|
||||
s.lmu.Lock()
|
||||
defer s.lmu.Unlock()
|
||||
_, file, line, ok := runtime.Caller(skip + 2)
|
||||
if ok {
|
||||
// Truncate file name at last file name separator.
|
||||
if index := strings.LastIndex(file, "/"); index >= 0 {
|
||||
file = file[index+1:]
|
||||
} else if index = strings.LastIndex(file, "\\"); index >= 0 {
|
||||
file = file[index+1:]
|
||||
}
|
||||
} else {
|
||||
file = "???"
|
||||
line = 1
|
||||
}
|
||||
fmt.Fprintf(&s.lb, "%s:%d: ", file, line)
|
||||
lines := strings.Split(str, "\n")
|
||||
if l := len(lines); l > 1 && lines[l-1] == "" {
|
||||
lines = lines[:l-1]
|
||||
}
|
||||
for i, line := range lines {
|
||||
if i > 0 {
|
||||
s.lb.WriteString("\n\t")
|
||||
}
|
||||
s.lb.WriteString(line)
|
||||
}
|
||||
s.lb.WriteByte('\n')
|
||||
}
|
||||
|
||||
func (s *Storage) logISkip(skip int, format string, args ...interface{}) {
|
||||
pc, _, _, ok := runtime.Caller(skip + 1)
|
||||
if ok {
|
||||
if f := runtime.FuncForPC(pc); f != nil {
|
||||
fname := f.Name()
|
||||
if index := strings.LastIndex(fname, "."); index >= 0 {
|
||||
fname = fname[index+1:]
|
||||
}
|
||||
format = fname + ": " + format
|
||||
}
|
||||
}
|
||||
s.log(skip+1, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (s *Storage) logI(format string, args ...interface{}) {
|
||||
s.logISkip(1, format, args...)
|
||||
}
|
||||
|
||||
func (s *Storage) Log(str string) {
|
||||
s.log(1, "Log: "+str)
|
||||
}
|
||||
|
||||
func (s *Storage) Lock() (r util.Releaser, err error) {
|
||||
r, err = s.Storage.Lock()
|
||||
if err != nil {
|
||||
s.logI("storage locking failed, err=%v", err)
|
||||
} else {
|
||||
s.logI("storage locked")
|
||||
r = storageLock{s, r}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Storage) GetFile(num uint64, t storage.FileType) storage.File {
|
||||
return &file{s, s.Storage.GetFile(num, t)}
|
||||
}
|
||||
|
||||
func (s *Storage) GetFiles(t storage.FileType) (files []storage.File, err error) {
|
||||
rfiles, err := s.Storage.GetFiles(t)
|
||||
if err != nil {
|
||||
s.logI("get files failed, err=%v", err)
|
||||
return
|
||||
}
|
||||
files = make([]storage.File, len(rfiles))
|
||||
for i, f := range rfiles {
|
||||
files[i] = &file{s, f}
|
||||
}
|
||||
s.logI("get files, type=0x%x count=%d", int(t), len(files))
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Storage) GetManifest() (f storage.File, err error) {
|
||||
manifest, err := s.Storage.GetManifest()
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
s.logI("get manifest failed, err=%v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.logI("get manifest, num=%d", manifest.Num())
|
||||
return &file{s, manifest}, nil
|
||||
}
|
||||
|
||||
func (s *Storage) SetManifest(f storage.File) error {
|
||||
f_, ok := f.(*file)
|
||||
ExpectWithOffset(1, ok).To(BeTrue())
|
||||
ExpectWithOffset(1, f_.Type()).To(Equal(storage.TypeManifest))
|
||||
err := s.Storage.SetManifest(f_.File)
|
||||
if err != nil {
|
||||
s.logI("set manifest failed, err=%v", err)
|
||||
} else {
|
||||
s.logI("set manifest, num=%d", f_.Num())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Storage) openFiles() string {
|
||||
out := "Open files:"
|
||||
for x, writer := range s.opens {
|
||||
num, t := unpackFile(x)
|
||||
out += fmt.Sprintf("\n · num=%d type=%v writer=%v", num, t, writer)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Storage) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ExpectWithOffset(1, s.opens).To(BeEmpty(), s.openFiles())
|
||||
err := s.Storage.Close()
|
||||
if err != nil {
|
||||
s.logI("storage closing failed, err=%v", err)
|
||||
} else {
|
||||
s.logI("storage closed")
|
||||
}
|
||||
if s.closeFn != nil {
|
||||
if err1 := s.closeFn(); err1 != nil {
|
||||
s.logI("close func error, err=%v", err1)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Storage) countNB(m StorageMode, t storage.FileType, n int) {
|
||||
s.counters[flattenType(m, t)]++
|
||||
s.bytesCounter[flattenType(m, t)] += int64(n)
|
||||
}
|
||||
|
||||
func (s *Storage) count(m StorageMode, t storage.FileType, n int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.countNB(m, t, n)
|
||||
}
|
||||
|
||||
func (s *Storage) ResetCounter(m StorageMode, t storage.FileType) {
|
||||
for _, x := range listFlattenType(m, t) {
|
||||
s.counters[x] = 0
|
||||
s.bytesCounter[x] = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Storage) Counter(m StorageMode, t storage.FileType) (count int, bytes int64) {
|
||||
for _, x := range listFlattenType(m, t) {
|
||||
count += s.counters[x]
|
||||
bytes += s.bytesCounter[x]
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Storage) emulateError(m StorageMode, t storage.FileType) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
err := s.emulatedError[flattenType(m, t)]
|
||||
if err != nil {
|
||||
return emulatedError{err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Storage) EmulateError(m StorageMode, t storage.FileType, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, x := range listFlattenType(m, t) {
|
||||
s.emulatedError[x] = err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Storage) stall(m StorageMode, t storage.FileType) {
|
||||
x := flattenType(m, t)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for s.stalled[x] {
|
||||
s.stallCond.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Storage) Stall(m StorageMode, t storage.FileType) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, x := range listFlattenType(m, t) {
|
||||
s.stalled[x] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Storage) Release(m StorageMode, t storage.FileType) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, x := range listFlattenType(m, t) {
|
||||
s.stalled[x] = false
|
||||
}
|
||||
s.stallCond.Broadcast()
|
||||
}
|
||||
|
||||
func NewStorage() *Storage {
|
||||
var stor storage.Storage
|
||||
var closeFn func() error
|
||||
if storageUseFS {
|
||||
for {
|
||||
storageMu.Lock()
|
||||
num := storageNum
|
||||
storageNum++
|
||||
storageMu.Unlock()
|
||||
path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
stor, err = storage.OpenFile(path)
|
||||
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "creating storage at %s", path)
|
||||
closeFn = func() error {
|
||||
if storageKeepFS {
|
||||
return nil
|
||||
}
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stor = storage.NewMemStorage()
|
||||
}
|
||||
s := &Storage{
|
||||
Storage: stor,
|
||||
closeFn: closeFn,
|
||||
opens: make(map[uint64]bool),
|
||||
}
|
||||
s.stallCond.L = &s.mu
|
||||
return s
|
||||
}
|
||||
-157
@@ -1,157 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/onsi/ginkgo/config"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
)
|
||||
|
||||
var (
|
||||
runfn = make(map[string][]func())
|
||||
runmu sync.Mutex
|
||||
)
|
||||
|
||||
func Defer(args ...interface{}) bool {
|
||||
var (
|
||||
group string
|
||||
fn func()
|
||||
)
|
||||
for _, arg := range args {
|
||||
v := reflect.ValueOf(arg)
|
||||
switch v.Kind() {
|
||||
case reflect.String:
|
||||
group = v.String()
|
||||
case reflect.Func:
|
||||
r := reflect.ValueOf(&fn).Elem()
|
||||
r.Set(v)
|
||||
}
|
||||
}
|
||||
if fn != nil {
|
||||
runmu.Lock()
|
||||
runfn[group] = append(runfn[group], fn)
|
||||
runmu.Unlock()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func RunDefer(groups ...string) bool {
|
||||
if len(groups) == 0 {
|
||||
groups = append(groups, "")
|
||||
}
|
||||
runmu.Lock()
|
||||
var runfn_ []func()
|
||||
for _, group := range groups {
|
||||
runfn_ = append(runfn_, runfn[group]...)
|
||||
delete(runfn, group)
|
||||
}
|
||||
runmu.Unlock()
|
||||
for _, fn := range runfn_ {
|
||||
fn()
|
||||
}
|
||||
return runfn_ != nil
|
||||
}
|
||||
|
||||
func RandomSeed() int64 {
|
||||
if !flag.Parsed() {
|
||||
panic("random seed not initialized")
|
||||
}
|
||||
return config.GinkgoConfig.RandomSeed
|
||||
}
|
||||
|
||||
func NewRand() *rand.Rand {
|
||||
return rand.New(rand.NewSource(RandomSeed()))
|
||||
}
|
||||
|
||||
var cmp = comparer.DefaultComparer
|
||||
|
||||
func BytesSeparator(a, b []byte) []byte {
|
||||
if bytes.Equal(a, b) {
|
||||
return b
|
||||
}
|
||||
i, n := 0, len(a)
|
||||
if n > len(b) {
|
||||
n = len(b)
|
||||
}
|
||||
for ; i < n && (a[i] == b[i]); i++ {
|
||||
}
|
||||
x := append([]byte{}, a[:i]...)
|
||||
if i < n {
|
||||
if c := a[i] + 1; c < b[i] {
|
||||
return append(x, c)
|
||||
}
|
||||
x = append(x, a[i])
|
||||
i++
|
||||
}
|
||||
for ; i < len(a); i++ {
|
||||
if c := a[i]; c < 0xff {
|
||||
return append(x, c+1)
|
||||
} else {
|
||||
x = append(x, c)
|
||||
}
|
||||
}
|
||||
if len(b) > i && b[i] > 0 {
|
||||
return append(x, b[i]-1)
|
||||
}
|
||||
return append(x, 'x')
|
||||
}
|
||||
|
||||
func BytesAfter(b []byte) []byte {
|
||||
var x []byte
|
||||
for _, c := range b {
|
||||
if c < 0xff {
|
||||
return append(x, c+1)
|
||||
} else {
|
||||
x = append(x, c)
|
||||
}
|
||||
}
|
||||
return append(x, 'x')
|
||||
}
|
||||
|
||||
func RandomIndex(rnd *rand.Rand, n, round int, fn func(i int)) {
|
||||
if rnd == nil {
|
||||
rnd = NewRand()
|
||||
}
|
||||
for x := 0; x < round; x++ {
|
||||
fn(rnd.Intn(n))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func ShuffledIndex(rnd *rand.Rand, n, round int, fn func(i int)) {
|
||||
if rnd == nil {
|
||||
rnd = NewRand()
|
||||
}
|
||||
for x := 0; x < round; x++ {
|
||||
for _, i := range rnd.Perm(n) {
|
||||
fn(i)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) {
|
||||
if rnd == nil {
|
||||
rnd = NewRand()
|
||||
}
|
||||
for x := 0; x < round; x++ {
|
||||
start := rnd.Intn(n)
|
||||
length := 0
|
||||
if j := n - start; j > 0 {
|
||||
length = rnd.Intn(j)
|
||||
}
|
||||
fn(start, start+length)
|
||||
}
|
||||
return
|
||||
}
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/testutil"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type testingDB struct {
|
||||
*DB
|
||||
ro *opt.ReadOptions
|
||||
wo *opt.WriteOptions
|
||||
stor *testutil.Storage
|
||||
}
|
||||
|
||||
func (t *testingDB) TestPut(key []byte, value []byte) error {
|
||||
return t.Put(key, value, t.wo)
|
||||
}
|
||||
|
||||
func (t *testingDB) TestDelete(key []byte) error {
|
||||
return t.Delete(key, t.wo)
|
||||
}
|
||||
|
||||
func (t *testingDB) TestGet(key []byte) (value []byte, err error) {
|
||||
return t.Get(key, t.ro)
|
||||
}
|
||||
|
||||
func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator {
|
||||
return t.NewIterator(slice, t.ro)
|
||||
}
|
||||
|
||||
func (t *testingDB) TestClose() {
|
||||
err := t.Close()
|
||||
ExpectWithOffset(1, err).NotTo(HaveOccurred())
|
||||
err = t.stor.Close()
|
||||
ExpectWithOffset(1, err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB {
|
||||
stor := testutil.NewStorage()
|
||||
db, err := Open(stor, o)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
return &testingDB{
|
||||
DB: db,
|
||||
ro: ro,
|
||||
wo: wo,
|
||||
stor: stor,
|
||||
}
|
||||
}
|
||||
-91
@@ -1,91 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
func shorten(str string) string {
|
||||
if len(str) <= 4 {
|
||||
return str
|
||||
}
|
||||
return str[:1] + ".." + str[len(str)-1:]
|
||||
}
|
||||
|
||||
var bunits = [...]string{"", "Ki", "Mi", "Gi"}
|
||||
|
||||
func shortenb(bytes int) string {
|
||||
i := 0
|
||||
for ; bytes > 1024 && i < 4; i++ {
|
||||
bytes /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%d%sB", bytes, bunits[i])
|
||||
}
|
||||
|
||||
func sshortenb(bytes int) string {
|
||||
if bytes == 0 {
|
||||
return "~"
|
||||
}
|
||||
sign := "+"
|
||||
if bytes < 0 {
|
||||
sign = "-"
|
||||
bytes *= -1
|
||||
}
|
||||
i := 0
|
||||
for ; bytes > 1024 && i < 4; i++ {
|
||||
bytes /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i])
|
||||
}
|
||||
|
||||
func sint(x int) string {
|
||||
if x == 0 {
|
||||
return "~"
|
||||
}
|
||||
sign := "+"
|
||||
if x < 0 {
|
||||
sign = "-"
|
||||
x *= -1
|
||||
}
|
||||
return fmt.Sprintf("%s%d", sign, x)
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type files []storage.File
|
||||
|
||||
func (p files) Len() int {
|
||||
return len(p)
|
||||
}
|
||||
|
||||
func (p files) Less(i, j int) bool {
|
||||
return p[i].Num() < p[j].Num()
|
||||
}
|
||||
|
||||
func (p files) Swap(i, j int) {
|
||||
p[i], p[j] = p[j], p[i]
|
||||
}
|
||||
|
||||
func (p files) sort() {
|
||||
sort.Sort(p)
|
||||
}
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// Hash return hash of the given data.
|
||||
func Hash(data []byte, seed uint32) uint32 {
|
||||
// Similar to murmur hash
|
||||
var m uint32 = 0xc6a4a793
|
||||
var r uint32 = 24
|
||||
h := seed ^ (uint32(len(data)) * m)
|
||||
|
||||
buf := bytes.NewBuffer(data)
|
||||
for buf.Len() >= 4 {
|
||||
var w uint32
|
||||
binary.Read(buf, binary.LittleEndian, &w)
|
||||
h += w
|
||||
h *= m
|
||||
h ^= (h >> 16)
|
||||
}
|
||||
|
||||
rest := buf.Bytes()
|
||||
switch len(rest) {
|
||||
default:
|
||||
panic("not reached")
|
||||
case 3:
|
||||
h += uint32(rest[2]) << 16
|
||||
fallthrough
|
||||
case 2:
|
||||
h += uint32(rest[1]) << 8
|
||||
fallthrough
|
||||
case 1:
|
||||
h += uint32(rest[0])
|
||||
h *= m
|
||||
h ^= (h >> r)
|
||||
case 0:
|
||||
}
|
||||
|
||||
return h
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
// Range is a key range.
|
||||
type Range struct {
|
||||
// Start of the key range, include in the range.
|
||||
Start []byte
|
||||
|
||||
// Limit of the key range, not include in the range.
|
||||
Limit []byte
|
||||
}
|
||||
-49
@@ -1,49 +0,0 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package util provides utilities used throughout leveldb.
|
||||
package util
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("leveldb: not found")
|
||||
)
|
||||
|
||||
// Releaser is the interface that wraps the basic Release method.
|
||||
type Releaser interface {
|
||||
// Release releases associated resources. Release should always success
|
||||
// and can be called multipe times without causing error.
|
||||
Release()
|
||||
}
|
||||
|
||||
// ReleaseSetter is the interface that wraps the basic SetReleaser method.
|
||||
type ReleaseSetter interface {
|
||||
// SetReleaser associates the given releaser to the resources. The
|
||||
// releaser will be called once coresponding resources released.
|
||||
// Calling SetReleaser with nil will clear the releaser.
|
||||
SetReleaser(releaser Releaser)
|
||||
}
|
||||
|
||||
// BasicReleaser provides basic implementation of Releaser and ReleaseSetter.
|
||||
type BasicReleaser struct {
|
||||
releaser Releaser
|
||||
}
|
||||
|
||||
// Release implements Releaser.Release.
|
||||
func (r *BasicReleaser) Release() {
|
||||
if r.releaser != nil {
|
||||
r.releaser.Release()
|
||||
r.releaser = nil
|
||||
}
|
||||
}
|
||||
|
||||
// SetReleaser implements ReleaseSetter.SetReleaser.
|
||||
func (r *BasicReleaser) SetReleaser(releaser Releaser) {
|
||||
r.releaser = releaser
|
||||
}
|
||||
-428
@@ -1,428 +0,0 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var levelMaxSize [kNumLevels]float64
|
||||
|
||||
func init() {
|
||||
// Precompute max size of each level
|
||||
for level := range levelMaxSize {
|
||||
res := float64(10 * 1048576)
|
||||
for n := level; n > 1; n-- {
|
||||
res *= 10
|
||||
}
|
||||
levelMaxSize[level] = res
|
||||
}
|
||||
}
|
||||
|
||||
type tSet struct {
|
||||
level int
|
||||
table *tFile
|
||||
}
|
||||
|
||||
type version struct {
|
||||
s *session
|
||||
|
||||
tables [kNumLevels]tFiles
|
||||
|
||||
// Level that should be compacted next and its compaction score.
|
||||
// Score < 1 means compaction is not strictly needed. These fields
|
||||
// are initialized by ComputeCompaction()
|
||||
cLevel int
|
||||
cScore float64
|
||||
|
||||
cSeek unsafe.Pointer
|
||||
|
||||
ref int
|
||||
next *version
|
||||
}
|
||||
|
||||
func (v *version) release_NB() {
|
||||
v.ref--
|
||||
if v.ref > 0 {
|
||||
return
|
||||
}
|
||||
if v.ref < 0 {
|
||||
panic("negative version ref")
|
||||
}
|
||||
|
||||
s := v.s
|
||||
|
||||
tables := make(map[uint64]bool)
|
||||
for _, tt := range v.next.tables {
|
||||
for _, t := range tt {
|
||||
num := t.file.Num()
|
||||
tables[num] = true
|
||||
}
|
||||
}
|
||||
|
||||
for _, tt := range v.tables {
|
||||
for _, t := range tt {
|
||||
num := t.file.Num()
|
||||
if _, ok := tables[num]; !ok {
|
||||
s.tops.remove(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
v.next.release_NB()
|
||||
v.next = nil
|
||||
}
|
||||
|
||||
func (v *version) release() {
|
||||
v.s.vmu.Lock()
|
||||
v.release_NB()
|
||||
v.s.vmu.Unlock()
|
||||
}
|
||||
|
||||
func (v *version) get(key iKey, ro *opt.ReadOptions) (value []byte, cstate bool, err error) {
|
||||
s := v.s
|
||||
|
||||
ukey := key.ukey()
|
||||
|
||||
var tset *tSet
|
||||
tseek := true
|
||||
|
||||
// We can search level-by-level since entries never hop across
|
||||
// levels. Therefore we are guaranteed that if we find data
|
||||
// in an smaller level, later levels are irrelevant.
|
||||
for level, ts := range v.tables {
|
||||
if len(ts) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if level == 0 {
|
||||
// Level-0 files may overlap each other. Find all files that
|
||||
// overlap user_key and process them in order from newest to
|
||||
var tmp tFiles
|
||||
for _, t := range ts {
|
||||
if s.icmp.uCompare(ukey, t.min.ukey()) >= 0 &&
|
||||
s.icmp.uCompare(ukey, t.max.ukey()) <= 0 {
|
||||
tmp = append(tmp, t)
|
||||
}
|
||||
}
|
||||
|
||||
if len(tmp) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
tmp.sortByNum()
|
||||
ts = tmp
|
||||
} else {
|
||||
i := ts.searchMax(key, s.icmp)
|
||||
if i >= len(ts) || s.icmp.uCompare(ukey, ts[i].min.ukey()) < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
ts = ts[i : i+1]
|
||||
}
|
||||
|
||||
var l0found bool
|
||||
var l0seq uint64
|
||||
var l0type vType
|
||||
var l0value []byte
|
||||
for _, t := range ts {
|
||||
if tseek {
|
||||
if tset == nil {
|
||||
tset = &tSet{level, t}
|
||||
} else if tset.table.incrSeek() <= 0 {
|
||||
cstate = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
|
||||
tseek = false
|
||||
}
|
||||
}
|
||||
|
||||
var _rkey, rval []byte
|
||||
_rkey, rval, err = s.tops.get(t, key, ro)
|
||||
if err == ErrNotFound {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
rkey := iKey(_rkey)
|
||||
if seq, t, ok := rkey.parseNum(); ok {
|
||||
if s.icmp.uCompare(ukey, rkey.ukey()) == 0 {
|
||||
if level == 0 {
|
||||
if seq >= l0seq {
|
||||
l0found = true
|
||||
l0seq = seq
|
||||
l0type = t
|
||||
l0value = rval
|
||||
}
|
||||
} else {
|
||||
switch t {
|
||||
case tVal:
|
||||
value = rval
|
||||
case tDel:
|
||||
err = ErrNotFound
|
||||
default:
|
||||
panic("invalid type")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err = errors.New("leveldb: internal key corrupted")
|
||||
return
|
||||
}
|
||||
}
|
||||
if level == 0 && l0found {
|
||||
switch l0type {
|
||||
case tVal:
|
||||
value = l0value
|
||||
case tDel:
|
||||
err = ErrNotFound
|
||||
default:
|
||||
panic("invalid type")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err = ErrNotFound
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) {
|
||||
s := v.s
|
||||
|
||||
// Merge all level zero files together since they may overlap
|
||||
for _, t := range v.tables[0] {
|
||||
it := s.tops.newIterator(t, slice, ro)
|
||||
its = append(its, it)
|
||||
}
|
||||
|
||||
strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
|
||||
for _, tt := range v.tables[1:] {
|
||||
if len(tt) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
it := iterator.NewIndexedIterator(tt.newIndexIterator(s.tops, s.icmp, slice, ro), strict, true)
|
||||
its = append(its, it)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) newStaging() *versionStaging {
|
||||
return &versionStaging{base: v}
|
||||
}
|
||||
|
||||
// Spawn a new version based on this version.
|
||||
func (v *version) spawn(r *sessionRecord) *version {
|
||||
staging := v.newStaging()
|
||||
staging.commit(r)
|
||||
return staging.finish()
|
||||
}
|
||||
|
||||
func (v *version) fillRecord(r *sessionRecord) {
|
||||
for level, ts := range v.tables {
|
||||
for _, t := range ts {
|
||||
r.addTableFile(level, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (v *version) tLen(level int) int {
|
||||
return len(v.tables[level])
|
||||
}
|
||||
|
||||
func (v *version) offsetOf(key iKey) (n uint64, err error) {
|
||||
for level, tt := range v.tables {
|
||||
for _, t := range tt {
|
||||
if v.s.icmp.Compare(t.max, key) <= 0 {
|
||||
// Entire file is before "key", so just add the file size
|
||||
n += t.size
|
||||
} else if v.s.icmp.Compare(t.min, key) > 0 {
|
||||
// Entire file is after "key", so ignore
|
||||
if level > 0 {
|
||||
// Files other than level 0 are sorted by meta->min, so
|
||||
// no further files in this level will contain data for
|
||||
// "key".
|
||||
break
|
||||
}
|
||||
} else {
|
||||
// "key" falls in the range for this table. Add the
|
||||
// approximate offset of "key" within the table.
|
||||
var nn uint64
|
||||
nn, err = v.s.tops.offsetOf(t, key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n += nn
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) pickLevel(min, max []byte) (level int) {
|
||||
if !v.tables[0].isOverlaps(min, max, false, v.s.icmp) {
|
||||
var r tFiles
|
||||
for ; level < kMaxMemCompactLevel; level++ {
|
||||
if v.tables[level+1].isOverlaps(min, max, true, v.s.icmp) {
|
||||
break
|
||||
}
|
||||
v.tables[level+2].getOverlaps(min, max, &r, true, v.s.icmp.ucmp)
|
||||
if r.size() > kMaxGrandParentOverlapBytes {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) computeCompaction() {
|
||||
// Precomputed best level for next compaction
|
||||
var bestLevel int = -1
|
||||
var bestScore float64 = -1
|
||||
|
||||
for level, ff := range v.tables {
|
||||
var score float64
|
||||
if level == 0 {
|
||||
// We treat level-0 specially by bounding the number of files
|
||||
// instead of number of bytes for two reasons:
|
||||
//
|
||||
// (1) With larger write-buffer sizes, it is nice not to do too
|
||||
// many level-0 compactions.
|
||||
//
|
||||
// (2) The files in level-0 are merged on every read and
|
||||
// therefore we wish to avoid too many files when the individual
|
||||
// file size is small (perhaps because of a small write-buffer
|
||||
// setting, or very high compression ratios, or lots of
|
||||
// overwrites/deletions).
|
||||
score = float64(len(ff)) / kL0_CompactionTrigger
|
||||
} else {
|
||||
score = float64(ff.size()) / levelMaxSize[level]
|
||||
}
|
||||
|
||||
if score > bestScore {
|
||||
bestLevel = level
|
||||
bestScore = score
|
||||
}
|
||||
}
|
||||
|
||||
v.cLevel = bestLevel
|
||||
v.cScore = bestScore
|
||||
}
|
||||
|
||||
func (v *version) needCompaction() bool {
|
||||
return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil
|
||||
}
|
||||
|
||||
type versionStaging struct {
|
||||
base *version
|
||||
tables [kNumLevels]struct {
|
||||
added map[uint64]ntRecord
|
||||
deleted map[uint64]struct{}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *versionStaging) commit(r *sessionRecord) {
|
||||
btt := p.base.tables
|
||||
|
||||
// deleted tables
|
||||
for _, tr := range r.deletedTables {
|
||||
tm := &(p.tables[tr.level])
|
||||
|
||||
bt := btt[tr.level]
|
||||
if len(bt) > 0 {
|
||||
if tm.deleted == nil {
|
||||
tm.deleted = make(map[uint64]struct{})
|
||||
}
|
||||
tm.deleted[tr.num] = struct{}{}
|
||||
}
|
||||
|
||||
if tm.added != nil {
|
||||
delete(tm.added, tr.num)
|
||||
}
|
||||
}
|
||||
|
||||
// new tables
|
||||
for _, tr := range r.addedTables {
|
||||
tm := &(p.tables[tr.level])
|
||||
|
||||
if tm.added == nil {
|
||||
tm.added = make(map[uint64]ntRecord)
|
||||
}
|
||||
tm.added[tr.num] = tr
|
||||
|
||||
if tm.deleted != nil {
|
||||
delete(tm.deleted, tr.num)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *versionStaging) finish() *version {
|
||||
s := p.base.s
|
||||
btt := p.base.tables
|
||||
|
||||
// build new version
|
||||
nv := &version{s: s}
|
||||
for level, tm := range p.tables {
|
||||
bt := btt[level]
|
||||
|
||||
n := len(bt) + len(tm.added) - len(tm.deleted)
|
||||
if n < 0 {
|
||||
n = 0
|
||||
}
|
||||
nt := make(tFiles, 0, n)
|
||||
|
||||
// base tables
|
||||
for _, t := range bt {
|
||||
if _, ok := tm.deleted[t.file.Num()]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := tm.added[t.file.Num()]; ok {
|
||||
continue
|
||||
}
|
||||
nt = append(nt, t)
|
||||
}
|
||||
|
||||
// new tables
|
||||
for _, tr := range tm.added {
|
||||
nt = append(nt, tr.makeFile(s))
|
||||
}
|
||||
|
||||
// sort tables
|
||||
nt.sortByKey(s.icmp)
|
||||
nv.tables[level] = nt
|
||||
}
|
||||
|
||||
// compute compaction score for new version
|
||||
nv.computeCompaction()
|
||||
|
||||
return nv
|
||||
}
|
||||
|
||||
type versionReleaser struct {
|
||||
v *version
|
||||
once bool
|
||||
}
|
||||
|
||||
func (vr *versionReleaser) Release() {
|
||||
v := vr.v
|
||||
v.s.vmu.Lock()
|
||||
if !vr.once {
|
||||
v.release_NB()
|
||||
vr.once = true
|
||||
}
|
||||
v.s.vmu.Unlock()
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user