Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6419f199c | ||
|
|
5a3cd577db | ||
|
|
f5ccb8efdc | ||
|
|
46ef6c8ab7 | ||
|
|
b09eabc478 | ||
|
|
e49fcea6e3 | ||
|
|
26964ecf0f | ||
|
|
77d19af359 | ||
|
|
120b82f243 | ||
|
|
a0fea30a11 | ||
|
|
8bb3e0a64d | ||
|
|
bbded12923 | ||
|
|
e8ba5d7606 | ||
|
|
af66b31a44 | ||
|
|
b000a40f28 | ||
|
|
3c85322523 | ||
|
|
a469b8bc04 | ||
|
|
78b8261a3a | ||
|
|
f20ca70c01 | ||
|
|
4e4148fc1c | ||
|
|
c22482f907 | ||
|
|
870e139fce | ||
|
|
4d58648c02 | ||
|
|
ebbb182537 | ||
|
|
1cd5377b45 | ||
|
|
1d1f0527ee | ||
|
|
37a5f77415 | ||
|
|
ced3c7efe4 | ||
|
|
c3b8f366ed | ||
|
|
de78876b1a | ||
|
|
64c25b049c | ||
|
|
8a4fe4f3ad | ||
|
|
8811b8c1fd | ||
|
|
190b01fdf9 | ||
|
|
f145d08c10 | ||
|
|
53382b7e15 | ||
|
|
6ad0242617 | ||
|
|
a7c2408c0a | ||
|
|
ce1eabaed6 | ||
|
|
f602541ede | ||
|
|
3f718e6efc | ||
|
|
ce7a985df6 | ||
|
|
6d83f7e7bd | ||
|
|
b73c00943c | ||
|
|
abaac8ef48 | ||
|
|
a2f8e7068e | ||
|
|
4d47873219 | ||
|
|
cf985486e5 | ||
|
|
b930c3fc93 | ||
|
|
dd26a96828 | ||
|
|
6865ddfc12 | ||
|
|
e888e69d4d | ||
|
|
2089059792 | ||
|
|
27739e0364 | ||
|
|
698ee271d6 | ||
|
|
543c75b293 | ||
|
|
b09c46f6f7 | ||
|
|
f2cc19e6aa | ||
|
|
814a0a123f | ||
|
|
179383540f | ||
|
|
8be1c8310d | ||
|
|
ef02d776ca | ||
|
|
750ff448ad | ||
|
|
e3abab6d4d | ||
|
|
d3ffae72fb | ||
|
|
87f751188e | ||
|
|
3469abaefd | ||
|
|
797364ee5c | ||
|
|
36c05fc4b9 | ||
|
|
79624febc0 | ||
|
|
0a9d4de126 | ||
|
|
b6f6994db4 | ||
|
|
ff7bed720a | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 | ||
|
|
51f573f1ea | ||
|
|
c8b7cd8862 | ||
|
|
21c112d059 | ||
|
|
9432e1b5b2 | ||
|
|
084f0ebdab | ||
|
|
58640c1521 | ||
|
|
7ffdfd12f8 | ||
|
|
cb9a30f285 | ||
|
|
e48bef809f | ||
|
|
f5d7570102 | ||
|
|
a600c60cf8 | ||
|
|
695eabd026 | ||
|
|
e81b51a647 | ||
|
|
b7e95ff090 | ||
|
|
3ca41be686 | ||
|
|
3152046173 | ||
|
|
3a98dc8a95 | ||
|
|
d737b28916 | ||
|
|
97c0dac876 | ||
|
|
006494b1fa | ||
|
|
149d343201 | ||
|
|
496cc79ba8 | ||
|
|
3591768745 | ||
|
|
d615f6761a | ||
|
|
5b29894048 | ||
|
|
d17f0cfa40 | ||
|
|
b2f70c7120 | ||
|
|
238707bd93 | ||
|
|
87cdf1e3a0 | ||
|
|
57fd47022e | ||
|
|
5eafd3bf6b | ||
|
|
640b5b0ea9 | ||
|
|
e4fb2f4680 | ||
|
|
a4909f54e4 | ||
|
|
f7065c6f0c | ||
|
|
64f7f837e7 | ||
|
|
130d36acd9 | ||
|
|
e94c28cfa2 | ||
|
|
a697dec30c | ||
|
|
bac09e9b9f | ||
|
|
b179f8f86a | ||
|
|
962d04ae17 | ||
|
|
f28e3bfe37 | ||
|
|
6474c86d32 | ||
|
|
769ea8cdfe | ||
|
|
052e77dd5a | ||
|
|
1a4b0cb12a | ||
|
|
f9aa95c987 | ||
|
|
a5d06fb4a4 | ||
|
|
d5e5d90bdc | ||
|
|
e62678e4e6 | ||
|
|
558ac24f7e | ||
|
|
6f22f6a59f | ||
|
|
5ec74f8abe | ||
|
|
16d7547e03 | ||
|
|
767112dcda | ||
|
|
81e23d1d8c | ||
|
|
ace90a4354 | ||
|
|
267e7096cc | ||
|
|
e7cded7511 | ||
|
|
33fbd715c0 | ||
|
|
25ba5b867c | ||
|
|
c8f431447c | ||
|
|
1b83a1d09a | ||
|
|
d2fb7a7151 | ||
|
|
7ad3447598 | ||
|
|
5e5bc8e705 | ||
|
|
886d1e8a19 | ||
|
|
6de7a035fa | ||
|
|
48d2497eb2 | ||
|
|
6494754ab9 | ||
|
|
a8153627c6 | ||
|
|
ed38966edb | ||
|
|
85551d1e54 | ||
|
|
ca31363180 | ||
|
|
a8f11eb3c3 | ||
|
|
151805121c | ||
|
|
0422d2a021 | ||
|
|
ffdf70257b | ||
|
|
6079d60aa1 | ||
|
|
d46d012f8c | ||
|
|
baf926b360 | ||
|
|
82339869dc | ||
|
|
f70343bb63 | ||
|
|
4ab56ae3d1 | ||
|
|
39b4031684 | ||
|
|
71f8965393 | ||
|
|
66857e72a4 | ||
|
|
15eb5d47eb | ||
|
|
86242139da | ||
|
|
460bdc4148 | ||
|
|
1052126522 | ||
|
|
fdb42ac876 | ||
|
|
6486f986e8 | ||
|
|
bc4075e2ed | ||
|
|
11525c7d0d | ||
|
|
bb12ae9ce6 | ||
|
|
66b4ad0c44 | ||
|
|
4bff2d718e | ||
|
|
af7a7b4dc6 | ||
|
|
ec20d9a2a8 | ||
|
|
b0aee3d335 | ||
|
|
5e2ddbfd86 | ||
|
|
f086ebbb8e | ||
|
|
e5bf6a5bfc | ||
|
|
76fa86d54b | ||
|
|
60d89506a5 | ||
|
|
2893f1eb9e | ||
|
|
a08cbf412d | ||
|
|
0bd65356f9 | ||
|
|
2e2b39455f | ||
|
|
1f9fbe34e4 | ||
|
|
506d7ae50e | ||
|
|
342cb00625 | ||
|
|
952fc01efd | ||
|
|
3eebe301fe | ||
|
|
19c0a81c42 | ||
|
|
9fc7f9904b | ||
|
|
b04a516063 | ||
|
|
241e946d91 | ||
|
|
e45375dc26 | ||
|
|
ec8e243323 | ||
|
|
af0e9368d4 | ||
|
|
285483b81a | ||
|
|
73a91d5569 | ||
|
|
e24c78be32 | ||
|
|
b11184de68 | ||
|
|
61d02fc5d7 | ||
|
|
f0217f6821 | ||
|
|
09644914a6 | ||
|
|
b5c24d6a48 | ||
|
|
bdd2ac2c25 | ||
|
|
76376f0d33 | ||
|
|
9749da46ae | ||
|
|
1c05295e89 | ||
|
|
4d0f28215a | ||
|
|
0bba72bc5a | ||
|
|
87089b8e83 | ||
|
|
6a58148a89 | ||
|
|
15091052be | ||
|
|
5cd7e9a9b8 | ||
|
|
7cb530c334 | ||
|
|
5e0324cc91 | ||
|
|
6293e227ea | ||
|
|
d6d83a5c76 | ||
|
|
93fc883b90 | ||
|
|
dd21803598 | ||
|
|
af2f2282c2 | ||
|
|
869a76c9bb | ||
|
|
44b2964a6a | ||
|
|
8fddb57e0a | ||
|
|
c7a5a0cab0 | ||
|
|
c13aab3ffc | ||
|
|
95a9a76598 | ||
|
|
923cfa3d50 | ||
|
|
2c489168c2 | ||
|
|
f3292b4d34 | ||
|
|
9769829b72 | ||
|
|
36243d15cc | ||
|
|
f4645f570c | ||
|
|
61603ccfce | ||
|
|
a26cfc639c | ||
|
|
90602931d8 | ||
|
|
d1b330028c | ||
|
|
7a6b1930bf | ||
|
|
c271361552 | ||
|
|
17789bc814 | ||
|
|
d7c16e161a | ||
|
|
e71c17d7e7 | ||
|
|
17d87eb157 | ||
|
|
bf11bcc084 | ||
|
|
bd291109df | ||
|
|
ac89ba9b8d | ||
|
|
5ab634e375 | ||
|
|
bb1978d976 | ||
|
|
bea94ce8ac | ||
|
|
9561f93594 | ||
|
|
d44d5c3304 | ||
|
|
ec541e2057 | ||
|
|
8169bf6b97 | ||
|
|
56c8ad1221 | ||
|
|
6814f2e38c | ||
|
|
5f043b9a78 | ||
|
|
c9092cd1c7 | ||
|
|
86cc65d894 | ||
|
|
8ef2647fa9 | ||
|
|
3ff9e99416 | ||
|
|
1731046011 | ||
|
|
c3d96184b6 | ||
|
|
c255c606a7 | ||
|
|
cc69faa1fd | ||
|
|
0f53cd86c8 | ||
|
|
212c8f188d | ||
|
|
dec248adec | ||
|
|
98e6a066f4 | ||
|
|
c90db54a3d | ||
|
|
927d156933 | ||
|
|
c71f91073f | ||
|
|
ab28387692 | ||
|
|
c4905d02b9 | ||
|
|
ee39880fb5 | ||
|
|
926b3e56b9 | ||
|
|
e060df5367 | ||
|
|
a3294d4a0d | ||
|
|
19ce0ab246 | ||
|
|
75a1cc0d33 | ||
|
|
05228529b0 | ||
|
|
3cbeadfbc3 | ||
|
|
3d4938c0e2 | ||
|
|
1b850b8a2b | ||
|
|
ebd33f1869 | ||
|
|
ac846b4df3 | ||
|
|
0d427c9b90 | ||
|
|
dc6194f862 | ||
|
|
de50214e1f | ||
|
|
5ec4a88c35 | ||
|
|
31c141e757 | ||
|
|
9d5760d899 | ||
|
|
7945e1ea3c | ||
|
|
d94c40e371 | ||
|
|
31f22122e8 | ||
|
|
506582aa2b | ||
|
|
a02b251c9b | ||
|
|
34cbb6fa79 | ||
|
|
33679b56fd | ||
|
|
8897c4d560 | ||
|
|
f0c18ccbe7 | ||
|
|
e560d83c51 | ||
|
|
eca7addc67 | ||
|
|
263f3ba5c9 | ||
|
|
f514123ef0 | ||
|
|
9f461db0d0 | ||
|
|
7de8670616 | ||
|
|
8c7908b200 | ||
|
|
e7722e039f | ||
|
|
039bc0208a | ||
|
|
d57900a069 | ||
|
|
c1153302aa | ||
|
|
1086a3297f | ||
|
|
5c613934ca | ||
|
|
6d7e9092f8 | ||
|
|
8bb6f328dc | ||
|
|
7f03b13579 | ||
|
|
09008ea991 | ||
|
|
ee234212e6 | ||
|
|
277b243f52 | ||
|
|
0973ac753f | ||
|
|
f41cfbfb97 | ||
|
|
a70980c81a | ||
|
|
12a962f656 | ||
|
|
5936444f3e | ||
|
|
c7d42f00c6 | ||
|
|
b1318a9958 | ||
|
|
d2560d260c | ||
|
|
f40a61cf9a | ||
|
|
e0623578bf | ||
|
|
ead8daaa14 | ||
|
|
23b4b9e230 | ||
|
|
947017e334 | ||
|
|
7fd0d6507f | ||
|
|
ad1e9ef086 | ||
|
|
2f1d08e417 | ||
|
|
55f4eb80ba | ||
|
|
7c160ff65e | ||
|
|
fc5d5faaed | ||
|
|
5394f1763c | ||
|
|
050b67c9d6 | ||
|
|
4b87cb45ee | ||
|
|
f0e1db319c | ||
|
|
3950715237 | ||
|
|
ae9ba14b59 | ||
|
|
ad6c6fbe35 | ||
|
|
ffa5c20c88 | ||
|
|
5a6218eeca | ||
|
|
200531dd96 | ||
|
|
15a2f048ac | ||
|
|
d317a4042c | ||
|
|
c2d717aba5 | ||
|
|
ef74777df1 | ||
|
|
f6dbf99fb5 | ||
|
|
0b2d9e15b4 | ||
|
|
d14e5e75e8 | ||
|
|
df11ef34d2 | ||
|
|
4fc619262c | ||
|
|
2750eee86d | ||
|
|
71cb03345b | ||
|
|
22c8ca6c40 | ||
|
|
237a8965eb | ||
|
|
7d872c7863 | ||
|
|
0d08858dfb | ||
|
|
5c6d63c93d | ||
|
|
c8c10d2bb0 | ||
|
|
f4d3e9ea6a | ||
|
|
3713d3488d | ||
|
|
9b46d03c91 | ||
|
|
fe4e738580 | ||
|
|
205b0c4263 | ||
|
|
78eef25f1a | ||
|
|
c9305fd070 | ||
|
|
20774280b9 | ||
|
|
74bd2557af | ||
|
|
605fc8ecd8 | ||
|
|
6b745c53dc | ||
|
|
98c60b38fc | ||
|
|
be4515ec9a | ||
|
|
dcb1b64696 | ||
|
|
1190a60c7a | ||
|
|
82ecf207c6 | ||
|
|
84c7f1bf58 | ||
|
|
5ac453fe07 | ||
|
|
c2f39d653d | ||
|
|
748178f536 | ||
|
|
ffbef69dc7 | ||
|
|
3757124323 | ||
|
|
e65f214b3c | ||
|
|
f14004e56b | ||
|
|
dadc004dd6 | ||
|
|
3697d9e1f0 | ||
|
|
94524f7330 | ||
|
|
17d8a3e5d6 | ||
|
|
39996d7612 | ||
|
|
601ee4320f | ||
|
|
6e4f6fa92d | ||
|
|
b7103c29dd | ||
|
|
e53209e5ae | ||
|
|
501fbd3114 | ||
|
|
5626f43e2b | ||
|
|
0185cb48fb | ||
|
|
9a49eb06da | ||
|
|
3fbc012231 | ||
|
|
3938cf33d8 | ||
|
|
38b3d0c87c | ||
|
|
15a2fa6199 | ||
|
|
e834209d40 | ||
|
|
9607d21828 | ||
|
|
6fbbf36143 | ||
|
|
db25f3282e | ||
|
|
ceb92838e7 | ||
|
|
afb98f2cfc | ||
|
|
ed785fb087 | ||
|
|
3921180d77 | ||
|
|
ef36e0311e | ||
|
|
bf4cbe1204 | ||
|
|
28eee4da25 | ||
|
|
933ccf6deb | ||
|
|
9ac41322e5 | ||
|
|
c3233e03ef | ||
|
|
111871bb28 | ||
|
|
bf5b5f43e3 | ||
|
|
2ed8ebff09 | ||
|
|
34eb856d09 | ||
|
|
2d644bdfb0 | ||
|
|
741e5cf103 | ||
|
|
d745c04fe6 | ||
|
|
3a3c63956a | ||
|
|
153bdf576a | ||
|
|
6525ee6510 | ||
|
|
1780ddd329 | ||
|
|
85ad972ca8 | ||
|
|
a90b60799a | ||
|
|
47e8f74da9 | ||
|
|
fac28072ab | ||
|
|
ab77541f36 | ||
|
|
9eb769f340 | ||
|
|
1e81d6f848 | ||
|
|
dc393f4b77 | ||
|
|
dd1de530c3 | ||
|
|
74cd887249 | ||
|
|
b40bb35652 | ||
|
|
dfe257af86 | ||
|
|
ebb2a09107 | ||
|
|
c157909b4e | ||
|
|
a744239b22 | ||
|
|
9b4417be87 | ||
|
|
84f807e278 | ||
|
|
448635a945 | ||
|
|
638e3e6b3d | ||
|
|
bfe711bd42 | ||
|
|
d5eed91e7f | ||
|
|
15abf49211 | ||
|
|
b7eec0586c | ||
|
|
e639124a69 | ||
|
|
22bca7a16e | ||
|
|
6f66b354e5 | ||
|
|
f98e3320ae | ||
|
|
73c958222b | ||
|
|
e43230e46a | ||
|
|
dd07bc29eb | ||
|
|
790bbfcc99 | ||
|
|
994e70d43a | ||
|
|
7d5e19716a | ||
|
|
11530c675d | ||
|
|
1c474352fe | ||
|
|
3b2d635390 | ||
|
|
055c532d31 | ||
|
|
55e08f96a6 | ||
|
|
3e0a9180bc | ||
|
|
01ed71b243 | ||
|
|
595c058517 | ||
|
|
e04b0a6995 | ||
|
|
8961848ab1 | ||
|
|
6fb1c72b7d | ||
|
|
400d79d6ab | ||
|
|
ce6018f387 | ||
|
|
65d9dca917 | ||
|
|
2232b08351 | ||
|
|
0732ab6ad5 | ||
|
|
fb568768c5 | ||
|
|
7fd56e0add | ||
|
|
38a3f8cf4c | ||
|
|
8d8a5b36b6 | ||
|
|
a70cc7beb1 | ||
|
|
c5421699b1 | ||
|
|
d9f809864f | ||
|
|
bb23f509d7 | ||
|
|
254e184677 | ||
|
|
6c07141abd | ||
|
|
5822fc1d5b | ||
|
|
2cd7ea257c | ||
|
|
0a21b72f9c | ||
|
|
80280f6f4a | ||
|
|
810a8e76d1 | ||
|
|
bd3b4906d1 | ||
|
|
d9c740f014 | ||
|
|
ec63a18960 | ||
|
|
fab0618b6b | ||
|
|
13e57e7aa8 | ||
|
|
1299c9162c | ||
|
|
a64f33dbcc | ||
|
|
836b66e110 | ||
|
|
802119d789 | ||
|
|
f94135cadd | ||
|
|
474c0f980e | ||
|
|
2cf0f791f2 | ||
|
|
aec4d1dc60 | ||
|
|
8d28bb5a66 | ||
|
|
1d5e553f9c | ||
|
|
9d79c52d20 | ||
|
|
1ac03e2f1d | ||
|
|
801f05f45e | ||
|
|
345196aa3b | ||
|
|
a01b52421e | ||
|
|
86cedfe8b2 | ||
|
|
709128225b | ||
|
|
59a5bacb2e | ||
|
|
3a21df31ee | ||
|
|
0e3407ec2f | ||
|
|
fd6609e961 | ||
|
|
b0b5e045ff | ||
|
|
f293d7cccc | ||
|
|
974ed29c36 | ||
|
|
53eb033034 | ||
|
|
f2545b98ab | ||
|
|
1bf51e855d | ||
|
|
e4da2a695b | ||
|
|
1a905d9a42 | ||
|
|
7d9d0c15d3 | ||
|
|
d34f5c2712 | ||
|
|
a56dd66c98 | ||
|
|
3946bc6662 | ||
|
|
6647dd8f08 | ||
|
|
9371122bed | ||
|
|
734cd8ee08 | ||
|
|
d85d9d05bb | ||
|
|
1d7b87b3a9 | ||
|
|
f0cfde36f2 | ||
|
|
7e220d2741 | ||
|
|
c4b1351a43 | ||
|
|
90d2597bc3 | ||
|
|
c1508c94c2 | ||
|
|
1a0d805bad | ||
|
|
1a802469f5 | ||
|
|
646ae7e2d1 | ||
|
|
563d76ff9f | ||
|
|
54d0473e85 | ||
|
|
ec8a61c3af | ||
|
|
0315eaff8a | ||
|
|
b34f44c867 | ||
|
|
ea7056835f | ||
|
|
0bb4cdadd9 | ||
|
|
08f6520557 | ||
|
|
38b108752e | ||
|
|
0ee7870bdf | ||
|
|
37d46ecdb2 | ||
|
|
2a7911c7d6 | ||
|
|
fc2d9b4fd2 | ||
|
|
832020fa81 | ||
|
|
78606ed4b8 | ||
|
|
a04ba606e6 | ||
|
|
ebf2e7ee70 | ||
|
|
b31881f424 | ||
|
|
c0a6f6fd08 | ||
|
|
7e1b1ca730 | ||
|
|
163d176f0a | ||
|
|
0f7eab5100 | ||
|
|
cbf78d474a | ||
|
|
3615e0de34 | ||
|
|
e73c081d36 | ||
|
|
663bf6e3af | ||
|
|
60194cbafd | ||
|
|
7d2eb5faf7 | ||
|
|
5969f3d213 | ||
|
|
6ddde88389 | ||
|
|
e9992ed6b4 | ||
|
|
4db823a3be | ||
|
|
78732513c8 | ||
|
|
79557bb878 | ||
|
|
4094b02ae9 | ||
|
|
9cc7c118a8 | ||
|
|
d3dad51c76 | ||
|
|
5bf35de955 | ||
|
|
f8928c654b | ||
|
|
b3903f4ffd | ||
|
|
46c3f76edc | ||
|
|
3488408b7a | ||
|
|
b7b164b84c | ||
|
|
7c353b7d76 | ||
|
|
3d9841b61a | ||
|
|
87e2d2c7b0 | ||
|
|
ff59619af4 | ||
|
|
b79f2b86b5 | ||
|
|
386fd7a459 | ||
|
|
d4f41901ef | ||
|
|
f170da3e78 | ||
|
|
5dc15be98a | ||
|
|
88c7cfc745 | ||
|
|
867478697d | ||
|
|
d27e8cf73a | ||
|
|
4490caa4f7 | ||
|
|
1b25f2c531 | ||
|
|
9d911d1fa0 | ||
|
|
494feb9f6d | ||
|
|
2a04943a67 | ||
|
|
731b29ce91 | ||
|
|
28fbb70858 | ||
|
|
44f402c64e | ||
|
|
c7f391ca44 | ||
|
|
5ed9c5e168 | ||
|
|
51b1a86d19 | ||
|
|
20c46677d0 | ||
|
|
efb5f232f7 | ||
|
|
9edf9561b2 | ||
|
|
5d4eb018ee | ||
|
|
10fbce056b | ||
|
|
cbc6d6a572 | ||
|
|
fb1af9e3d2 | ||
|
|
c9e410a6f4 | ||
|
|
c8d72b59c0 | ||
|
|
cd71d4a2f7 | ||
|
|
ed0cc59e66 | ||
|
|
d4f1b48f52 | ||
|
|
08bec02170 | ||
|
|
c9f27b83e1 | ||
|
|
8327585b3b | ||
|
|
727815d6dd | ||
|
|
2968f74c6c | ||
|
|
431499f43f | ||
|
|
d809283d23 | ||
|
|
b49275fe6b | ||
|
|
87c7ce588e | ||
|
|
26d0a7c742 | ||
|
|
b01887c7ec | ||
|
|
745031c020 | ||
|
|
3ccfc09e01 | ||
|
|
e8750275c9 | ||
|
|
f0d7ea4cc6 | ||
|
|
ea5322af82 | ||
|
|
0b23f68fb2 | ||
|
|
1c00129f76 | ||
|
|
454700af05 | ||
|
|
d9b482d90a | ||
|
|
09c2c97069 | ||
|
|
88faf0ce78 | ||
|
|
aef83eec31 | ||
|
|
5bd222c266 | ||
|
|
d3e7f789b2 | ||
|
|
87c6e292f1 | ||
|
|
b597ad0b3f | ||
|
|
40dd99dea3 | ||
|
|
6ea2686219 | ||
|
|
6bca90c4f7 | ||
|
|
20fa7bd852 | ||
|
|
6b80119eff | ||
|
|
1fc5469eb1 | ||
|
|
bf2769a0c2 | ||
|
|
325b601bea | ||
|
|
126642912e | ||
|
|
57ee16d565 | ||
|
|
9f0fba063d | ||
|
|
757fb51415 | ||
|
|
b1c5655138 | ||
|
|
4d97dcaa93 | ||
|
|
96db1b4685 | ||
|
|
9e21c52d04 | ||
|
|
5fa801f9f2 | ||
|
|
0301768b79 | ||
|
|
4c6dedfa4f | ||
|
|
6652d55455 | ||
|
|
3e03dd3e80 | ||
|
|
84b8a22423 | ||
|
|
ada03e12fc | ||
|
|
d66c801350 | ||
|
|
91846939f6 | ||
|
|
c85127d76b | ||
|
|
166bf65e88 | ||
|
|
f68bdc406f | ||
|
|
8390482d6c | ||
|
|
783f454f8e | ||
|
|
86a67bd245 | ||
|
|
099b0b65c0 | ||
|
|
505652efa0 | ||
|
|
a20088330f | ||
|
|
a20e2504bb | ||
|
|
b07bc150e1 | ||
|
|
46c6588aa1 | ||
|
|
e36c672c9a | ||
|
|
c044e1ea8c | ||
|
|
25b6c1b6ca | ||
|
|
c25d72d911 | ||
|
|
73613d1583 | ||
|
|
1e6542d12d | ||
|
|
3cffaa43f7 | ||
|
|
08792e794f | ||
|
|
7fa26d59da | ||
|
|
74469829e5 | ||
|
|
ba414742a6 | ||
|
|
f6fcfb6dde | ||
|
|
c5b90c0144 | ||
|
|
f669881849 | ||
|
|
d0b70ea6df | ||
|
|
51af43d492 | ||
|
|
8a49ad4586 | ||
|
|
2ae068c2d2 | ||
|
|
5e07e8dbbd | ||
|
|
8d2c68505e | ||
|
|
44f1e7c401 | ||
|
|
fd2945469e | ||
|
|
b0f0827cff | ||
|
|
c1e2c415c4 | ||
|
|
a258b96602 | ||
|
|
82ef76b1cb | ||
|
|
aaa9982932 | ||
|
|
1cdb4bc9e7 | ||
|
|
fddb0c9e66 | ||
|
|
5c78d2b6d2 | ||
|
|
a4113cbb52 | ||
|
|
afa3bda3c8 | ||
|
|
61d7edd15f | ||
|
|
c2106de7e1 | ||
|
|
0bf1be6198 | ||
|
|
af7c20e440 | ||
|
|
17771a2b89 | ||
|
|
8840097fe2 | ||
|
|
7d333efd45 | ||
|
|
ec65815c57 | ||
|
|
640b0c71e6 | ||
|
|
4208876ef3 | ||
|
|
bff9117360 | ||
|
|
6b3c4c70d2 | ||
|
|
0292fd8d91 | ||
|
|
94a41270d0 | ||
|
|
75d78bfa53 | ||
|
|
88263db831 | ||
|
|
3ef96404ee | ||
|
|
13f9370f8c | ||
|
|
1c349270bf | ||
|
|
c28f0f6286 | ||
|
|
b20939d211 | ||
|
|
b76c45fc2b | ||
|
|
1bbfccd082 | ||
|
|
cee7e06832 | ||
|
|
03dff4b382 | ||
|
|
f546b20a01 | ||
|
|
0e8d9a264f | ||
|
|
66a4c98bee | ||
|
|
36101db500 | ||
|
|
6bff298d1e | ||
|
|
c832f8ffbb | ||
|
|
d7fe46dbde | ||
|
|
5951186463 | ||
|
|
7191042bb8 | ||
|
|
a80fbcc252 | ||
|
|
c321758afd | ||
|
|
ca8f59a643 | ||
|
|
8110882617 | ||
|
|
a283b66541 | ||
|
|
569de37e16 | ||
|
|
669f3394c7 | ||
|
|
87ec2c7378 | ||
|
|
ea7a31cad1 | ||
|
|
c894ade25e | ||
|
|
a3fe127a92 | ||
|
|
7f526be879 | ||
|
|
71ae174e29 | ||
|
|
2277d134cc | ||
|
|
880aa265f1 | ||
|
|
ea639e64a4 | ||
|
|
b60fdee5e2 | ||
|
|
44f07fb81d | ||
|
|
60a85fadd3 | ||
|
|
68367125e1 | ||
|
|
6112f60681 | ||
|
|
ecc74e2ae5 | ||
|
|
c2dae43965 | ||
|
|
6e3a25d255 | ||
|
|
4e13cf1d85 | ||
|
|
0fee1897eb | ||
|
|
9dd51467fe | ||
|
|
cec6dca72f | ||
|
|
58df2b9bcf | ||
|
|
cc6bdd6295 | ||
|
|
7a044f9f78 | ||
|
|
e77acde005 | ||
|
|
ece5082096 | ||
|
|
0723026a56 | ||
|
|
0b7346f4a1 | ||
|
|
a357db1ff6 | ||
|
|
a2287f8d0a | ||
|
|
076decd6e9 | ||
|
|
d40461de94 | ||
|
|
4b1492be0d | ||
|
|
1676246743 | ||
|
|
f7c9f6d142 | ||
|
|
6022339091 | ||
|
|
25d97afa60 | ||
|
|
df635b963c | ||
|
|
6d0a054170 | ||
|
|
177d3af77f | ||
|
|
35f71c619b | ||
|
|
31a80ccce4 | ||
|
|
3194cec8df | ||
|
|
57fd3c5801 | ||
|
|
86617f1bda | ||
|
|
35f36c598c | ||
|
|
36bd7816fe | ||
|
|
fd632392d4 | ||
|
|
423b48866f | ||
|
|
745bc24343 | ||
|
|
04a823c7d0 | ||
|
|
56af4c2fcb | ||
|
|
77daf6812a | ||
|
|
c7e964ec42 | ||
|
|
694e2c2a4f | ||
|
|
df6537fae8 | ||
|
|
71f6542571 | ||
|
|
5a6294c08e | ||
|
|
f0390786d6 | ||
|
|
719d18c670 | ||
|
|
a116dcd3bb | ||
|
|
e2dd3dcf88 | ||
|
|
822f0c3cb2 | ||
|
|
63fb39016b | ||
|
|
e79f8d58ca | ||
|
|
a469dfb583 | ||
|
|
6cbc545f84 | ||
|
|
ea1e24d8f9 | ||
|
|
6fcb4d93c9 | ||
|
|
36031c9d0f | ||
|
|
a70f5e2440 | ||
|
|
6c42aab567 | ||
|
|
27631a1746 | ||
|
|
320f5cebc7 | ||
|
|
ea2a583803 | ||
|
|
9965801258 | ||
|
|
24ed0521c7 | ||
|
|
d8fed79810 | ||
|
|
c30db15efa | ||
|
|
817869b915 | ||
|
|
3038dd9440 | ||
|
|
276b2c737a | ||
|
|
62543b94c9 | ||
|
|
4d607ed276 | ||
|
|
26e2865e35 | ||
|
|
f7c94c9eb5 | ||
|
|
25a91f89fd | ||
|
|
246f055145 | ||
|
|
8c3bf0d4cb | ||
|
|
e49f24e95f | ||
|
|
48a18ff771 | ||
|
|
37161057c6 | ||
|
|
68d1157495 | ||
|
|
19c3b2f4b6 | ||
|
|
7051c332a1 | ||
|
|
0f2a0b7c97 | ||
|
|
bb6ef42d0c | ||
|
|
3c591d4968 | ||
|
|
cf356a8e35 | ||
|
|
3a0ce58cda | ||
|
|
2d7038dc18 | ||
|
|
25a0e67841 | ||
|
|
be64209292 | ||
|
|
0006cb1c27 | ||
|
|
836d38ed2a | ||
|
|
ec71017785 |
No files matched your search
@@ -0,0 +1,81 @@
|
||||
name: ci
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- working-directory: lbry
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
test:
|
||||
- datanetwork
|
||||
- blockchain
|
||||
- other
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: Setup
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(matrix.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
@@ -1,2 +1,17 @@
|
||||
/.idea
|
||||
/.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.coverage*
|
||||
/lbry-venv
|
||||
/venv
|
||||
|
||||
lbry.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/blockchain/files
|
||||
/tests/.coverage.*
|
||||
|
||||
/lbry/wallet/bin
|
||||
@@ -0,0 +1,211 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
|
||||
|
||||
#cache:
|
||||
# directories:
|
||||
# - $HOME/venv
|
||||
# - $HOME/.cache/pip
|
||||
# - $HOME/Library/Caches/pip
|
||||
# - $HOME/Library/Caches/Homebrew
|
||||
# - $TRAVIS_BUILD_DIR/.tox
|
||||
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- assets
|
||||
- release
|
||||
|
||||
|
||||
.tagged:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
|
||||
when: on_success
|
||||
|
||||
|
||||
|
||||
test:lint:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- make lint
|
||||
|
||||
test:unit:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e datanetwork --recreate
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
artifacts:
|
||||
expire_in: 1 day
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller
|
||||
- pip install -e .
|
||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
||||
- dist/lbrynet --version
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
- pip install lbry-libtorrent
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
tags: [macos] # makes gitlab use the mac runner
|
||||
variables:
|
||||
OS: mac
|
||||
GIT_DEPTH: 5
|
||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
||||
before_script:
|
||||
# - brew upgrade python || true
|
||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip3 install --user --upgrade pip virtualenv
|
||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
||||
- virtualenv --python=python3.7 "${VENV}"
|
||||
- source "${VENV}/bin/activate"
|
||||
after_script:
|
||||
- rm -rf "${VENV}"
|
||||
|
||||
build:windows:
|
||||
extends: .build
|
||||
tags: [windows] # makes gitlab use the windows runner
|
||||
variables:
|
||||
OS: windows
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.5
|
||||
- pip install -e .
|
||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
- dist/lbrynet.exe --version
|
||||
|
||||
|
||||
|
||||
# s3 = upload asset to s3 (build.lbry.io)
|
||||
.s3:
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
|
||||
- pip install awscli
|
||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
||||
|
||||
s3:linux:
|
||||
extends: .s3
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
s3:mac:
|
||||
extends: .s3
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
s3:windows:
|
||||
extends: .s3
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
# github = upload assets to github when there's a tagged release
|
||||
.github:
|
||||
extends: .tagged
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
||||
|
||||
github:linux:
|
||||
extends: .github
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
github:mac:
|
||||
extends: .github
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
github:windows:
|
||||
extends: .github
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
publish:
|
||||
extends: .tagged
|
||||
stage: release
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
|
||||
- >
|
||||
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
|
||||
@@ -1,146 +0,0 @@
|
||||
dist: xenial
|
||||
language: python
|
||||
python: "3.7"
|
||||
|
||||
jobs:
|
||||
include:
|
||||
|
||||
- stage: code quality
|
||||
name: "pylint & mypy"
|
||||
install:
|
||||
- make install
|
||||
script: make lint
|
||||
|
||||
- stage: test
|
||||
name: "LBRY Unit Tests"
|
||||
install:
|
||||
- make install
|
||||
script:
|
||||
- cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
after_success:
|
||||
- coverage combine lbry/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "LBRY Integration Tests"
|
||||
install:
|
||||
- pip install coverage tox-travis
|
||||
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
|
||||
script: cd lbry && tox
|
||||
after_success:
|
||||
- coverage combine lbry
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- &torba-tests
|
||||
name: "Torba Unit Tests"
|
||||
env: TESTTYPE=unit
|
||||
install:
|
||||
- pip install coverage tox-travis
|
||||
script: cd torba && tox
|
||||
after_success:
|
||||
- coverage combine torba/tests
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *torba-tests
|
||||
name: "Torba Integration Tests"
|
||||
env: TESTTYPE=integration
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- make install
|
||||
script:
|
||||
- cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine lbry
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- &build
|
||||
stage: build
|
||||
name: "Linux"
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip install pyinstaller awscli
|
||||
- cd torba && pip install -e . && cd ..
|
||||
- cd lbry
|
||||
- python scripts/set_build.py
|
||||
- pip install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbry/extras/cli.py
|
||||
- cd dist
|
||||
- chmod +x lbrynet
|
||||
- zip -j lbrynet-${OS}.zip lbrynet
|
||||
- shasum -a 256 -b lbrynet-${OS}.zip
|
||||
- ./lbrynet --version
|
||||
after_success:
|
||||
- aws configure set aws_access_key_id $ARTIFACTS_KEY
|
||||
- aws configure set aws_secret_access_key $ARTIFACTS_SECRET
|
||||
- aws configure set region us-east-1
|
||||
- export S3_PATH="daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})"
|
||||
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key:
|
||||
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
|
||||
file: lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
|
||||
- <<: *build
|
||||
name: "Mac"
|
||||
os: osx
|
||||
osx_image: xcode8.3
|
||||
language: shell
|
||||
env: OS=mac
|
||||
before_install:
|
||||
- brew uninstall mercurial
|
||||
- brew upgrade python || true
|
||||
- pip3 install --user --upgrade pip virtualenv
|
||||
- /Users/travis/Library/Python/3.7/bin/virtualenv --clear $HOME/venv
|
||||
- source $HOME/venv/bin/activate
|
||||
before_cache:
|
||||
- brew cleanup
|
||||
|
||||
- <<: *build
|
||||
name: "Windows"
|
||||
os: windows
|
||||
language: shell
|
||||
env:
|
||||
- OS=windows
|
||||
- PATH=/c/Python37:/c/Python37/Scripts:/C/Windows/System32/downlevel:$PATH
|
||||
before_install:
|
||||
- choco install python --version=3.7.4 --x86
|
||||
- python -m pip install --upgrade pip
|
||||
- pip install pywin32
|
||||
script:
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- cd dist
|
||||
- 7z a -tzip lbrynet-windows.zip lbrynet.exe
|
||||
- sha256sum -b lbrynet-windows.zip
|
||||
- ./lbrynet.exe --version
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f lbry/scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f lbry/scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/venv
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $HOME/Library/Caches/Homebrew
|
||||
- $TRAVIS_BUILD_DIR/.tox
|
||||
@@ -88,9 +88,6 @@ To install on Windows:
|
||||
> lbry-venv\Scripts\activate
|
||||
|
||||
Install packages:
|
||||
> cd torba
|
||||
> pip install -e .
|
||||
> cd ../lbry
|
||||
> pip install -e .
|
||||
```
|
||||
|
||||
@@ -99,7 +96,7 @@ To install on Windows:
|
||||
To run the unit tests from the repo directory:
|
||||
|
||||
```
|
||||
python -m unittest discover -s lbry tests.unit
|
||||
python -m unittest discover tests.unit
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2019 LBRY Inc
|
||||
Copyright (c) 2015-2020 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
include README.md
|
||||
include CHANGELOG.md
|
||||
include LICENSE
|
||||
recursive-include torba *.txt *.py
|
||||
recursive-include lbry *.txt *.py
|
||||
@@ -1,18 +1,24 @@
|
||||
.PHONY: install tools lint test idea
|
||||
|
||||
install:
|
||||
cd torba && pip install -e .
|
||||
cd lbry && pip install -e .
|
||||
pip install mypy==0.701
|
||||
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
|
||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
--global-option=build --global-option=--enable --global-option=fts5
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
cd lbry && pylint lbry
|
||||
cd torba && pylint --rcfile=setup.cfg torba
|
||||
cd torba && mypy --ignore-missing-imports torba
|
||||
pylint --rcfile=setup.cfg lbry
|
||||
#mypy --ignore-missing-imports lbry
|
||||
|
||||
test:
|
||||
cd lbry && tox
|
||||
cd torba && tox
|
||||
tox
|
||||
|
||||
idea:
|
||||
mkdir -p .idea
|
||||
cp -r lbry/scripts/idea/* .idea
|
||||
cp -r scripts/idea/* .idea
|
||||
@@ -1,15 +1,15 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry-sdk) [](https://codecov.io/gh/lbryio/lbry-sdk)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://ci.lbry.tech/lbry/lbry-sdk)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -53,4 +53,4 @@ The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/example_daemon_settings.yml).
|
||||
@@ -0,0 +1,52 @@
|
||||
FROM ubuntu:19.10
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
ARG host=0.0.0.0
|
||||
ARG tcp_port=50001
|
||||
ARG daemon_url=http://lbry:lbry@localhost:9245/
|
||||
VOLUME $db_dir
|
||||
ENV TCP_PORT=$tcp_port
|
||||
ENV HOST=$host
|
||||
ENV DAEMON_URL=$daemon_url
|
||||
ENV DB_DIRECTORY=$db_dir
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENV EVENT_LOOP_POLICY=uvloop
|
||||
COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -0,0 +1,34 @@
|
||||
version: "3"
|
||||
|
||||
volumes:
|
||||
lbrycrd:
|
||||
wallet_server:
|
||||
|
||||
services:
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
|
||||
restart: always
|
||||
ports: # accessible from host
|
||||
- "9246:9246" # rpc port
|
||||
expose: # internal to docker network. also this doesn't do anything. its for documentation only.
|
||||
- "9245" # node-to-node comms port
|
||||
volumes:
|
||||
- "lbrycrd:/data/.lbrycrd"
|
||||
environment:
|
||||
- RUN_MODE=default
|
||||
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
|
||||
- RPC_ALLOW_IP=0.0.0.0/0
|
||||
wallet_server:
|
||||
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
restart: always
|
||||
ports:
|
||||
- "50001:50001" # rpc port
|
||||
- "50005:50005" # websocket port
|
||||
#- "2112:2112" # uncomment to enable prometheus
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
environment:
|
||||
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../.." ## make sure we're in the right place. Docker Hub screws this up sometimes
|
||||
echo "docker build dir: $(pwd)"
|
||||
|
||||
docker build --build-arg DOCKER_TAG=$DOCKER_TAG --build-arg DOCKER_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .
|
||||
@@ -0,0 +1,11 @@
|
||||
# requires powershell and .NET 4+. see https://chocolatey.org/install for more info.
|
||||
|
||||
$chocoVersion = powershell choco -v
|
||||
if(-not($chocoVersion)){
|
||||
Write-Output "Chocolatey is not installed, installing now"
|
||||
Write-Output "IF YOU KEEP GETTING THIS MESSAGE ON EVERY BUILD, TRY RESTARTING THE GITLAB RUNNER SO IT GETS CHOCO INTO IT'S ENV"
|
||||
Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
|
||||
}
|
||||
else{
|
||||
Write-Output "Chocolatey version $chocoVersion is already installed"
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import lbry.build_info as build_info_mod
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def _check_and_set(d: dict, key: str, value: str):
|
||||
try:
|
||||
d[key]
|
||||
except KeyError:
|
||||
raise Exception(f"{key} var does not exist in {build_info_mod.__file__}")
|
||||
d[key] = value
|
||||
|
||||
|
||||
def main():
|
||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||
|
||||
docker_tag = os.getenv('DOCKER_TAG')
|
||||
if docker_tag:
|
||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||
_check_and_set(build_info, "BUILD", "docker")
|
||||
else:
|
||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
||||
|
||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||
with open(build_info_mod.__file__, 'w') as f:
|
||||
f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
|
||||
# entrypoint for wallet server Docker image
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
|
||||
|
||||
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
|
||||
files="$(ls)"
|
||||
echo "Downloading wallet snapshot from $SNAPSHOT_URL"
|
||||
wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL"
|
||||
echo "Extracting snapshot..."
|
||||
filename="$(grep -vf <(echo "$files") <(ls))" # finds the file that was not there before
|
||||
case "$filename" in
|
||||
*.tgz|*.tar.gz|*.tar.bz2 ) tar xvf "$filename" --directory /database ;;
|
||||
*.zip ) unzip "$filename" -d /database ;;
|
||||
* ) echo "Don't know how to extract ${filename}. SNAPSHOT COULD NOT BE LOADED" && exit 1 ;;
|
||||
esac
|
||||
rm "$filename"
|
||||
fi
|
||||
|
||||
/home/lbry/.local/bin/torba-server "$@"
|
||||
|
Before Width: | Height: | Size: 7.4 KiB After Width: | Height: | Size: 7.4 KiB |
|
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 19 KiB |
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 2.6 KiB |
|
Before Width: | Height: | Size: 6.1 KiB After Width: | Height: | Size: 6.1 KiB |
|
Before Width: | Height: | Size: 97 KiB After Width: | Height: | Size: 97 KiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 361 KiB After Width: | Height: | Size: 361 KiB |
|
Before Width: | Height: | Size: 5.3 KiB After Width: | Height: | Size: 5.3 KiB |
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 31 KiB After Width: | Height: | Size: 31 KiB |
|
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 2.6 KiB |
@@ -1 +0,0 @@
|
||||
/CHANGELOG.md merge=union
|
||||
@@ -1,13 +0,0 @@
|
||||
.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
/lbry-venv
|
||||
|
||||
lbry.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
@@ -1,441 +0,0 @@
|
||||
[MASTER]
|
||||
|
||||
# Specify a configuration file.
|
||||
#rcfile=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS,schema
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
# `\.#.*` - add emacs tmp files to the blacklist
|
||||
ignore-patterns=\.#.*
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=yes
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=4
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
# extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
# be used to obtain the result of joining multiple strings with the addition
|
||||
# operator. Joining a lot of strings can lead to a maximum recursion error in
|
||||
# Pylint and this flag can prevent that. It has one side effect, the resulting
|
||||
# AST will be different than the one from reality.
|
||||
optimize-ast=no
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||
confidence=
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then re-enable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
# --disable=W"
|
||||
disable=
|
||||
anomalous-backslash-in-string,
|
||||
arguments-differ,
|
||||
attribute-defined-outside-init,
|
||||
bad-continuation,
|
||||
bare-except,
|
||||
broad-except,
|
||||
cell-var-from-loop,
|
||||
consider-iterating-dictionary,
|
||||
dangerous-default-value,
|
||||
duplicate-code,
|
||||
fixme,
|
||||
invalid-name,
|
||||
len-as-condition,
|
||||
locally-disabled,
|
||||
logging-not-lazy,
|
||||
missing-docstring,
|
||||
no-else-return,
|
||||
no-init,
|
||||
no-member,
|
||||
no-self-use,
|
||||
protected-access,
|
||||
redefined-builtin,
|
||||
redefined-outer-name,
|
||||
redefined-variable-type,
|
||||
relative-import,
|
||||
signature-differs,
|
||||
super-init-not-called,
|
||||
too-few-public-methods,
|
||||
too-many-arguments,
|
||||
too-many-branches,
|
||||
too-many-instance-attributes,
|
||||
too-many-lines,
|
||||
too-many-locals,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
too-many-statements,
|
||||
trailing-newlines,
|
||||
undefined-loop-variable,
|
||||
ungrouped-imports,
|
||||
unnecessary-lambda,
|
||||
unused-argument,
|
||||
unused-variable,
|
||||
wrong-import-order,
|
||||
wrong-import-position,
|
||||
deprecated-lambda,
|
||||
simplifiable-if-statement,
|
||||
unidiomatic-typecheck,
|
||||
inconsistent-return-statements,
|
||||
keyword-arg-before-vararg,
|
||||
assignment-from-no-return,
|
||||
useless-return,
|
||||
assignment-from-none,
|
||||
stop-iteration-return
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html. You can also give a reporter class, eg
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
output-format=text
|
||||
|
||||
# Put messages in a separate file for each module / package specified on the
|
||||
# command line instead of printing them on stdout. Reports (if any) will be
|
||||
# written in a file name "pylint_global.[txt|html]".
|
||||
files-output=no
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=no
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details
|
||||
#msg-template=
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||
# not used).
|
||||
dummy-variables-rgx=_$|dummy
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,_cb
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format
|
||||
logging-modules=logging
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# List of builtins function names that should not be used, separated by a comma
|
||||
bad-functions=map,filter,input
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
# allow `d` as its used frequently for deferred callback chains
|
||||
good-names=i,j,k,ex,Run,_,d
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=foo,bar,baz,toto,tutu,tata
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name
|
||||
include-naming-hint=no
|
||||
|
||||
# Regular expression matching correct function names
|
||||
function-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for function names
|
||||
function-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct variable names
|
||||
variable-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for variable names
|
||||
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct constant names
|
||||
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Naming hint for constant names
|
||||
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Regular expression matching correct attribute names
|
||||
attr-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for attribute names
|
||||
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct argument names
|
||||
argument-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for argument names
|
||||
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct class attribute names
|
||||
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Naming hint for class attribute names
|
||||
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Regular expression matching correct inline iteration names
|
||||
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Naming hint for inline iteration names
|
||||
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class names
|
||||
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Naming hint for class names
|
||||
class-name-hint=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Regular expression matching correct module names
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Naming hint for module names
|
||||
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Regular expression matching correct method names
|
||||
method-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for method names
|
||||
method-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=^_
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=-1
|
||||
|
||||
|
||||
[ELIF]
|
||||
|
||||
# Maximum number of nested blocks for function / method body
|
||||
max-nested-blocks=5
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||
# install python-enchant package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to indicated private dictionary in
|
||||
# --spelling-private-dict-file option instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=120
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=no
|
||||
|
||||
# List of optional constructs for which whitespace checking is disabled. `dict-
|
||||
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
|
||||
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
|
||||
# `empty-line` allows space-only lines.
|
||||
no-space-check=trailing-comma,dict-separator
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=1000
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,XXX,TODO
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Ignore imports when computing similarities.
|
||||
ignore-imports=no
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||
ignore-mixin-members=yes
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=leveldb,distutils
|
||||
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
# ignored-classes=
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,TERMIOS,Bastion,rexec
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=10
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
ignored-argument-names=_.*
|
||||
|
||||
# Maximum number of locals for function / method body
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of return / yield for function / method body
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of branch for function / method body
|
||||
max-branches=12
|
||||
|
||||
# Maximum number of statements in function / method body
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=8
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
# Maximum number of boolean expressions in a if statement
|
||||
max-bool-expr=5
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,__new__,setUp
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,_fields,_replace,_source,_make
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when being caught. Defaults to
|
||||
# "Exception"
|
||||
overgeneral-exceptions=Exception
|
||||
@@ -1,56 +0,0 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry-sdk) [](https://codecov.io/gh/lbryio/lbry-sdk)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
|
||||
## Usage
|
||||
|
||||
Run `lbrynet start` to launch the API server.
|
||||
|
||||
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
|
||||
|
||||
With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
|
||||
## Running from source
|
||||
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
|
||||
## License
|
||||
|
||||
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
|
||||
@@ -0,0 +1,2 @@
|
||||
__version__ = "0.73.1"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -0,0 +1,6 @@
|
||||
from lbry.utils import get_lbry_hash_obj
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
BLOBHASH_LENGTH = get_lbry_hash_obj().digest_size * 2
|
||||
@@ -9,18 +9,20 @@ from io import BytesIO
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
from lbry.cryptoutils import backend, get_lbry_hash_obj
|
||||
from lbry.utils import get_lbry_hash_obj
|
||||
from lbry.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
|
||||
|
||||
from lbry.blob import MAX_BLOB_SIZE, blobhash_length
|
||||
from lbry.blob import MAX_BLOB_SIZE, BLOBHASH_LENGTH
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
from lbry.blob.writer import HashBlobWriter
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
HEXMATCH = re.compile("^[a-f,0-9]+$")
|
||||
BACKEND = default_backend()
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
@@ -31,11 +33,11 @@ def is_valid_blobhash(blobhash: str) -> bool:
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
|
||||
return len(blobhash) == BLOBHASH_LENGTH and HEXMATCH.match(blobhash)
|
||||
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND)
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
|
||||
@@ -47,7 +49,7 @@ def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tupl
|
||||
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
|
||||
if len(data) != length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
|
||||
@@ -108,7 +110,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -142,7 +144,7 @@ class AbstractBlob:
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
peer, writer = self.writers.popitem()
|
||||
_, writer = self.writers.popitem()
|
||||
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
|
||||
writer.finished.cancel()
|
||||
while self.readers:
|
||||
@@ -163,7 +165,10 @@ class AbstractBlob:
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
try:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
except (ConnectionError, BrokenPipeError, RuntimeError, OSError, AttributeError):
|
||||
return -1
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
@@ -193,11 +198,17 @@ class AbstractBlob:
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -256,9 +267,11 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -314,8 +327,14 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
from lbry.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbry.cryptoutils import get_lbry_hash_obj
|
||||
from lbry.utils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -95,7 +95,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]:
|
||||
async def _download_blob(self) -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]: # pylint: disable=too-many-return-statements
|
||||
"""
|
||||
:return: download success (bool), connected protocol (BlobExchangeClientProtocol)
|
||||
"""
|
||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
@@ -213,11 +215,11 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
def connection_lost(self, exc):
|
||||
if self.connection_manager:
|
||||
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(exc),
|
||||
str(type(exc)))
|
||||
self.close()
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ class BlobDownloader:
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connection_failures: typing.List['KademliaPeer'] = []
|
||||
self.connection_failures: typing.Set['KademliaPeer'] = set()
|
||||
self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {}
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
@@ -48,7 +48,7 @@ class BlobDownloader:
|
||||
connection_id=connection_id, connection_manager=self.blob_manager.connection_manager
|
||||
)
|
||||
if not bytes_received and not protocol and peer not in self.connection_failures:
|
||||
self.connection_failures.append(peer)
|
||||
self.connection_failures.add(peer)
|
||||
if not protocol and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
@@ -87,32 +87,42 @@ class BlobDownloader:
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
tried_for_this_blob: typing.Set['KademliaPeer'] = set()
|
||||
try:
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set()
|
||||
batch: typing.Set['KademliaPeer'] = set(self.connections.keys())
|
||||
while not self.peer_queue.empty():
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active, %s connections",
|
||||
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
re_add: typing.Set['KademliaPeer'] = set()
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored or peer in self.active_connections:
|
||||
if peer in self.ignored:
|
||||
continue
|
||||
if peer in tried_for_this_blob:
|
||||
continue
|
||||
if peer in self.active_connections:
|
||||
if peer not in re_add:
|
||||
re_add.add(peer)
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
just_probe = len(self.connections) == 0
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id, just_probe))
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
tried_for_this_blob.add(peer)
|
||||
if not re_add:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
blob.close()
|
||||
if self.loop.is_running():
|
||||
self.loop.call_soon(self.cleanup_active)
|
||||
|
||||
def close(self):
|
||||
self.connection_failures.clear()
|
||||
@@ -46,7 +46,7 @@ class BlobAvailabilityRequest(BlobMessage):
|
||||
|
||||
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
|
||||
**kwargs) -> None:
|
||||
assert len(requested_blobs)
|
||||
assert len(requested_blobs) > 0
|
||||
self.requested_blobs = requested_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
@@ -134,9 +134,9 @@ class BlobErrorResponse(BlobMessage):
|
||||
}
|
||||
|
||||
|
||||
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest, # pylint: disable=invalid-name
|
||||
BlobPaymentAddressRequest]
|
||||
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse, # pylint: disable=invalid-name
|
||||
BlobErrorResponse, BlobPaymentAddressResponse]
|
||||
|
||||
|
||||
@@ -157,10 +157,10 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
|
||||
except ValueError:
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
@@ -179,7 +179,7 @@ class BlobRequest:
|
||||
return d
|
||||
|
||||
def _get_request(self, request_type: blob_request_types):
|
||||
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
|
||||
request = tuple(filter(lambda r: type(r) == request_type, self.requests)) # pylint: disable=unidiomatic-typecheck
|
||||
if request:
|
||||
return request[0]
|
||||
|
||||
@@ -235,7 +235,7 @@ class BlobResponse:
|
||||
return d
|
||||
|
||||
def _get_response(self, response_type: blob_response_types):
|
||||
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
|
||||
response = tuple(filter(lambda r: type(r) == response_type, self.responses)) # pylint: disable=unidiomatic-typecheck
|
||||
if response:
|
||||
return response[0]
|
||||
|
||||
@@ -280,4 +280,3 @@ class BlobResponse:
|
||||
if response_type.key in response
|
||||
])
|
||||
return cls(requests, extra)
|
||||
|
||||
@@ -96,22 +96,30 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
bh = blob.blob_hash[:8]
|
||||
log.debug("send %s to %s:%i", bh, peer_address, peer_port)
|
||||
blob_hash = blob.blob_hash[:8]
|
||||
log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port)
|
||||
self.started_transfer.set()
|
||||
try:
|
||||
sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout, loop=self.loop)
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
|
||||
log.info("sent %s (%i bytes) to %s:%i", bh, sent, peer_address, peer_port)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError, asyncio.TimeoutError) as err:
|
||||
if isinstance(err, asyncio.TimeoutError):
|
||||
log.debug("timed out sending blob %s to %s", bh, peer_address)
|
||||
if sent and sent > 0:
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port)
|
||||
else:
|
||||
log.debug("stopped sending %s to %s:%i", bh, peer_address, peer_port)
|
||||
self.close()
|
||||
log.debug("stopped sending %s to %s:%i", blob_hash, peer_address, peer_port)
|
||||
return
|
||||
except (OSError, ValueError, asyncio.TimeoutError) as err:
|
||||
if isinstance(err, asyncio.TimeoutError):
|
||||
log.debug("timed out sending blob %s to %s", blob_hash, peer_address)
|
||||
else:
|
||||
log.warning("could not read blob %s to send %s:%i", blob_hash, peer_address, peer_port)
|
||||
self.close()
|
||||
return
|
||||
finally:
|
||||
self.transfer_finished.set()
|
||||
if responses:
|
||||
else:
|
||||
log.info("don't have %s to send %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
if responses and not self.transport.is_closing():
|
||||
self.send_response(responses)
|
||||
|
||||
def data_received(self, data):
|
||||
@@ -122,7 +130,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
return
|
||||
if data:
|
||||
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
_, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
return
|
||||
@@ -1,3 +1,4 @@
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
BUILD_COMMIT = "source installation"
|
||||
COMMIT_HASH = "none"
|
||||
DOCKER_TAG = "none"
|
||||
@@ -3,18 +3,19 @@ import re
|
||||
import sys
|
||||
import typing
|
||||
import logging
|
||||
import yaml
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
|
||||
import yaml
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
from lbry.error import InvalidCurrencyError
|
||||
from lbry.dht import constants
|
||||
from torba.client.coinselection import STRATEGIES
|
||||
from lbry.wallet.coinselection import STRATEGIES
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
NOT_SET = type('NOT_SET', (object,), {})
|
||||
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
CURRENCIES = {
|
||||
@@ -35,7 +36,7 @@ class Setting(typing.Generic[T]):
|
||||
self.metavar = metavar
|
||||
|
||||
def __set_name__(self, owner, name):
|
||||
self.name = name
|
||||
self.name = name # pylint: disable=attribute-defined-outside-init
|
||||
|
||||
@property
|
||||
def cli_name(self):
|
||||
@@ -63,13 +64,13 @@ class Setting(typing.Generic[T]):
|
||||
for location in obj.modify_order:
|
||||
location[self.name] = val
|
||||
|
||||
def validate(self, val):
|
||||
def validate(self, value):
|
||||
raise NotImplementedError()
|
||||
|
||||
def deserialize(self, value):
|
||||
def deserialize(self, value): # pylint: disable=no-self-use
|
||||
return value
|
||||
|
||||
def serialize(self, value):
|
||||
def serialize(self, value): # pylint: disable=no-self-use
|
||||
return value
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
@@ -82,14 +83,18 @@ class Setting(typing.Generic[T]):
|
||||
|
||||
|
||||
class String(Setting[str]):
|
||||
def validate(self, val):
|
||||
assert isinstance(val, str), \
|
||||
def validate(self, value):
|
||||
assert isinstance(value, str), \
|
||||
f"Setting '{self.name}' must be a string."
|
||||
|
||||
# TODO: removes this after pylint starts to understand generics
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
return super().__get__(obj, owner)
|
||||
|
||||
|
||||
class Integer(Setting[int]):
|
||||
def validate(self, val):
|
||||
assert isinstance(val, int), \
|
||||
def validate(self, value):
|
||||
assert isinstance(value, int), \
|
||||
f"Setting '{self.name}' must be an integer."
|
||||
|
||||
def deserialize(self, value):
|
||||
@@ -97,8 +102,8 @@ class Integer(Setting[int]):
|
||||
|
||||
|
||||
class Float(Setting[float]):
|
||||
def validate(self, val):
|
||||
assert isinstance(val, float), \
|
||||
def validate(self, value):
|
||||
assert isinstance(value, float), \
|
||||
f"Setting '{self.name}' must be a decimal."
|
||||
|
||||
def deserialize(self, value):
|
||||
@@ -106,8 +111,8 @@ class Float(Setting[float]):
|
||||
|
||||
|
||||
class Toggle(Setting[bool]):
|
||||
def validate(self, val):
|
||||
assert isinstance(val, bool), \
|
||||
def validate(self, value):
|
||||
assert isinstance(value, bool), \
|
||||
f"Setting '{self.name}' must be a true/false value."
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
@@ -127,10 +132,10 @@ class Toggle(Setting[bool]):
|
||||
|
||||
|
||||
class Path(String):
|
||||
def __init__(self, doc: str, default: str = '', *args, **kwargs):
|
||||
def __init__(self, doc: str, *args, default: str = '', **kwargs):
|
||||
super().__init__(doc, default, *args, **kwargs)
|
||||
|
||||
def __get__(self, obj, owner):
|
||||
def __get__(self, obj, owner) -> str:
|
||||
value = super().__get__(obj, owner)
|
||||
if isinstance(value, str):
|
||||
return os.path.expanduser(os.path.expandvars(value))
|
||||
@@ -204,16 +209,16 @@ class StringChoice(String):
|
||||
raise ValueError(f"Default value must be one of: {', '.join(valid_values)}")
|
||||
self.valid_values = valid_values
|
||||
|
||||
def validate(self, val):
|
||||
super().validate(val)
|
||||
if val not in self.valid_values:
|
||||
def validate(self, value):
|
||||
super().validate(value)
|
||||
if value not in self.valid_values:
|
||||
raise ValueError(f"Setting '{self.name}' value must be one of: {', '.join(self.valid_values)}")
|
||||
|
||||
|
||||
class ListSetting(Setting[list]):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
def validate(self, value):
|
||||
assert isinstance(value, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list."
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
@@ -226,10 +231,10 @@ class ListSetting(Setting[list]):
|
||||
|
||||
class Servers(ListSetting):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
def validate(self, value):
|
||||
assert isinstance(value, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list of servers."
|
||||
for idx, server in enumerate(val):
|
||||
for idx, server in enumerate(value):
|
||||
assert isinstance(server, (tuple, list)) and len(server) == 2, \
|
||||
f"Server defined '{server}' at index {idx} in setting " \
|
||||
f"'{self.name}' must be a tuple or list of two items."
|
||||
@@ -260,10 +265,10 @@ class Servers(ListSetting):
|
||||
|
||||
class Strings(ListSetting):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
def validate(self, value):
|
||||
assert isinstance(value, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list of strings."
|
||||
for idx, string in enumerate(val):
|
||||
for idx, string in enumerate(value):
|
||||
assert isinstance(string, str), \
|
||||
f"Value of '{string}' at index {idx} in setting " \
|
||||
f"'{self.name}' must be a string."
|
||||
@@ -272,14 +277,23 @@ class Strings(ListSetting):
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
def __init__(self, environ: dict):
|
||||
self.environ = environ
|
||||
def __init__(self, config: 'BaseConfig', environ: dict):
|
||||
self.configuration = config
|
||||
self.data = {}
|
||||
if environ:
|
||||
self.load(environ)
|
||||
|
||||
def load(self, environ):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.data[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return f'{self.PREFIX}{item.upper()}' in self.environ
|
||||
return item in self.data
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.environ[f'{self.PREFIX}{item.upper()}']
|
||||
return self.data[item]
|
||||
|
||||
|
||||
class ArgumentAccess:
|
||||
@@ -438,7 +452,7 @@ class BaseConfig:
|
||||
self.arguments = ArgumentAccess(self, args)
|
||||
|
||||
def set_environment(self, environ=None):
|
||||
self.environment = EnvironmentAccess(environ or os.environ)
|
||||
self.environment = EnvironmentAccess(self, environ or os.environ)
|
||||
|
||||
def set_persisted(self, config_file_path=None):
|
||||
if config_file_path is None:
|
||||
@@ -457,7 +471,27 @@ class BaseConfig:
|
||||
self.persisted.save()
|
||||
|
||||
|
||||
class CLIConfig(BaseConfig):
|
||||
class TranscodeConfig(BaseConfig):
|
||||
|
||||
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
|
||||
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
|
||||
'', previous_names=['ffmpeg_folder'])
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
class CLIConfig(TranscodeConfig):
|
||||
|
||||
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT')
|
||||
|
||||
@@ -497,6 +531,7 @@ class Config(CLIConfig):
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# routing table
|
||||
@@ -511,7 +546,7 @@ class Config(CLIConfig):
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
@@ -539,6 +574,7 @@ class Config(CLIConfig):
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
@@ -550,19 +586,24 @@ class Config(CLIConfig):
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
|
||||
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
|
||||
('cdn.reflector.lbry.com', 5567)
|
||||
])
|
||||
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('spv1.lbry.com', 50001),
|
||||
('spv2.lbry.com', 50001),
|
||||
('spv3.lbry.com', 50001),
|
||||
('spv4.lbry.com', 50001),
|
||||
('spv5.lbry.com', 50001),
|
||||
('spv6.lbry.com', 50001),
|
||||
('spv7.lbry.com', 50001),
|
||||
('spv8.lbry.com', 50001),
|
||||
('spv9.lbry.com', 50001),
|
||||
('spv11.lbry.com', 50001),
|
||||
('spv12.lbry.com', 50001),
|
||||
('spv13.lbry.com', 50001),
|
||||
('spv14.lbry.com', 50001),
|
||||
('spv15.lbry.com', 50001),
|
||||
('spv16.lbry.com', 50001),
|
||||
('spv17.lbry.com', 50001),
|
||||
('spv18.lbry.com', 50001),
|
||||
('spv19.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
@@ -575,14 +616,12 @@ class Config(CLIConfig):
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
@@ -597,6 +636,11 @@ class Config(CLIConfig):
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
STRATEGIES, "standard")
|
||||
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
)
|
||||
|
||||
@property
|
||||
def streaming_host(self):
|
||||
return self.streaming_server.split(':')[0]
|
||||
@@ -630,9 +674,13 @@ class Config(CLIConfig):
|
||||
|
||||
|
||||
def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle # pylint: disable=import-outside-toplevel
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
||||
PathNotFoundException # pylint: disable=import-outside-toplevel
|
||||
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
try:
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
except PathNotFoundException:
|
||||
download_dir = os.getcwd()
|
||||
|
||||
# old
|
||||
appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
|
||||
@@ -644,7 +692,6 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
# new
|
||||
data_dir = user_data_dir('lbrynet', 'lbry')
|
||||
lbryum_dir = user_data_dir('lbryum', 'lbry')
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
@@ -658,9 +705,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
@@ -72,13 +72,13 @@ class ConnectionManager:
|
||||
self._status['outgoing_bps'].clear()
|
||||
now = time.perf_counter()
|
||||
while self.outgoing:
|
||||
k, v = self.outgoing.popitem()
|
||||
self._status['total_sent'] += v
|
||||
self._status['outgoing_bps'][k] = v / (now - last)
|
||||
k, sent = self.outgoing.popitem()
|
||||
self._status['total_sent'] += sent
|
||||
self._status['outgoing_bps'][k] = sent / (now - last)
|
||||
while self.incoming:
|
||||
k, v = self.incoming.popitem()
|
||||
self._status['total_received'] += v
|
||||
self._status['incoming_bps'][k] = v / (now - last)
|
||||
k, received = self.incoming.popitem()
|
||||
self._status['total_received'] += received
|
||||
self._status['incoming_bps'][k] = received / (now - last)
|
||||
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
|
||||
)) / 1000000.0
|
||||
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
|
||||
@@ -0,0 +1,2 @@
|
||||
CENT = 1000000
|
||||
COIN = 100*CENT
|
||||
@@ -0,0 +1,86 @@
|
||||
from lbry.crypto.hash import double_sha256
|
||||
from lbry.crypto.util import bytes_to_int, int_to_bytes
|
||||
|
||||
|
||||
class Base58Error(Exception):
|
||||
""" Exception used for Base58 errors. """
|
||||
|
||||
|
||||
class Base58:
|
||||
""" Class providing base 58 functionality. """
|
||||
|
||||
chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
assert len(chars) == 58
|
||||
char_map = {c: n for n, c in enumerate(chars)}
|
||||
|
||||
@classmethod
|
||||
def char_value(cls, c):
|
||||
val = cls.char_map.get(c)
|
||||
if val is None:
|
||||
raise Base58Error(f'invalid base 58 character "{c}"')
|
||||
return val
|
||||
|
||||
@classmethod
|
||||
def decode(cls, txt):
|
||||
""" Decodes txt into a big-endian bytearray. """
|
||||
if isinstance(txt, memoryview):
|
||||
txt = str(txt)
|
||||
|
||||
if isinstance(txt, bytes):
|
||||
txt = txt.decode()
|
||||
|
||||
if not isinstance(txt, str):
|
||||
raise TypeError('a string is required')
|
||||
|
||||
if not txt:
|
||||
raise Base58Error('string cannot be empty')
|
||||
|
||||
value = 0
|
||||
for c in txt:
|
||||
value = value * 58 + cls.char_value(c)
|
||||
|
||||
result = int_to_bytes(value)
|
||||
|
||||
# Prepend leading zero bytes if necessary
|
||||
count = 0
|
||||
for c in txt:
|
||||
if c != '1':
|
||||
break
|
||||
count += 1
|
||||
if count:
|
||||
result = bytes((0,)) * count + result
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def encode(cls, be_bytes):
|
||||
"""Converts a big-endian bytearray into a base58 string."""
|
||||
value = bytes_to_int(be_bytes)
|
||||
|
||||
txt = ''
|
||||
while value:
|
||||
value, mod = divmod(value, 58)
|
||||
txt += cls.chars[mod]
|
||||
|
||||
for byte in be_bytes:
|
||||
if byte != 0:
|
||||
break
|
||||
txt += '1'
|
||||
|
||||
return txt[::-1]
|
||||
|
||||
@classmethod
|
||||
def decode_check(cls, txt, hash_fn=double_sha256):
|
||||
""" Decodes a Base58Check-encoded string to a payload. The version prefixes it. """
|
||||
be_bytes = cls.decode(txt)
|
||||
result, check = be_bytes[:-4], be_bytes[-4:]
|
||||
if check != hash_fn(result)[:4]:
|
||||
raise Base58Error(f'invalid base 58 checksum for {txt}')
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def encode_check(cls, payload, hash_fn=double_sha256):
|
||||
""" Encodes a payload bytearray (which includes the version byte(s))
|
||||
into a Base58Check string."""
|
||||
be_bytes = payload + hash_fn(payload)[:4]
|
||||
return cls.encode(be_bytes)
|
||||
@@ -0,0 +1,71 @@
|
||||
import os
|
||||
import base64
|
||||
import typing
|
||||
from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
from lbry.error import InvalidPasswordError
|
||||
from lbry.crypto.hash import double_sha256
|
||||
|
||||
|
||||
def aes_encrypt(secret: str, value: str, init_vector: bytes = None) -> str:
|
||||
if init_vector is not None:
|
||||
assert len(init_vector) == 16
|
||||
else:
|
||||
init_vector = os.urandom(16)
|
||||
key = double_sha256(secret.encode())
|
||||
encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor()
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
padded_data = padder.update(value.encode()) + padder.finalize()
|
||||
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
|
||||
return base64.b64encode(init_vector + encrypted_data).decode()
|
||||
|
||||
|
||||
def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]:
|
||||
try:
|
||||
data = base64.b64decode(value.encode())
|
||||
key = double_sha256(secret.encode())
|
||||
init_vector, data = data[:16], data[16:]
|
||||
decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor()
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
result = unpadder.update(decryptor.update(data)) + unpadder.finalize()
|
||||
return result.decode(), init_vector
|
||||
except UnicodeDecodeError:
|
||||
raise InvalidPasswordError()
|
||||
except ValueError as e:
|
||||
if e.args[0] == 'Invalid padding bytes.':
|
||||
raise InvalidPasswordError()
|
||||
raise
|
||||
|
||||
|
||||
def better_aes_encrypt(secret: str, value: bytes) -> bytes:
|
||||
init_vector = os.urandom(16)
|
||||
key = scrypt(secret.encode(), salt=init_vector)
|
||||
encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor()
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
padded_data = padder.update(value) + padder.finalize()
|
||||
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
|
||||
return base64.b64encode(b's:8192:16:1:' + init_vector + encrypted_data)
|
||||
|
||||
|
||||
def better_aes_decrypt(secret: str, value: bytes) -> bytes:
|
||||
try:
|
||||
data = base64.b64decode(value)
|
||||
_, scryp_n, scrypt_r, scrypt_p, data = data.split(b':', maxsplit=4)
|
||||
init_vector, data = data[:16], data[16:]
|
||||
key = scrypt(secret.encode(), init_vector, int(scryp_n), int(scrypt_r), int(scrypt_p))
|
||||
decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor()
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
return unpadder.update(decryptor.update(data)) + unpadder.finalize()
|
||||
except ValueError as e:
|
||||
if e.args[0] == 'Invalid padding bytes.':
|
||||
raise InvalidPasswordError()
|
||||
raise
|
||||
|
||||
|
||||
def scrypt(passphrase, salt, scrypt_n=1<<13, scrypt_r=16, scrypt_p=1):
|
||||
kdf = Scrypt(salt, length=32, n=scrypt_n, r=scrypt_r, p=scrypt_p, backend=default_backend())
|
||||
return kdf.derive(passphrase)
|
||||
@@ -0,0 +1,47 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
|
||||
def sha256(x):
|
||||
""" Simple wrapper of hashlib sha256. """
|
||||
return hashlib.sha256(x).digest()
|
||||
|
||||
|
||||
def sha512(x):
|
||||
""" Simple wrapper of hashlib sha512. """
|
||||
return hashlib.sha512(x).digest()
|
||||
|
||||
|
||||
def ripemd160(x):
|
||||
""" Simple wrapper of hashlib ripemd160. """
|
||||
h = hashlib.new('ripemd160')
|
||||
h.update(x)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def double_sha256(x):
|
||||
""" SHA-256 of SHA-256, as used extensively in bitcoin. """
|
||||
return sha256(sha256(x))
|
||||
|
||||
|
||||
def hmac_sha512(key, msg):
|
||||
""" Use SHA-512 to provide an HMAC. """
|
||||
return hmac.new(key, msg, hashlib.sha512).digest()
|
||||
|
||||
|
||||
def hash160(x):
|
||||
""" RIPEMD-160 of SHA-256.
|
||||
Used to make bitcoin addresses from pubkeys. """
|
||||
return ripemd160(sha256(x))
|
||||
|
||||
|
||||
def hash_to_hex_str(x):
|
||||
""" Convert a big-endian binary hash to displayed hex string.
|
||||
Display form of a binary hash is reversed and converted to hex. """
|
||||
return hexlify(reversed(x))
|
||||
|
||||
|
||||
def hex_str_to_hash(x):
|
||||
""" Convert a displayed hex string to a binary hash. """
|
||||
return reversed(unhexlify(x))
|
||||
@@ -0,0 +1,13 @@
|
||||
from binascii import unhexlify, hexlify
|
||||
|
||||
|
||||
def bytes_to_int(be_bytes):
|
||||
""" Interprets a big-endian sequence of bytes as an integer. """
|
||||
return int(hexlify(be_bytes), 16)
|
||||
|
||||
|
||||
def int_to_bytes(value):
|
||||
""" Converts an integer to a big-endian sequence of bytes. """
|
||||
length = (value.bit_length() + 7) // 8
|
||||
s = '%x' % value
|
||||
return unhexlify(('0' * (len(s) % 2) + s).zfill(length * 2))
|
||||
@@ -24,7 +24,7 @@ class BlobAnnouncer:
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
@@ -38,7 +38,7 @@ class BlobAnnouncer:
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue):
|
||||
while len(self.announce_queue) > 0:
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
@@ -0,0 +1,41 @@
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
HASH_CLASS = hashlib.sha384 # pylint: disable=invalid-name
|
||||
HASH_LENGTH = HASH_CLASS().digest_size
|
||||
HASH_BITS = HASH_LENGTH * 8
|
||||
ALPHA = 5
|
||||
K = 8
|
||||
SPLIT_BUCKETS_UNDER_INDEX = 1
|
||||
REPLACEMENT_CACHE_SIZE = 8
|
||||
RPC_TIMEOUT = 5.0
|
||||
RPC_ATTEMPTS = 5
|
||||
RPC_ATTEMPTS_PRUNING_WINDOW = 600
|
||||
ITERATIVE_LOOKUP_DELAY = RPC_TIMEOUT / 2.0 # TODO: use config val / 2 if rpc timeout is provided
|
||||
REFRESH_INTERVAL = 3600 # 1 hour
|
||||
REPLICATE_INTERVAL = REFRESH_INTERVAL
|
||||
DATA_EXPIRATION = 86400 # 24 hours
|
||||
TOKEN_SECRET_REFRESH_INTERVAL = 300 # 5 minutes
|
||||
MAYBE_PING_DELAY = 300 # 5 minutes
|
||||
CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
|
||||
RPC_ID_LENGTH = 20
|
||||
PROTOCOL_VERSION = 1
|
||||
BOTTOM_OUT_LIMIT = 3
|
||||
MSG_SIZE_LIMIT = 1400
|
||||
|
||||
|
||||
def digest(data: bytes) -> bytes:
|
||||
h = HASH_CLASS()
|
||||
h.update(data)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def generate_id(num=None) -> bytes:
|
||||
if num is not None:
|
||||
return digest(str(num).encode())
|
||||
else:
|
||||
return digest(os.urandom(32))
|
||||
|
||||
|
||||
def generate_rpc_id(num=None) -> bytes:
|
||||
return generate_id(num)[:RPC_ID_LENGTH]
|
||||
@@ -2,6 +2,7 @@ import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
@@ -18,8 +19,9 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX,
|
||||
storage: typing.Optional['SQLiteStorage'] = None):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
@@ -28,6 +30,7 @@ class Node:
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
self._refresh_task: asyncio.Task = None
|
||||
self._storage = storage
|
||||
|
||||
async def refresh_node(self, force_once=False):
|
||||
while True:
|
||||
@@ -59,7 +62,7 @@ class Node:
|
||||
if force_once:
|
||||
break
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
|
||||
self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None)
|
||||
await fut
|
||||
continue
|
||||
|
||||
@@ -67,16 +70,18 @@ class Node:
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
if self._storage:
|
||||
await self._storage.save_kademlia_peers(self.protocol.routing_table.get_peers())
|
||||
if force_once:
|
||||
break
|
||||
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
|
||||
self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None)
|
||||
await fut
|
||||
|
||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
assert len(hash_value) == constants.hash_length
|
||||
assert len(hash_value) == constants.HASH_LENGTH
|
||||
peers = await self.peer_search(hash_value)
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
@@ -89,8 +94,10 @@ class Node:
|
||||
)
|
||||
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
||||
if stored_to:
|
||||
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
len(stored_to), len(peers))
|
||||
log.debug(
|
||||
"Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
len(stored_to), len(peers)
|
||||
)
|
||||
else:
|
||||
log.debug("Failed announcing %s, stored to 0 peers", blob_hash[:8])
|
||||
return stored_to
|
||||
@@ -111,7 +118,7 @@ class Node:
|
||||
self.listening_port = None
|
||||
log.info("Stopped DHT node")
|
||||
|
||||
async def start_listening(self, interface: str = '') -> None:
|
||||
async def start_listening(self, interface: str = '0.0.0.0') -> None:
|
||||
if not self.listening_port:
|
||||
self.listening_port, _ = await self.loop.create_datagram_endpoint(
|
||||
lambda: self.protocol, (interface, self.internal_udp_port)
|
||||
@@ -121,60 +128,57 @@ class Node:
|
||||
else:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
async def join_network(self, interface: str = '0.0.0.0',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
def peers_from_urls(urls: typing.Optional[typing.List[typing.Tuple[bytes, str, int, int]]]):
|
||||
peer_addresses = []
|
||||
for node_id, address, udp_port, tcp_port in urls:
|
||||
if (node_id, address, udp_port, tcp_port) not in peer_addresses and \
|
||||
(address, udp_port) != (self.protocol.external_ip, self.protocol.udp_port):
|
||||
peer_addresses.append((node_id, address, udp_port, tcp_port))
|
||||
return [make_kademlia_peer(*peer_address) for peer_address in peer_addresses]
|
||||
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses and\
|
||||
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
if known_node_addresses:
|
||||
peers = [
|
||||
make_kademlia_peer(None, address, port)
|
||||
for (address, port) in known_node_addresses
|
||||
]
|
||||
while True:
|
||||
if not self.protocol.routing_table.get_peers():
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
self.protocol.peer_manager.reset()
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*peers, delay=0.0)
|
||||
peers.extend(await self.peer_search(self.protocol.node_id, shortlist=peers, count=32))
|
||||
if self.protocol.routing_table.get_peers():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
else:
|
||||
while True:
|
||||
if self.protocol.routing_table.get_peers():
|
||||
if not self.joined.is_set():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"joined dht, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts()
|
||||
)
|
||||
else:
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
seed_peers = peers_from_urls(
|
||||
await self._storage.get_persisted_kademlia_peers()
|
||||
) if self._storage else []
|
||||
if not seed_peers:
|
||||
try:
|
||||
seed_peers.extend(peers_from_urls([
|
||||
(None, await resolve_host(address, udp_port, 'udp'), udp_port, None)
|
||||
for address, udp_port in known_node_urls or []
|
||||
]))
|
||||
except socket.gaierror:
|
||||
await asyncio.sleep(30, loop=self.loop)
|
||||
continue
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
self.protocol.peer_manager.reset()
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0)
|
||||
await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32)
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
self.join_network(
|
||||
interface=interface, known_node_urls=known_node_urls
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.bottom_out_limit,
|
||||
max_results: int = constants.k) -> IterativeNodeFinder:
|
||||
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
|
||||
max_results: int = constants.K) -> IterativeNodeFinder:
|
||||
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
@@ -186,7 +190,7 @@ class Node:
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
|
||||
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
) -> typing.List['KademliaPeer']:
|
||||
peers = []
|
||||
@@ -197,25 +201,28 @@ class Node:
|
||||
peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
return peers[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
async def _accumulate_peers_for_value(self, search_queue: asyncio.Queue, result_queue: asyncio.Queue):
|
||||
tasks = []
|
||||
try:
|
||||
while True:
|
||||
blob_hash = await search_queue.get()
|
||||
tasks.append(self.loop.create_task(self._value_producer(blob_hash, result_queue)))
|
||||
tasks.append(self.loop.create_task(self._peers_for_value_producer(blob_hash, result_queue)))
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
|
||||
async def _value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
|
||||
async def ping(_peer):
|
||||
async def _peers_for_value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
|
||||
async def put_into_result_queue_after_pong(_peer):
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(_peer).ping()
|
||||
result_queue.put_nowait([_peer])
|
||||
log.debug("pong from %s:%i for %s", _peer.address, _peer.udp_port, blob_hash)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
# prioritize peers who reply to a dht ping first
|
||||
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
|
||||
|
||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
||||
to_put = []
|
||||
for peer in results:
|
||||
@@ -223,25 +230,31 @@ class Node:
|
||||
continue
|
||||
is_good = self.protocol.peer_manager.peer_is_good(peer)
|
||||
if is_good:
|
||||
# the peer has replied recently over UDP, it can probably be reached on the TCP port
|
||||
to_put.append(peer)
|
||||
elif is_good is None:
|
||||
udp_port_to_try = 4444
|
||||
if 3400 > peer.tcp_port > 3332:
|
||||
if not peer.udp_port:
|
||||
udp_port_to_try = (peer.tcp_port - 3333) + 4444
|
||||
elif 4500 > peer.tcp_port > 4443:
|
||||
if not peer.udp_port:
|
||||
udp_port_to_try = peer.tcp_port
|
||||
if not peer.udp_port:
|
||||
peer = make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
|
||||
self.loop.create_task(ping(peer))
|
||||
# TODO: use the same port for TCP and UDP
|
||||
# the udp port must be guessed
|
||||
# default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
|
||||
# including on a network with several nodes, then assume the udp port is proportionately
|
||||
# based on a starting port of 4444
|
||||
udp_port_to_try = peer.tcp_port
|
||||
if 3400 > peer.tcp_port > 3332:
|
||||
udp_port_to_try = (peer.tcp_port - 3333) + 4444
|
||||
self.loop.create_task(put_into_result_queue_after_pong(
|
||||
make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
|
||||
))
|
||||
else:
|
||||
self.loop.create_task(put_into_result_queue_after_pong(peer))
|
||||
else:
|
||||
# the peer is known to be bad/unreachable, skip trying to connect to it over TCP
|
||||
log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
|
||||
if to_put:
|
||||
result_queue.put_nowait(to_put)
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue(loop=self.loop)
|
||||
return q, self.loop.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None
|
||||
) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
|
||||
queue = peer_queue or asyncio.Queue(loop=self.loop)
|
||||
return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue))
|
||||
@@ -15,14 +15,29 @@ log = logging.getLogger(__name__)
|
||||
@lru_cache(1024)
|
||||
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
|
||||
udp_port: typing.Optional[int] = None,
|
||||
tcp_port: typing.Optional[int] = None) -> 'KademliaPeer':
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port)
|
||||
tcp_port: typing.Optional[int] = None,
|
||||
allow_localhost: bool = False) -> 'KademliaPeer':
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
||||
parsed_ip.is_reserved,
|
||||
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
@@ -70,7 +85,7 @@ class PeerManager:
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (0, None))
|
||||
if ts and ts > self._loop.time() - constants.token_secret_refresh_interval:
|
||||
if ts and ts > self._loop.time() - constants.TOKEN_SECRET_REFRESH_INTERVAL:
|
||||
return token
|
||||
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
@@ -93,23 +108,23 @@ class PeerManager:
|
||||
now = self._loop.time()
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
if last_failure and last_failure < now - constants.RPC_ATTEMPTS_PRUNING_WINDOW:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
for node_id, (age, token) in self._node_tokens.items(): # pylint: disable=unused-variable
|
||||
if age < now - constants.TOKEN_SECRET_REFRESH_INTERVAL:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int): # pylint: disable=too-many-return-statements
|
||||
"""
|
||||
:return: False if peer is bad, None if peer is unknown, or True if peer is good
|
||||
"""
|
||||
|
||||
delay = self._loop.time() - constants.check_refresh_interval
|
||||
delay = self._loop.time() - constants.CHECK_REFRESH_INTERVAL
|
||||
|
||||
# fixme: find a way to re-enable that without breaking other parts
|
||||
# if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
@@ -139,7 +154,7 @@ class PeerManager:
|
||||
def peer_is_good(self, peer: 'KademliaPeer'):
|
||||
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
|
||||
|
||||
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':
|
||||
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use
|
||||
node_id, address, tcp_port = decode_compact_address(compact_address)
|
||||
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
|
||||
|
||||
@@ -151,16 +166,17 @@ class KademliaPeer:
|
||||
udp_port: typing.Optional[int] = field(hash=True)
|
||||
tcp_port: typing.Optional[int] = field(compare=False, hash=False)
|
||||
protocol_version: typing.Optional[int] = field(default=1, compare=False, hash=False)
|
||||
allow_localhost: bool = field(default=False, compare=False, hash=False)
|
||||
|
||||
def __post_init__(self):
|
||||
if self._node_id is not None:
|
||||
if not len(self._node_id) == constants.hash_length:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_ipv4(self.address):
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
def update_tcp_port(self, tcp_port: int):
|
||||
@@ -22,7 +22,7 @@ class DictDataStore:
|
||||
for key in keys:
|
||||
to_remove = []
|
||||
for (peer, ts) in self._data_store[key]:
|
||||
if ts + constants.data_expiration < now or self._peer_manager.peer_is_good(peer) is False:
|
||||
if ts + constants.DATA_EXPIRATION < now or self._peer_manager.peer_is_good(peer) is False:
|
||||
to_remove.append((peer, ts))
|
||||
for item in to_remove:
|
||||
self._data_store[key].remove(item)
|
||||
@@ -43,7 +43,7 @@ class DictDataStore:
|
||||
"""
|
||||
now = self.loop.time()
|
||||
for (peer, ts) in self._data_store.get(key, []):
|
||||
if ts + constants.data_expiration > now:
|
||||
if ts + constants.DATA_EXPIRATION > now:
|
||||
yield peer
|
||||
|
||||
def has_peers_for_blob(self, key: bytes) -> bool:
|
||||
@@ -53,7 +53,7 @@ class DictDataStore:
|
||||
now = self.loop.time()
|
||||
if key in self._data_store:
|
||||
current = list(filter(lambda x: x[0] == contact, self._data_store[key]))
|
||||
if len(current):
|
||||
if len(current) > 0:
|
||||
self._data_store[key][self._data_store[key].index(current[0])] = contact, now
|
||||
else:
|
||||
self._data_store[key].append((contact, now))
|
||||
@@ -65,6 +65,6 @@ class DictDataStore:
|
||||
|
||||
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
|
||||
peers = set()
|
||||
for key, stored in self._data_store.items():
|
||||
for _, stored in self._data_store.items():
|
||||
peers.update(set(map(lambda tup: tup[0], stored)))
|
||||
return list(peers)
|
||||
@@ -9,17 +9,17 @@ class Distance:
|
||||
"""
|
||||
|
||||
def __init__(self, key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
self.key = key
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two: bytes) -> int:
|
||||
if len(key_two) != constants.hash_length:
|
||||
if len(key_two) != constants.HASH_LENGTH:
|
||||
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
def is_closer(self, a: bytes, b: bytes) -> bool:
|
||||
"""Returns true is `a` is closer to `key` than `b` is"""
|
||||
return self(a) < self(b)
|
||||
def is_closer(self, key_a: bytes, key_b: bytes) -> bool:
|
||||
"""Returns true is `key_a` is closer to `key` than `key_b` is"""
|
||||
return self(key_a) < self(key_b)
|
||||
@@ -1,15 +1,16 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
from collections import defaultdict
|
||||
import typing
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.serialization.datagram import PAGE_KEY
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
if TYPE_CHECKING:
|
||||
from lbry.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbry.dht.protocol.protocol import KademliaProtocol
|
||||
@@ -46,6 +47,7 @@ class FindValueResponse(FindResponse):
|
||||
self.token = result_dict[b'token']
|
||||
self.close_triples: typing.List[typing.Tuple[bytes, bytes, int]] = result_dict.get(b'contacts', [])
|
||||
self.found_compact_addresses = result_dict.get(key, [])
|
||||
self.pages = int(result_dict.get(PAGE_KEY, 0))
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
@@ -65,7 +67,7 @@ def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
:param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
|
||||
peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
|
||||
"""
|
||||
if len(key) != constants.hash_length:
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
return shortlist or routing_table.find_close_peers(key)
|
||||
|
||||
@@ -73,10 +75,10 @@ def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
class IterativeFinder:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
if len(key) != constants.hash_length:
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.loop = loop
|
||||
self.peer_manager = peer_manager
|
||||
@@ -130,7 +132,7 @@ class IterativeFinder:
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']: #pylint: disable=no-self-use
|
||||
"""
|
||||
Get an initial or cached result to be put into the Queue. Used for findValue requests where the blob
|
||||
has peers in the local data store of blobs announced to us
|
||||
@@ -151,7 +153,11 @@ class IterativeFinder:
|
||||
self._add_active(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
self._add_active(make_kademlia_peer(node_id, address, udp_port))
|
||||
try:
|
||||
self._add_active(make_kademlia_peer(node_id, address, udp_port))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
|
||||
peer.udp_port, address, udp_port)
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
@@ -179,7 +185,7 @@ class IterativeFinder:
|
||||
to_probe = list(self.active - self.contacted)
|
||||
to_probe.sort(key=lambda peer: self.distance(self.key))
|
||||
for peer in to_probe:
|
||||
if added >= constants.alpha:
|
||||
if added >= constants.ALPHA:
|
||||
break
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude:
|
||||
@@ -210,7 +216,7 @@ class IterativeFinder:
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.ITERATIVE_LOOKUP_DELAY):
|
||||
try:
|
||||
if self.running:
|
||||
await self._search_round()
|
||||
@@ -257,7 +263,7 @@ class IterativeFinder:
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
@@ -276,11 +282,11 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
not_yet_yielded = [
|
||||
peer for peer in from_iter
|
||||
if peer not in self.yielded_peers
|
||||
and peer.node_id != self.protocol.node_id
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
and peer.node_id != self.protocol.node_id
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
to_yield = not_yet_yielded[:min(constants.K, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
self.yielded_peers.update(to_yield)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
@@ -308,16 +314,47 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
# this tracks the index of the most recent page we requested from each peer
|
||||
self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int)
|
||||
# this tracks the set of blob peers returned by each peer
|
||||
self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key)
|
||||
return FindValueResponse(self.key, response)
|
||||
page = self.peer_pages[peer]
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page)
|
||||
parsed = FindValueResponse(self.key, response)
|
||||
if not parsed.found:
|
||||
return parsed
|
||||
already_known = len(self.discovered_peers[peer])
|
||||
decoded_peers = set()
|
||||
for compact_addr in parsed.found_compact_addresses:
|
||||
try:
|
||||
decoded_peers.add(self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned invalid peer for blob",
|
||||
peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
parsed.found_compact_addresses.clear()
|
||||
return parsed
|
||||
self.discovered_peers[peer].update(decoded_peers)
|
||||
log.debug("probed %s:%i page %i, %i known", peer.address, peer.udp_port, page,
|
||||
already_known + len(parsed.found_compact_addresses))
|
||||
if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses):
|
||||
log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port)
|
||||
parsed.found_compact_addresses.clear()
|
||||
elif len(parsed.found_compact_addresses) >= constants.K and self.peer_pages[peer] < parsed.pages:
|
||||
# the peer returned a full page and indicates it has more
|
||||
self.peer_pages[peer] += 1
|
||||
if peer in self.contacted:
|
||||
# the peer must be removed from self.contacted so that it will be probed for the next page
|
||||
self.contacted.remove(peer)
|
||||
return parsed
|
||||
|
||||
def check_result_ready(self, response: FindValueResponse):
|
||||
if response.found:
|
||||
@@ -23,7 +23,7 @@ if typing.TYPE_CHECKING:
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
old_protocol_errors = {
|
||||
OLD_PROTOCOL_ERRORS = {
|
||||
"findNode() takes exactly 2 arguments (5 given)": "0.19.1",
|
||||
"findValue() takes exactly 2 arguments (5 given)": "0.19.1"
|
||||
}
|
||||
@@ -48,13 +48,13 @@ class KademliaRPC:
|
||||
return b'pong'
|
||||
|
||||
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int) -> bytes:
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
if len(blob_hash) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not 0 < port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {port}")
|
||||
rpc_contact.update_tcp_port(port)
|
||||
if not self.verify_token(token, rpc_contact.compact_ip()):
|
||||
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
|
||||
if self.loop.time() - self.protocol.started_listening_time < constants.TOKEN_SECRET_REFRESH_INTERVAL:
|
||||
pass
|
||||
else:
|
||||
raise ValueError("Invalid token")
|
||||
@@ -64,19 +64,19 @@ class KademliaRPC:
|
||||
return b'OK'
|
||||
|
||||
def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
if len(key) != constants.hash_length:
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid contact node_id length: %i" % len(key))
|
||||
|
||||
contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id)
|
||||
contact_triples = []
|
||||
for contact in contacts[:constants.k * 2]:
|
||||
for contact in contacts[:constants.K * 2]:
|
||||
contact_triples.append((contact.node_id, contact.address, contact.udp_port))
|
||||
return contact_triples
|
||||
|
||||
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes, page: int = 0):
|
||||
page = page if page > 0 else 0
|
||||
|
||||
if len(key) != constants.hash_length:
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid blob_exchange hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
@@ -84,7 +84,7 @@ class KademliaRPC:
|
||||
}
|
||||
|
||||
if not page:
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)[:constants.k]
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)[:constants.K]
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
@@ -96,16 +96,16 @@ class KademliaRPC:
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
if len(peers) < constants.K and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if not peers:
|
||||
response[PAGE_KEY] = 0
|
||||
else:
|
||||
response[PAGE_KEY] = (len(peers) // (constants.k + 1)) + 1 # how many pages of peers we have for the blob
|
||||
if len(peers) > constants.k:
|
||||
response[PAGE_KEY] = (len(peers) // (constants.K + 1)) + 1 # how many pages of peers we have for the blob
|
||||
if len(peers) > constants.K:
|
||||
random.Random(self.protocol.node_id).shuffle(peers)
|
||||
if page * constants.k < len(peers):
|
||||
response[key] = peers[page * constants.k:page * constants.k + constants.k]
|
||||
if page * constants.K < len(peers):
|
||||
response[key] = peers[page * constants.K:page * constants.K + constants.K]
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
@@ -154,7 +154,7 @@ class RemoteKademliaRPC:
|
||||
:param blob_hash: blob hash as bytes
|
||||
:return: b'OK'
|
||||
"""
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
if len(blob_hash) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not self.protocol.peer_port or not 0 < self.protocol.peer_port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {self.protocol.peer_port}")
|
||||
@@ -171,7 +171,7 @@ class RemoteKademliaRPC:
|
||||
"""
|
||||
:return: [(node_id, address, udp_port), ...]
|
||||
"""
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
if len(key) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of find node key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_node(self.protocol.node_id, key)
|
||||
@@ -186,7 +186,7 @@ class RemoteKademliaRPC:
|
||||
<key bytes>: [<blob_peer_compact_address, ...]
|
||||
}
|
||||
"""
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
if len(key) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of find value key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key, page=page)
|
||||
@@ -203,7 +203,7 @@ class PingQueue:
|
||||
self._process_task: asyncio.Task = None
|
||||
self._running = False
|
||||
self._running_pings: typing.Set[asyncio.Task] = set()
|
||||
self._default_delay = constants.maybe_ping_delay
|
||||
self._default_delay = constants.MAYBE_PING_DELAY
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
@@ -224,7 +224,7 @@ class PingQueue:
|
||||
self._protocol.add_peer(peer)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
|
||||
task = self._loop.create_task(ping_task())
|
||||
@@ -260,8 +260,8 @@ class PingQueue:
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
@@ -271,7 +271,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.is_seed_node = False
|
||||
self.partial_messages: typing.Dict[bytes, typing.Dict[bytes, bytes]] = {}
|
||||
self.sent_messages: typing.Dict[bytes, typing.Tuple['KademliaPeer', asyncio.Future, RequestDatagram]] = {}
|
||||
self.protocol_version = constants.protocol_version
|
||||
self.protocol_version = constants.PROTOCOL_VERSION
|
||||
self.started_listening_time = 0
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
@@ -326,9 +326,9 @@ class KademliaProtocol(DatagramProtocol):
|
||||
if not peer.node_id:
|
||||
log.warning("Tried adding a peer with no node id!")
|
||||
return False
|
||||
for p in self.routing_table.get_peers():
|
||||
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(p)
|
||||
for my_peer in self.routing_table.get_peers():
|
||||
if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(my_peer)
|
||||
self.routing_table.join_buckets()
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
@@ -367,10 +367,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
|
||||
not_recently_replied = []
|
||||
for p in not_good_contacts:
|
||||
last_replied = self.peer_manager.get_last_replied(p.address, p.udp_port)
|
||||
for my_peer in not_good_contacts:
|
||||
last_replied = self.peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
|
||||
if not last_replied or last_replied + 60 < self.loop.time():
|
||||
not_recently_replied.append(p)
|
||||
not_recently_replied.append(my_peer)
|
||||
if not_recently_replied:
|
||||
to_replace = not_recently_replied[0]
|
||||
else:
|
||||
@@ -421,20 +421,20 @@ class KademliaProtocol(DatagramProtocol):
|
||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||
if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]:
|
||||
# args don't need reformatting
|
||||
args, kw = tuple(message.args[:-1]), message.args[-1]
|
||||
args, kwargs = tuple(message.args[:-1]), message.args[-1]
|
||||
else:
|
||||
args, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
|
||||
args, kwargs = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
|
||||
sender_contact.address, sender_contact.udp_port)
|
||||
|
||||
if method == b'ping':
|
||||
result = self.node_rpc.ping()
|
||||
elif method == b'store':
|
||||
blob_hash, token, port, original_publisher_id, age = args[:5]
|
||||
blob_hash, token, port, original_publisher_id, age = args[:5] # pylint: disable=unused-variable
|
||||
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
|
||||
else:
|
||||
key = args[0]
|
||||
page = kw.get(PAGE_KEY, 0)
|
||||
page = kwargs.get(PAGE_KEY, 0)
|
||||
if method == b'findNode':
|
||||
result = self.node_rpc.find_node(sender_contact, key)
|
||||
else:
|
||||
@@ -484,25 +484,25 @@ class KademliaProtocol(DatagramProtocol):
|
||||
def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
|
||||
# Find the message that triggered this response
|
||||
if response_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages[response_datagram.rpc_id]
|
||||
peer, future, _ = self.sent_messages[response_datagram.rpc_id]
|
||||
if peer.address != address[0]:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}, expected {peer.address}")
|
||||
future.set_exception(
|
||||
RemoteException(f"response from {address[0]}, expected {peer.address}")
|
||||
)
|
||||
return
|
||||
|
||||
# We got a result from the RPC
|
||||
if peer.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("node has our node id"))
|
||||
future.set_exception(RemoteException("node has our node id"))
|
||||
return
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
future.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
peer = make_kademlia_peer(response_datagram.node_id, address[0], address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
if not future.cancelled():
|
||||
future.set_result(response_datagram)
|
||||
self.add_peer(peer)
|
||||
else:
|
||||
log.warning("%s:%i replied, but after we cancelled the request attempt",
|
||||
@@ -516,11 +516,13 @@ class KademliaProtocol(DatagramProtocol):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
|
||||
if error_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
peer, future, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
future.set_exception(
|
||||
RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}"
|
||||
)
|
||||
)
|
||||
return
|
||||
error_msg = f"" \
|
||||
@@ -529,23 +531,27 @@ class KademliaProtocol(DatagramProtocol):
|
||||
f"Raised: {str(remote_exception)}"
|
||||
if 'Invalid token' in error_msg:
|
||||
log.debug(error_msg)
|
||||
elif error_datagram.response not in old_protocol_errors:
|
||||
elif error_datagram.response not in OLD_PROTOCOL_ERRORS:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.debug("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
|
||||
df.set_exception(remote_exception)
|
||||
log.debug(
|
||||
"known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, OLD_PROTOCOL_ERRORS[error_datagram.response]
|
||||
)
|
||||
future.set_exception(remote_exception)
|
||||
return
|
||||
else:
|
||||
if error_datagram.response not in old_protocol_errors:
|
||||
if error_datagram.response not in OLD_PROTOCOL_ERRORS:
|
||||
msg = f"Received error from {address[0]}:{address[1]}, but it isn't in response to a " \
|
||||
f"pending request: {str(remote_exception)}"
|
||||
log.warning(msg)
|
||||
else:
|
||||
log.debug("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
address[0], address[1], old_protocol_errors[error_datagram.response])
|
||||
log.debug(
|
||||
"known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
|
||||
)
|
||||
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
@@ -589,12 +595,12 @@ class KademliaProtocol(DatagramProtocol):
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
if len(data) > constants.msg_size_limit:
|
||||
if len(data) > constants.MSG_SIZE_LIMIT:
|
||||
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
|
||||
constants.msg_size_limit, len(data))
|
||||
constants.MSG_SIZE_LIMIT, len(data))
|
||||
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
|
||||
raise ValueError(
|
||||
f"cannot send datagram larger than {constants.msg_size_limit} bytes (packet is {len(data)} bytes)"
|
||||
f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
|
||||
)
|
||||
if isinstance(message, (RequestDatagram, ResponseDatagram)):
|
||||
assert message.node_id == self.node_id, message
|
||||
@@ -637,16 +643,16 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return constants.digest(self.token_secret + compact_ip)
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = constants.hash_class()
|
||||
h = constants.HASH_CLASS()
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = constants.hash_class()
|
||||
h = constants.HASH_CLASS()
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer',
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer', # pylint: disable=too-many-return-statements
|
||||
retry: bool = True) -> typing.Tuple[bytes, bool]:
|
||||
async def __store():
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
@@ -661,14 +667,14 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, False
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
log.error("Unexpected response: %s", err)
|
||||
return peer.node_id, False
|
||||
except RemoteException as err:
|
||||
if 'findValue() takes exactly 2 arguments (5 given)' in str(err):
|
||||
log.debug("peer %s:%i is running an incompatible version of lbrynet", peer.address, peer.udp_port)
|
||||
return peer.node_id, False
|
||||
if 'Invalid token' not in str(err):
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
log.warning("Unexpected error while storing blob_hash: %s", err)
|
||||
return peer.node_id, False
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
if not retry:
|
||||
@@ -51,12 +51,12 @@ class KBucket:
|
||||
return True
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
p = self.peers[i]
|
||||
if p.node_id == peer.node_id:
|
||||
self.peers.remove(p)
|
||||
local_peer = self.peers[i]
|
||||
if local_peer.node_id == peer.node_id:
|
||||
self.peers.remove(local_peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
if len(self.peers) < constants.k:
|
||||
if len(self.peers) < constants.K:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
@@ -101,8 +101,8 @@ class KBucket:
|
||||
current_len = len(peers)
|
||||
|
||||
# If count greater than k - return only k contacts
|
||||
if count > constants.k:
|
||||
count = constants.k
|
||||
if count > constants.K:
|
||||
count = constants.K
|
||||
|
||||
if not current_len:
|
||||
return peers
|
||||
@@ -164,14 +164,14 @@ class TreeRoutingTable:
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self._split_buckets_under_index = split_buckets_under_index
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id
|
||||
)
|
||||
]
|
||||
|
||||
@@ -185,7 +185,7 @@ class TreeRoutingTable:
|
||||
contacts = self.get_peers()
|
||||
distance = Distance(self._parent_node_id)
|
||||
contacts.sort(key=lambda c: distance(c.node_id))
|
||||
kth_contact = contacts[-1] if len(contacts) < constants.k else contacts[constants.k - 1]
|
||||
kth_contact = contacts[-1] if len(contacts) < constants.K else contacts[constants.K - 1]
|
||||
return distance(to_add) < distance(kth_contact.node_id)
|
||||
|
||||
def find_close_peers(self, key: bytes, count: typing.Optional[int] = None,
|
||||
@@ -193,7 +193,7 @@ class TreeRoutingTable:
|
||||
exclude = [self._parent_node_id]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
count = count or constants.k
|
||||
count = count or constants.K
|
||||
distance = Distance(key)
|
||||
contacts = self.get_peers()
|
||||
contacts = [c for c in contacts if c.node_id not in exclude]
|
||||
@@ -214,7 +214,7 @@ class TreeRoutingTable:
|
||||
refresh_ids = []
|
||||
now = int(self._loop.time())
|
||||
for bucket in self.buckets[start_index:]:
|
||||
if force or now - bucket.last_accessed >= constants.refresh_interval:
|
||||
if force or now - bucket.last_accessed >= constants.REFRESH_INTERVAL:
|
||||
to_search = self.midpoint_id_in_bucket_range(bucket_index)
|
||||
refresh_ids.append(to_search)
|
||||
bucket_index += 1
|
||||
@@ -248,13 +248,13 @@ class TreeRoutingTable:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return Distance(
|
||||
self._parent_node_id
|
||||
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
|
||||
)(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return Distance(self._parent_node_id)(
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
).to_bytes(constants.hash_length, 'big')
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
@@ -283,7 +283,7 @@ class TreeRoutingTable:
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
return
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
|
||||
if not to_pop:
|
||||
return
|
||||
log.info("join buckets %i", len(to_pop))
|
||||
@@ -314,6 +314,6 @@ class TreeRoutingTable:
|
||||
def buckets_with_contacts(self) -> int:
|
||||
count = 0
|
||||
for bucket in self.buckets:
|
||||
if len(bucket):
|
||||
if len(bucket) > 0:
|
||||
count += 1
|
||||
return count
|
||||
@@ -52,8 +52,7 @@ def _bdecode(data: bytes, start_index: int = 0) -> typing.Tuple[typing.Union[int
|
||||
raise DecodeError(err)
|
||||
start_index = split_pos + 1
|
||||
end_pos = start_index + length
|
||||
b = data[start_index:end_pos]
|
||||
return b, end_pos
|
||||
return data[start_index:end_pos], end_pos
|
||||
|
||||
|
||||
def bencode(data: typing.Dict) -> bytes:
|
||||
@@ -34,9 +34,9 @@ class KademliaDatagramBase:
|
||||
self.packet_type = packet_type
|
||||
if self.expected_packet_type != packet_type:
|
||||
raise ValueError(f"invalid packet type: {packet_type}, expected {self.expected_packet_type}")
|
||||
if len(rpc_id) != constants.rpc_id_length:
|
||||
if len(rpc_id) != constants.RPC_ID_LENGTH:
|
||||
raise ValueError(f"invalid rpc node_id: {len(rpc_id)} bytes (expected 20)")
|
||||
if not len(node_id) == constants.hash_length:
|
||||
if not len(node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError(f"invalid node node_id: {len(node_id)} bytes (expected 48)")
|
||||
self.rpc_id = rpc_id
|
||||
self.node_id = node_id
|
||||
@@ -46,9 +46,9 @@ class KademliaDatagramBase:
|
||||
i: getattr(self, k) for i, k in enumerate(self.required_fields)
|
||||
}
|
||||
for i, k in enumerate(OPTIONAL_FIELDS):
|
||||
v = getattr(self, k, None)
|
||||
if v is not None:
|
||||
datagram[i + OPTIONAL_ARG_OFFSET] = v
|
||||
value = getattr(self, k, None)
|
||||
if value is not None:
|
||||
datagram[i + OPTIONAL_ARG_OFFSET] = value
|
||||
return bencode(datagram)
|
||||
|
||||
|
||||
@@ -77,18 +77,18 @@ class RequestDatagram(KademliaDatagramBase):
|
||||
|
||||
@classmethod
|
||||
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
|
||||
|
||||
@classmethod
|
||||
def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH]
|
||||
if len(blob_hash) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid blob hash length: {len(blob_hash)}")
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f"invalid port: {port}")
|
||||
if len(token) != constants.hash_bits // 8:
|
||||
if len(token) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid token length: {len(token)}")
|
||||
store_args = [blob_hash, token, port, from_node_id, 0]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args)
|
||||
@@ -96,16 +96,16 @@ class RequestDatagram(KademliaDatagramBase):
|
||||
@classmethod
|
||||
def make_find_node(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH]
|
||||
if len(key) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key])
|
||||
|
||||
@classmethod
|
||||
def make_find_value(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None, page: int = 0) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH]
|
||||
if len(key) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
if page < 0:
|
||||
raise ValueError(f"cannot request a negative page ({page})")
|
||||
@@ -162,7 +162,7 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
for i, k in enumerate(datagram_class.required_fields)
|
||||
if i in primitive # pylint: disable=unsupported-membership-test
|
||||
}
|
||||
for i, k in enumerate(OPTIONAL_FIELDS):
|
||||
for i, _ in enumerate(OPTIONAL_FIELDS):
|
||||
if i + OPTIONAL_ARG_OFFSET in primitive:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
|
||||
return datagram_class(**decoded)
|
||||
@@ -179,7 +179,7 @@ def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
compact_ip = make_compact_ip(address)
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
@@ -190,6 +190,6 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
||||
node_id = compact_address[6:]
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return node_id, address, port
|
||||
@@ -0,0 +1,5 @@
|
||||
generate:
|
||||
python generate.py generate > __init__.py
|
||||
|
||||
analyze:
|
||||
python generate.py analyze
|
||||
@@ -0,0 +1,83 @@
|
||||
# Exceptions
|
||||
|
||||
Exceptions in LBRY are defined and generated from the Markdown table at the end of this README.
|
||||
|
||||
## Guidelines
|
||||
|
||||
When possible, use [built-in Python exceptions](https://docs.python.org/3/library/exceptions.html) or `aiohttp` [general client](https://docs.aiohttp.org/en/latest/client_reference.html#client-exceptions) / [HTTP](https://docs.aiohttp.org/en/latest/web_exceptions.html) exceptions, unless:
|
||||
1. You want to provide a better error message (extend the closest built-in/`aiohttp` exception in this case).
|
||||
2. You need to represent a new situation.
|
||||
|
||||
When defining your own exceptions, consider:
|
||||
1. Extending a built-in Python or `aiohttp` exception.
|
||||
2. Using contextual variables in the error message.
|
||||
|
||||
## Table Column Definitions
|
||||
|
||||
Column | Meaning
|
||||
---|---
|
||||
Code | Codes are used only to define the hierarchy of exceptions and do not end up in the generated output, it is okay to re-number things as necessary at anytime to achieve the desired hierarchy.
|
||||
Name | Becomes the class name of the exception with "Error" appended to the end. Changing names of existing exceptions makes the API backwards incompatible. When extending other exceptions you must specify the full class name, manually adding "Error" as necessary (if extending another SDK exception).
|
||||
Message | User friendly error message explaining the exceptional event. Supports Python formatted strings: any variables used in the string will be generated as arguments in the `__init__` method. Use `--` to provide a doc string after the error message to be added to the class definition.
|
||||
|
||||
## Exceptions Table
|
||||
|
||||
Code | Name | Message
|
||||
---:|---|---
|
||||
**1xx** | UserInput | User input errors.
|
||||
**10x** | Command | Errors preparing to execute commands.
|
||||
101 | CommandDoesNotExist | Command '{command}' does not exist.
|
||||
102 | CommandDeprecated | Command '{command}' is deprecated.
|
||||
103 | CommandInvalidArgument | Invalid argument '{argument}' to command '{command}'.
|
||||
104 | CommandTemporarilyUnavailable | Command '{command}' is temporarily unavailable. -- Such as waiting for required components to start.
|
||||
105 | CommandPermanentlyUnavailable | Command '{command}' is permanently unavailable. -- such as when required component was intentionally configured not to start.
|
||||
**11x** | InputValue(ValueError) | Invalid argument value provided to command.
|
||||
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
|
||||
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
|
||||
**2xx** | Configuration | Configuration errors.
|
||||
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
|
||||
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
|
||||
203 | ConfigParse | Failed to parse the configuration file '{path}'. -- Includes the syntax error / line number to help user fix it.
|
||||
204 | ConfigMissing | Configuration file '{path}' is missing setting that has no default / fallback.
|
||||
205 | ConfigInvalid | Configuration file '{path}' has setting with invalid value.
|
||||
**3xx** | Network | **Networking**
|
||||
301 | NoInternet | No internet connection.
|
||||
302 | NoUPnPSupport | Router does not support UPnP.
|
||||
**4xx** | Wallet | **Wallet Errors**
|
||||
401 | TransactionRejected | Transaction rejected, unknown reason.
|
||||
402 | TransactionFeeTooLow | Fee too low.
|
||||
403 | TransactionInvalidSignature | Invalid signature.
|
||||
404 | InsufficientFunds | Not enough funds to cover this transaction. -- determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX being created and sent but then rejected by lbrycrd for unspendable utxos.
|
||||
405 | ChannelKeyNotFound | Channel signing key not found.
|
||||
406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
502 | BlobTooBig | Blob is too big.
|
||||
503 | BlobEmpty | Blob is empty.
|
||||
510 | BlobFailedDecryption | Failed to decrypt blob.
|
||||
511 | CorruptBlob | Blobs is corrupted.
|
||||
520 | BlobFailedEncryption | Failed to encrypt blob.
|
||||
531 | DownloadCancelled | Download was canceled.
|
||||
532 | DownloadSDTimeout | Failed to download sd blob {download} within timeout.
|
||||
533 | DownloadDataTimeout | Failed to download data blobs for sd hash {download} within timeout.
|
||||
534 | InvalidStreamDescriptor | {message}
|
||||
535 | InvalidData | {message}
|
||||
536 | InvalidBlobHash | {message}
|
||||
**6xx** | Component | **Components**
|
||||
601 | ComponentStartConditionNotMet | Unresolved dependencies for: {components}
|
||||
602 | ComponentsNotStarted | {message}
|
||||
**7xx** | CurrencyExchange | **Currency Exchange**
|
||||
701 | InvalidExchangeRateResponse | Failed to get exchange rate from {source}: {reason}
|
||||
702 | CurrencyConversion | {message}
|
||||
703 | InvalidCurrency | Invalid currency: {currency} is not a supported currency.
|
||||
@@ -0,0 +1,400 @@
|
||||
from .base import BaseError, claim_id
|
||||
|
||||
|
||||
class UserInputError(BaseError):
|
||||
"""
|
||||
User input errors.
|
||||
"""
|
||||
|
||||
|
||||
class CommandError(UserInputError):
|
||||
"""
|
||||
Errors preparing to execute commands.
|
||||
"""
|
||||
|
||||
|
||||
class CommandDoesNotExistError(CommandError):
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' does not exist.")
|
||||
|
||||
|
||||
class CommandDeprecatedError(CommandError):
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is deprecated.")
|
||||
|
||||
|
||||
class CommandInvalidArgumentError(CommandError):
|
||||
|
||||
def __init__(self, argument, command):
|
||||
self.argument = argument
|
||||
self.command = command
|
||||
super().__init__(f"Invalid argument '{argument}' to command '{command}'.")
|
||||
|
||||
|
||||
class CommandTemporarilyUnavailableError(CommandError):
|
||||
"""
|
||||
Such as waiting for required components to start.
|
||||
"""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is temporarily unavailable.")
|
||||
|
||||
|
||||
class CommandPermanentlyUnavailableError(CommandError):
|
||||
"""
|
||||
such as when required component was intentionally configured not to start.
|
||||
"""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is permanently unavailable.")
|
||||
|
||||
|
||||
class InputValueError(UserInputError, ValueError):
|
||||
"""
|
||||
Invalid argument value provided to command.
|
||||
"""
|
||||
|
||||
|
||||
class GenericInputValueError(InputValueError):
|
||||
|
||||
def __init__(self, value, argument):
|
||||
self.value = value
|
||||
self.argument = argument
|
||||
super().__init__(f"The value '{value}' for argument '{argument}' is not valid.")
|
||||
|
||||
|
||||
class InputValueIsNoneError(InputValueError):
|
||||
|
||||
def __init__(self, argument):
|
||||
self.argument = argument
|
||||
super().__init__(f"None or null is not valid value for argument '{argument}'.")
|
||||
|
||||
|
||||
class ConfigurationError(BaseError):
|
||||
"""
|
||||
Configuration errors.
|
||||
"""
|
||||
|
||||
|
||||
class ConfigWriteError(ConfigurationError):
|
||||
"""
|
||||
When writing the default config fails on startup, such as due to permission issues.
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Cannot write configuration file '{path}'.")
|
||||
|
||||
|
||||
class ConfigReadError(ConfigurationError):
|
||||
"""
|
||||
Can't open the config file user provided via command line args.
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Cannot find provided configuration file '{path}'.")
|
||||
|
||||
|
||||
class ConfigParseError(ConfigurationError):
|
||||
"""
|
||||
Includes the syntax error / line number to help user fix it.
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Failed to parse the configuration file '{path}'.")
|
||||
|
||||
|
||||
class ConfigMissingError(ConfigurationError):
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
|
||||
|
||||
|
||||
class ConfigInvalidError(ConfigurationError):
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Configuration file '{path}' has setting with invalid value.")
|
||||
|
||||
|
||||
class NetworkError(BaseError):
|
||||
"""
|
||||
**Networking**
|
||||
"""
|
||||
|
||||
|
||||
class NoInternetError(NetworkError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("No internet connection.")
|
||||
|
||||
|
||||
class NoUPnPSupportError(NetworkError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Router does not support UPnP.")
|
||||
|
||||
|
||||
class WalletError(BaseError):
|
||||
"""
|
||||
**Wallet Errors**
|
||||
"""
|
||||
|
||||
|
||||
class TransactionRejectedError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Transaction rejected, unknown reason.")
|
||||
|
||||
|
||||
class TransactionFeeTooLowError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Fee too low.")
|
||||
|
||||
|
||||
class TransactionInvalidSignatureError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Invalid signature.")
|
||||
|
||||
|
||||
class InsufficientFundsError(WalletError):
|
||||
"""
|
||||
determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX
|
||||
being created and sent but then rejected by lbrycrd for unspendable utxos.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Not enough funds to cover this transaction.")
|
||||
|
||||
|
||||
class ChannelKeyNotFoundError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Channel signing key not found.")
|
||||
|
||||
|
||||
class ChannelKeyInvalidError(WalletError):
|
||||
"""
|
||||
For example, channel was updated but you don't have the updated key.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Channel signing key is out of date.")
|
||||
|
||||
|
||||
class DataDownloadError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Failed to download blob. *generic*")
|
||||
|
||||
|
||||
class ResolveError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
super().__init__(f"Failed to resolve '{url}'.")
|
||||
|
||||
|
||||
class ResolveTimeoutError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
super().__init__(f"Failed to resolve '{url}' within the timeout.")
|
||||
|
||||
|
||||
class ResolveCensoredError(WalletError):
|
||||
|
||||
def __init__(self, url, censor_hash):
|
||||
self.url = url
|
||||
self.censor_hash = censor_hash
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidPasswordError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Password is invalid.")
|
||||
|
||||
|
||||
class IncompatibleWalletServerError(WalletError):
|
||||
|
||||
def __init__(self, server, port):
|
||||
self.server = server
|
||||
self.port = port
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
self.address = address
|
||||
super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentWalletLockedError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
||||
|
||||
def __init__(self, daily_fee, max_fee):
|
||||
self.daily_fee = daily_fee
|
||||
self.max_fee = max_fee
|
||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
"""
|
||||
|
||||
|
||||
class BlobNotFoundError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Blob not found.")
|
||||
|
||||
|
||||
class BlobPermissionDeniedError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Permission denied to read blob.")
|
||||
|
||||
|
||||
class BlobTooBigError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Blob is too big.")
|
||||
|
||||
|
||||
class BlobEmptyError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Blob is empty.")
|
||||
|
||||
|
||||
class BlobFailedDecryptionError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Failed to decrypt blob.")
|
||||
|
||||
|
||||
class CorruptBlobError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Blobs is corrupted.")
|
||||
|
||||
|
||||
class BlobFailedEncryptionError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Failed to encrypt blob.")
|
||||
|
||||
|
||||
class DownloadCancelledError(BlobError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Download was canceled.")
|
||||
|
||||
|
||||
class DownloadSDTimeoutError(BlobError):
|
||||
|
||||
def __init__(self, download):
|
||||
self.download = download
|
||||
super().__init__(f"Failed to download sd blob {download} within timeout.")
|
||||
|
||||
|
||||
class DownloadDataTimeoutError(BlobError):
|
||||
|
||||
def __init__(self, download):
|
||||
self.download = download
|
||||
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidDataError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidBlobHashError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class ComponentError(BaseError):
|
||||
"""
|
||||
**Components**
|
||||
"""
|
||||
|
||||
|
||||
class ComponentStartConditionNotMetError(ComponentError):
|
||||
|
||||
def __init__(self, components):
|
||||
self.components = components
|
||||
super().__init__(f"Unresolved dependencies for: {components}")
|
||||
|
||||
|
||||
class ComponentsNotStartedError(ComponentError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class CurrencyExchangeError(BaseError):
|
||||
"""
|
||||
**Currency Exchange**
|
||||
"""
|
||||
|
||||
|
||||
class InvalidExchangeRateResponseError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, source, reason):
|
||||
self.source = source
|
||||
self.reason = reason
|
||||
super().__init__(f"Failed to get exchange rate from {source}: {reason}")
|
||||
|
||||
|
||||
class CurrencyConversionError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidCurrencyError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, currency):
|
||||
self.currency = currency
|
||||
super().__init__(f"Invalid currency: {currency} is not a supported currency.")
|
||||
@@ -0,0 +1,9 @@
|
||||
from binascii import hexlify
|
||||
|
||||
|
||||
def claim_id(claim_hash):
|
||||
return hexlify(claim_hash[::-1]).decode()
|
||||
|
||||
|
||||
class BaseError(Exception):
|
||||
pass
|
||||
@@ -0,0 +1,167 @@
|
||||
import re
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from textwrap import fill, indent
|
||||
|
||||
|
||||
INDENT = ' ' * 4
|
||||
|
||||
CLASS = """
|
||||
|
||||
class {name}({parents}):{doc}
|
||||
"""
|
||||
|
||||
INIT = """
|
||||
def __init__({args}):{fields}
|
||||
super().__init__({format}"{message}")
|
||||
"""
|
||||
|
||||
FUNCTIONS = ['claim_id']
|
||||
|
||||
|
||||
class ErrorClass:
|
||||
|
||||
def __init__(self, hierarchy, name, message):
|
||||
self.hierarchy = hierarchy.replace('**', '')
|
||||
self.other_parents = []
|
||||
if '(' in name:
|
||||
assert ')' in name, f"Missing closing parenthesis in '{name}'."
|
||||
self.other_parents = name[name.find('(')+1:name.find(')')].split(',')
|
||||
name = name[:name.find('(')]
|
||||
self.name = name
|
||||
self.class_name = name+'Error'
|
||||
self.message = message
|
||||
self.comment = ""
|
||||
if '--' in message:
|
||||
self.message, self.comment = message.split('--')
|
||||
self.message = self.message.strip()
|
||||
self.comment = self.comment.strip()
|
||||
|
||||
@property
|
||||
def is_leaf(self):
|
||||
return 'x' not in self.hierarchy
|
||||
|
||||
@property
|
||||
def code(self):
|
||||
return self.hierarchy.replace('x', '')
|
||||
|
||||
@property
|
||||
def parent_codes(self):
|
||||
return self.hierarchy[0:2], self.hierarchy[0]
|
||||
|
||||
def get_arguments(self):
|
||||
args = ['self']
|
||||
for arg in re.findall('{([a-z0-1_()]+)}', self.message):
|
||||
for func in FUNCTIONS:
|
||||
if arg.startswith(f'{func}('):
|
||||
arg = arg[len(f'{func}('):-1]
|
||||
break
|
||||
args.append(arg)
|
||||
return args
|
||||
|
||||
@staticmethod
|
||||
def get_fields(args):
|
||||
if len(args) > 1:
|
||||
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
def get_doc_string(doc):
|
||||
if doc:
|
||||
return f'\n{INDENT}"""\n{indent(fill(doc, 100), INDENT)}\n{INDENT}"""'
|
||||
return ""
|
||||
|
||||
def render(self, out, parent):
|
||||
if not parent:
|
||||
parents = ['BaseError']
|
||||
else:
|
||||
parents = [parent.class_name]
|
||||
parents += self.other_parents
|
||||
args = self.get_arguments()
|
||||
if self.is_leaf:
|
||||
out.write((CLASS + INIT).format(
|
||||
name=self.class_name, parents=', '.join(parents),
|
||||
args=', '.join(args), fields=self.get_fields(args),
|
||||
message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else ''
|
||||
))
|
||||
else:
|
||||
out.write(CLASS.format(
|
||||
name=self.class_name, parents=', '.join(parents),
|
||||
doc=self.get_doc_string(self.comment or self.message)
|
||||
))
|
||||
|
||||
|
||||
def get_errors():
|
||||
with open('README.md', 'r') as readme:
|
||||
lines = iter(readme.readlines())
|
||||
for line in lines:
|
||||
if line.startswith('## Exceptions Table'):
|
||||
break
|
||||
for line in lines:
|
||||
if line.startswith('---:|'):
|
||||
break
|
||||
for line in lines:
|
||||
if not line:
|
||||
break
|
||||
yield ErrorClass(*[c.strip() for c in line.split('|')])
|
||||
|
||||
|
||||
def find_parent(stack, child):
|
||||
for parent_code in child.parent_codes:
|
||||
parent = stack.get(parent_code)
|
||||
if parent:
|
||||
return parent
|
||||
|
||||
|
||||
def generate(out):
|
||||
out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n")
|
||||
stack = {}
|
||||
for error in get_errors():
|
||||
error.render(out, find_parent(stack, error))
|
||||
if not error.is_leaf:
|
||||
assert error.code not in stack, f"Duplicate code: {error.code}"
|
||||
stack[error.code] = error
|
||||
|
||||
|
||||
def analyze():
|
||||
errors = {e.class_name: [] for e in get_errors() if e.is_leaf}
|
||||
here = Path(__file__).absolute().parents[0]
|
||||
module = here.parent
|
||||
for file_path in module.glob('**/*.py'):
|
||||
if here in file_path.parents:
|
||||
continue
|
||||
with open(file_path) as src_file:
|
||||
src = src_file.read()
|
||||
for error in errors.keys():
|
||||
found = src.count(error)
|
||||
if found > 0:
|
||||
errors[error].append((file_path, found))
|
||||
|
||||
print('Unused Errors:\n')
|
||||
for error, used in errors.items():
|
||||
if used:
|
||||
print(f' - {error}')
|
||||
for use in used:
|
||||
print(f' {use[0].relative_to(module.parent)} {use[1]}')
|
||||
print('')
|
||||
|
||||
print('')
|
||||
print('Unused Errors:')
|
||||
for error, used in errors.items():
|
||||
if not used:
|
||||
print(f' - {error}')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=['generate', 'analyze'])
|
||||
args = parser.parse_args()
|
||||
if args.action == "analyze":
|
||||
analyze()
|
||||
elif args.action == "generate":
|
||||
generate(sys.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import signal
|
||||
import pathlib
|
||||
import json
|
||||
@@ -14,8 +15,8 @@ from docopt import docopt
|
||||
|
||||
from lbry import __version__ as lbrynet_version
|
||||
from lbry.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
from lbry.conf import Config, CLIConfig
|
||||
from lbry.extras.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger('lbry')
|
||||
|
||||
@@ -167,16 +168,16 @@ def add_command_parser(parent, command):
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
main = ArgumentParser(
|
||||
root = ArgumentParser(
|
||||
'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False,
|
||||
)
|
||||
main.add_argument(
|
||||
root.add_argument(
|
||||
'-v', '--version', dest='cli_version', action="store_true",
|
||||
help='Show lbrynet CLI version and exit.'
|
||||
)
|
||||
main.set_defaults(group=None, command=None)
|
||||
CLIConfig.contribute_to_argparse(main)
|
||||
sub = main.add_subparsers(metavar='COMMAND')
|
||||
root.set_defaults(group=None, command=None)
|
||||
CLIConfig.contribute_to_argparse(root)
|
||||
sub = root.add_subparsers(metavar='COMMAND')
|
||||
start = sub.add_parser(
|
||||
'start',
|
||||
usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...',
|
||||
@@ -186,11 +187,19 @@ def get_argument_parser():
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--no-logging', dest='no_logging', action="store_true",
|
||||
help='Disable all logging of any kind.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
start.add_argument(
|
||||
'--initial-headers', dest='initial_headers',
|
||||
help='Specify path to initial blockchain headers, faster than downloading them on first run.'
|
||||
)
|
||||
Config.contribute_to_argparse(start)
|
||||
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
|
||||
|
||||
@@ -212,7 +221,7 @@ def get_argument_parser():
|
||||
else:
|
||||
add_command_parser(groups[command['group']], command)
|
||||
|
||||
return main
|
||||
return root
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
@@ -220,7 +229,7 @@ def ensure_directory_exists(path: str):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
LOG_MODULES = ('lbry', 'torba', 'aioupnp')
|
||||
LOG_MODULES = 'lbry', 'aioupnp'
|
||||
|
||||
|
||||
def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config):
|
||||
@@ -236,7 +245,6 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
logger.getChild(module_name).addHandler(handler)
|
||||
|
||||
logger.getChild('lbry').setLevel(logging.INFO)
|
||||
logger.getChild('torba').setLevel(logging.INFO)
|
||||
logger.getChild('aioupnp').setLevel(logging.WARNING)
|
||||
logger.getChild('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
@@ -247,17 +255,17 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
else:
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
loggly_handler = get_loggly_handler(conf)
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
setup_logging(logging.getLogger(), args, conf)
|
||||
if not args.no_logging:
|
||||
setup_logging(logging.getLogger(), args, conf)
|
||||
daemon = Daemon(conf)
|
||||
|
||||
def __exit():
|
||||
@@ -297,6 +305,16 @@ def main(argv=None):
|
||||
if args.help:
|
||||
args.start_parser.print_help()
|
||||
else:
|
||||
if args.initial_headers:
|
||||
ledger_path = os.path.join(conf.wallet_dir, 'lbc_mainnet')
|
||||
ensure_directory_exists(ledger_path)
|
||||
current_size = 0
|
||||
headers_path = os.path.join(ledger_path, 'headers')
|
||||
if os.path.exists(headers_path):
|
||||
current_size = os.stat(headers_path).st_size
|
||||
if os.stat(args.initial_headers).st_size > current_size:
|
||||
log.info('Copying header from %s to %s', args.initial_headers, headers_path)
|
||||
shutil.copy(args.initial_headers, headers_path)
|
||||
run_daemon(args, conf)
|
||||
elif args.command is not None:
|
||||
doc = args.doc
|
||||
@@ -1,8 +1,8 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
import aiohttp
|
||||
import typing
|
||||
import aiohttp
|
||||
from lbry import utils
|
||||
from lbry.conf import Config
|
||||
from lbry.extras import system_info
|
||||
@@ -66,7 +66,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"use_fixed_peers": len(conf.fixed_peers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
@@ -110,7 +110,6 @@ class AnalyticsManager:
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
@@ -118,20 +117,24 @@ class AnalyticsManager:
|
||||
self.task: typing.Optional[asyncio.Task] = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self.conf.share_usage_data
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
async def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
if self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
if self.enabled:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
@@ -154,7 +157,7 @@ class AnalyticsManager:
|
||||
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
if self.enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post(event)
|
||||
|
||||
@@ -3,16 +3,16 @@ import time
|
||||
import hashlib
|
||||
import binascii
|
||||
|
||||
from lbry import utils
|
||||
import ecdsa
|
||||
from torba.client.hash import sha256
|
||||
from lbry import utils
|
||||
from lbry.crypto.hash import sha256
|
||||
from lbry.wallet.transaction import Output
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_encoded_signature(signature):
|
||||
signature = signature.encode() if type(signature) is str else signature
|
||||
signature = signature.encode() if isinstance(signature, str) else signature
|
||||
r = int(signature[:int(len(signature) / 2)], 16)
|
||||
s = int(signature[int(len(signature) / 2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
|
||||
@@ -23,7 +23,7 @@ def cid2hash(claim_id: str) -> bytes:
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
|
||||
if type(channel) is Output:
|
||||
if isinstance(channel, Output):
|
||||
try:
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [
|
||||
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -9,7 +9,7 @@ log = logging.getLogger(__name__)
|
||||
class ComponentType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
if name != "Component":
|
||||
if name != "Component" and newattrs['__module__'] != 'lbry.testcase':
|
||||
ComponentManager.default_component_classes[klass.component_name] = klass
|
||||
return klass
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from lbry.conf import Config
|
||||
from lbry.error import ComponentStartConditionNotMet
|
||||
from lbry.error import ComponentStartConditionNotMetError
|
||||
from lbry.dht.peer import PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -106,7 +106,7 @@ class ComponentManager:
|
||||
staged.update(to_stage)
|
||||
steps.append(step)
|
||||
elif components:
|
||||
raise ComponentStartConditionNotMet("Unresolved dependencies for: %s" % components)
|
||||
raise ComponentStartConditionNotMetError(components)
|
||||
if reverse:
|
||||
steps.reverse()
|
||||
return steps
|
||||
@@ -158,11 +158,14 @@ class ComponentManager:
|
||||
for component in self.components
|
||||
}
|
||||
|
||||
def get_component(self, component_name):
|
||||
def get_actual_component(self, component_name):
|
||||
for component in self.components:
|
||||
if component.component_name == component_name:
|
||||
return component.component
|
||||
return component
|
||||
raise NameError(component_name)
|
||||
|
||||
def get_component(self, component_name):
|
||||
return self.get_actual_component(component_name).component
|
||||
|
||||
def has_component(self, component_name):
|
||||
return any(component for component in self.components if component_name == component.component_name)
|
||||
@@ -1,8 +1,7 @@
|
||||
import hashlib
|
||||
import math
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
import base58
|
||||
@@ -13,15 +12,22 @@ from aioupnp.fault import UPnPError
|
||||
|
||||
from lbry import utils
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import is_valid_public_ipv4
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.extras.daemon.Component import Component
|
||||
from lbry.file.file_manager import FileManager
|
||||
from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import LbryWalletManager
|
||||
from lbry.wallet.header import Headers
|
||||
from lbry.torrent.torrent_manager import TorrentManager
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
try:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
except ImportError:
|
||||
TorrentSession = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,14 +35,15 @@ log = logging.getLogger(__name__)
|
||||
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
HEADERS_COMPONENT = "blockchain_headers"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
LIBTORRENT_COMPONENT = "libtorrent_component"
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -52,7 +59,7 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 12
|
||||
return 14
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
@@ -97,135 +104,9 @@ class DatabaseComponent(Component):
|
||||
self.storage = None
|
||||
|
||||
|
||||
class HeadersComponent(Component):
|
||||
component_name = HEADERS_COMPONENT
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_file = os.path.join(self.headers_dir, 'headers')
|
||||
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
|
||||
self.headers = Headers(self.headers_file)
|
||||
self.is_downloading_headers = False
|
||||
self._headers_progress_percent = 0
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def _round_progress(self, local_height, remote_height):
|
||||
return min(max(math.ceil(float(local_height) / float(remote_height) * 100), 0), 100)
|
||||
|
||||
async def get_status(self) -> dict:
|
||||
progress = None
|
||||
if self.is_downloading_headers:
|
||||
progress = self._headers_progress_percent
|
||||
elif self.component_manager.has_component(WALLET_COMPONENT):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
if wallet_manager and wallet_manager.ledger.network.remote_height > 0:
|
||||
local_height = wallet_manager.ledger.headers.height
|
||||
remote_height = wallet_manager.ledger.network.remote_height
|
||||
progress = self._round_progress(local_height, remote_height)
|
||||
return {
|
||||
'downloading_headers': True,
|
||||
'download_progress': progress
|
||||
} if progress is not None and progress < 100 else {}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
if final_size_after_download % self.headers.header_size != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while not response.content.at_eof():
|
||||
local_header_size += fd.write(await response.content.readany())
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
local_header_size, final_size_after_download
|
||||
)
|
||||
|
||||
def local_header_file_size(self) -> int:
|
||||
if os.path.isfile(self.headers_file):
|
||||
return os.stat(self.headers_file).st_size
|
||||
return 0
|
||||
|
||||
async def get_downloadable_header_height(self) -> typing.Optional[int]:
|
||||
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return -1
|
||||
return response.content_length // self.headers.header_size
|
||||
|
||||
async def should_download_headers_from_s3(self) -> bool:
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return False
|
||||
s3_headers_depth = self.conf.s3_headers_depth
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
|
||||
local_height = self.local_header_file_size() // self.headers.header_size
|
||||
remote_height = await self.get_downloadable_header_height()
|
||||
if remote_height is not None:
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
if remote_height > (local_height + s3_headers_depth):
|
||||
return True
|
||||
return False
|
||||
|
||||
def verify_checkpoint(self):
|
||||
expected_hash, at_height = self.CHECKPOINT
|
||||
if self.local_header_file_size() // self.headers.header_size < at_height:
|
||||
return False
|
||||
hash = hashlib.sha256()
|
||||
chunk_size = self.headers.header_size * 1000
|
||||
with open(self.headers_file, 'rb') as header_file:
|
||||
data = header_file.read(chunk_size)
|
||||
while data and header_file.tell() <= at_height * self.headers.header_size:
|
||||
hash.update(data)
|
||||
data = header_file.read(chunk_size)
|
||||
return hash.hexdigest() == expected_hash
|
||||
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
|
||||
try:
|
||||
if await self.should_download_headers_from_s3():
|
||||
self.is_downloading_headers = True
|
||||
await self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
# fixme: workaround, this should happen before download but happens after because headers.connect fail
|
||||
if not self.verify_checkpoint():
|
||||
log.info("Checkpoint failed, verifying headers using slower method.")
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
await self.headers.close()
|
||||
else:
|
||||
log.info("Header checkpoint verified.")
|
||||
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
component_name = WALLET_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, HEADERS_COMPONENT]
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -236,27 +117,52 @@ class WalletComponent(Component):
|
||||
return self.wallet_manager
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.wallet_manager.ledger.network.remote_height:
|
||||
local_height = self.wallet_manager.ledger.headers.height
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
||||
sessions = session_pool.sessions
|
||||
connected = None
|
||||
if self.wallet_manager.ledger.network.client:
|
||||
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
|
||||
if addr_and_port:
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
'servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
'port': session.server[1],
|
||||
'latency': session.connection_latency,
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(sessions),
|
||||
'available_servers': len(list(session_pool.available_sessions))
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
local_height = self.wallet_manager.ledger.local_height_including_downloaded_height
|
||||
disk_height = len(self.wallet_manager.ledger.headers)
|
||||
remote_height = self.wallet_manager.ledger.network.remote_height
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'connected_servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
'port': session.server[1],
|
||||
'latency': round(session.connection_latency, 2),
|
||||
} for session in self.wallet_manager.ledger.network.session_pool.sessions
|
||||
if session and session.available
|
||||
],
|
||||
download_height, target_height = local_height - disk_height, remote_height - disk_height
|
||||
if target_height > 0:
|
||||
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
|
||||
else:
|
||||
progress = 100
|
||||
best_hash = await self.wallet_manager.get_best_blockhash()
|
||||
result.update({
|
||||
'headers_synchronization_progress': progress,
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
|
||||
log.info("Starting wallet")
|
||||
self.wallet_manager = await WalletManager.from_lbrynet_config(self.conf)
|
||||
await self.wallet_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
@@ -264,6 +170,34 @@ class WalletComponent(Component):
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
class WalletServerPaymentsComponent(Component):
|
||||
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
depends_on = [WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.usage_payment_service = WalletServerPayer(
|
||||
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
|
||||
)
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[WalletServerPayer]:
|
||||
return self.usage_payment_service
|
||||
|
||||
async def start(self):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
|
||||
|
||||
async def stop(self):
|
||||
await self.usage_payment_service.stop()
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'max_fee': self.usage_payment_service.max_fee,
|
||||
'running': self.usage_payment_service.running
|
||||
}
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
@@ -304,7 +238,7 @@ class BlobComponent(Component):
|
||||
|
||||
class DHTComponent(Component):
|
||||
component_name = DHT_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT]
|
||||
depends_on = [UPNP_COMPONENT, DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -338,6 +272,7 @@ class DHTComponent(Component):
|
||||
self.external_peer_port = upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip = await utils.get_external_ip()
|
||||
if not external_ip:
|
||||
@@ -352,11 +287,10 @@ class DHTComponent(Component):
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index,
|
||||
storage=storage
|
||||
)
|
||||
self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes)
|
||||
log.info("Started the dht")
|
||||
|
||||
async def stop(self):
|
||||
@@ -392,41 +326,77 @@ class HashAnnouncerComponent(Component):
|
||||
}
|
||||
|
||||
|
||||
class StreamManagerComponent(Component):
|
||||
component_name = STREAM_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
self.file_manager: typing.Optional[FileManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
return self.stream_manager
|
||||
def component(self) -> typing.Optional[FileManager]:
|
||||
return self.file_manager
|
||||
|
||||
async def get_status(self):
|
||||
if not self.stream_manager:
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
'managed_files': len(self.file_manager.get_filtered()),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT)\
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
self.file_manager = FileManager(
|
||||
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.file_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.stream_manager.stop()
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.torrent_session = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[TorrentSession]:
|
||||
return self.torrent_session
|
||||
|
||||
async def get_status(self):
|
||||
if not self.torrent_session:
|
||||
return
|
||||
return {
|
||||
'running': True, # TODO: what to return here?
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
if TorrentSession:
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
|
||||
async def stop(self):
|
||||
if self.torrent_session:
|
||||
await self.torrent_session.pause()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -443,9 +413,8 @@ class PeerProtocolServerComponent(Component):
|
||||
|
||||
async def start(self):
|
||||
log.info("start blob server")
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
wallet: WalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
@@ -487,7 +456,7 @@ class UPnPComponent(Component):
|
||||
self.upnp = await UPnP.discover(loop=self.component_manager.loop)
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
self.upnp = None
|
||||
@@ -501,9 +470,8 @@ class UPnPComponent(Component):
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError, NotImplementedError):
|
||||
pass
|
||||
|
||||
if external_ip == "0.0.0.0" or (external_ip and external_ip.startswith("192.")):
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
@@ -553,9 +521,9 @@ class UPnPComponent(Component):
|
||||
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
|
||||
except (asyncio.TimeoutError, UPnPError, NotImplementedError):
|
||||
del self.upnp_redirects['TCP']
|
||||
if ('TCP' in self.upnp_redirects
|
||||
and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and (
|
||||
'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
|
||||
if ('TCP' in self.upnp_redirects and
|
||||
PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and \
|
||||
('UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
|
||||
if self.upnp_redirects:
|
||||
log.debug("upnp redirects are still active")
|
||||
|
||||
@@ -581,8 +549,10 @@ class UPnPComponent(Component):
|
||||
if self.external_ip:
|
||||
log.info("detected external ip using lbry.com fallback")
|
||||
if self.component_manager.analytics_manager:
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
self.component_manager.loop.create_task(
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
)
|
||||
self._maintain_redirects_task = self.component_manager.loop.create_task(
|
||||
self._repeatedly_maintain_redirects(now=False)
|
||||
@@ -0,0 +1,216 @@
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from typing import Optional, Iterable, Type
|
||||
from aiohttp.client_exceptions import ContentTypeError
|
||||
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
||||
from lbry.utils import aiohttp_request
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ExchangeRate:
|
||||
def __init__(self, market, spot, ts):
|
||||
if not int(time.time()) - ts < 600:
|
||||
raise ValueError('The timestamp is too dated.')
|
||||
if not spot > 0:
|
||||
raise ValueError('Spot must be greater than 0.')
|
||||
self.currency_pair = (market[0:3], market[3:6])
|
||||
self.spot = spot
|
||||
self.ts = ts
|
||||
|
||||
def __repr__(self):
|
||||
return f"Currency pair:{self.currency_pair}, spot:{self.spot}, ts:{self.ts}"
|
||||
|
||||
def as_dict(self):
|
||||
return {'spot': self.spot, 'ts': self.ts}
|
||||
|
||||
|
||||
class MarketFeed:
|
||||
name: str = ""
|
||||
market: str = ""
|
||||
url: str = ""
|
||||
params = {}
|
||||
fee = 0
|
||||
|
||||
update_interval = 300
|
||||
request_timeout = 50
|
||||
|
||||
def __init__(self):
|
||||
self.rate: Optional[float] = None
|
||||
self.last_check = 0
|
||||
self._last_response = None
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self.event = asyncio.Event()
|
||||
|
||||
@property
|
||||
def has_rate(self):
|
||||
return self.rate is not None
|
||||
|
||||
@property
|
||||
def is_online(self):
|
||||
return self.last_check+self.update_interval+self.request_timeout > time.time()
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def get_response(self):
|
||||
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
|
||||
try:
|
||||
self._last_response = await response.json()
|
||||
except ContentTypeError as e:
|
||||
self._last_response = {}
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
|
||||
log.debug(await response.text())
|
||||
return self._last_response
|
||||
|
||||
async def get_rate(self):
|
||||
try:
|
||||
data = await self.get_response()
|
||||
rate = self.get_rate_from_response(data)
|
||||
rate = rate / (1.0 - self.fee)
|
||||
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
|
||||
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
||||
self.last_check = time.time()
|
||||
self.event.set()
|
||||
return self.rate
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
||||
except json.JSONDecodeError as e:
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc)
|
||||
except InvalidExchangeRateResponseError as e:
|
||||
log.warning(str(e))
|
||||
except Exception as e:
|
||||
log.exception("Exchange rate error (%s from %s):", self.market, self.name)
|
||||
|
||||
async def keep_updated(self):
|
||||
while True:
|
||||
await self.get_rate()
|
||||
await asyncio.sleep(self.update_interval)
|
||||
|
||||
def start(self):
|
||||
if not self._task:
|
||||
self._task = asyncio.create_task(self.keep_updated())
|
||||
|
||||
def stop(self):
|
||||
if self._task and not self._task.done():
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
self.event.clear()
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
name = "Bittrex"
|
||||
market = "BTCLBC"
|
||||
url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
|
||||
params = {'market': 'BTC-LBC', 'count': 50}
|
||||
fee = 0.0025
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'result' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
trades = json_response['result']
|
||||
if len(trades) == 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
|
||||
totals = sum([i['Total'] for i in trades])
|
||||
qtys = sum([i['Quantity'] for i in trades])
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return float(1.0 / vwap)
|
||||
|
||||
|
||||
class LBRYFeed(MarketFeed):
|
||||
name = "lbry.com"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.lbry.com/lbc/exchange_rate"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
|
||||
|
||||
class LBRYBTCFeed(LBRYFeed):
|
||||
market = "USDBTC"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
name = "cryptonator.com"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(CryptonatorFeed):
|
||||
market = "USDBTC"
|
||||
url = "https://api.cryptonator.com/api/ticker/usd-btc"
|
||||
|
||||
|
||||
FEEDS: Iterable[Type[MarketFeed]] = (
|
||||
LBRYFeed,
|
||||
LBRYBTCFeed,
|
||||
BittrexFeed,
|
||||
# CryptonatorFeed,
|
||||
# CryptonatorBTCFeed,
|
||||
)
|
||||
|
||||
|
||||
class ExchangeRateManager:
|
||||
def __init__(self, feeds=FEEDS):
|
||||
self.market_feeds = [Feed() for Feed in feeds]
|
||||
|
||||
def wait(self):
|
||||
return asyncio.wait(
|
||||
[feed.event.wait() for feed in self.market_feeds],
|
||||
)
|
||||
|
||||
def start(self):
|
||||
log.info("Starting exchange rate manager")
|
||||
for feed in self.market_feeds:
|
||||
feed.start()
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping exchange rate manager")
|
||||
for source in self.market_feeds:
|
||||
source.stop()
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
|
||||
if from_currency == to_currency:
|
||||
return round(amount, 8)
|
||||
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return round(amount * Decimal(market.rate.spot), 8)
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return round(self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
def to_dewies(self, currency, amount) -> int:
|
||||
converted = self.convert_currency(currency, "LBC", amount)
|
||||
return lbc_to_dewies(str(converted))
|
||||
|
||||
def fee_dict(self):
|
||||
return {market: market.rate.as_dict() for market in self.market_feeds}
|
||||
@@ -6,11 +6,10 @@ from json import JSONEncoder
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.bip32 import PubKey
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.ledger import MainNetLedger, Account
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||
from lbry.wallet.bip32 import PubKey
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
|
||||
@@ -27,15 +26,19 @@ def encode_txo_doc():
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_received': "true if txo was sent from external account to this account",
|
||||
'is_spent': "true if txo is spent",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'payment'",
|
||||
'type': "one of 'claim', 'support' or 'purchase'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim' or 'support', this is the claim id",
|
||||
'claim_id': "when type is 'claim', 'support' or 'purchase', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'claim': "for purchase outputs only, metadata of purchased claim",
|
||||
'reposted_claim': "for repost claims only, metadata of claim being reposted",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
'purchase_receipt': "metadata for the purchase transaction associated with this claim"
|
||||
@@ -106,23 +109,25 @@ def encode_file_doc():
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'reflector_progress': '(int) reflector upload progress, 0 to 100',
|
||||
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
|
||||
}
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
|
||||
def __init__(self, *args, ledger: Ledger, include_protobuf=False, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
def default(self, obj): # pylint: disable=method-hidden,arguments-differ,too-many-return-statements
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
if isinstance(obj, (ManagedStream, TorrentSource)):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
@@ -164,12 +169,22 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
if txo.is_spent is not None:
|
||||
output['is_spent'] = txo.is_spent
|
||||
if txo.is_my_output is not None:
|
||||
output['is_my_output'] = txo.is_my_output
|
||||
if txo.is_my_input is not None:
|
||||
output['is_my_input'] = txo.is_my_input
|
||||
if txo.sent_supports is not None:
|
||||
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
|
||||
if txo.sent_tips is not None:
|
||||
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
|
||||
if txo.received_tips is not None:
|
||||
output['received_tips'] = dewies_to_lbc(txo.received_tips)
|
||||
if txo.is_internal_transfer is not None:
|
||||
output['is_internal_transfer'] = txo.is_internal_transfer
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
@@ -201,6 +216,10 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['short_url'] = output['meta'].pop('short_url')
|
||||
if 'canonical_url' in output['meta']:
|
||||
output['canonical_url'] = output['meta'].pop('canonical_url')
|
||||
if txo.claims is not None:
|
||||
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
||||
if txo.reposted_claim is not None:
|
||||
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
try:
|
||||
output['value'] = txo.claim
|
||||
@@ -228,7 +247,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
if isinstance(value, int):
|
||||
meta[key] = dewies_to_lbc(value)
|
||||
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
|
||||
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
|
||||
meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
|
||||
return meta
|
||||
|
||||
def encode_input(self, txi):
|
||||
@@ -244,7 +263,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_wallet(self, wallet):
|
||||
@staticmethod
|
||||
def encode_wallet(wallet):
|
||||
return {
|
||||
'id': wallet.id,
|
||||
'name': wallet.name
|
||||
@@ -254,26 +274,32 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
return {
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
is_stream = hasattr(managed_stream, 'stream_hash')
|
||||
if is_stream:
|
||||
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
|
||||
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
|
||||
else:
|
||||
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
|
||||
result = {
|
||||
'streaming_url': None,
|
||||
'completed': managed_stream.completed,
|
||||
'file_name': managed_stream.file_name if output_exists else None,
|
||||
'download_directory': managed_stream.download_directory if output_exists else None,
|
||||
'download_path': managed_stream.full_path if output_exists else None,
|
||||
'file_name': None,
|
||||
'download_directory': None,
|
||||
'download_path': None,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'stream_hash': None,
|
||||
'stream_name': None,
|
||||
'suggested_file_name': None,
|
||||
'sd_hash': None,
|
||||
'mime_type': None,
|
||||
'key': None,
|
||||
'total_bytes_lower_bound': total_bytes_lower_bound,
|
||||
'total_bytes': total_bytes,
|
||||
'written_bytes': managed_stream.written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'blobs_completed': None,
|
||||
'blobs_in_stream': None,
|
||||
'blobs_remaining': None,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
@@ -289,8 +315,38 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'added_on': managed_stream.added_on,
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||
'is_fully_reflected': False,
|
||||
'reflector_progress': False,
|
||||
'uploading_to_reflector': False
|
||||
}
|
||||
if is_stream:
|
||||
result.update({
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
})
|
||||
else:
|
||||
result.update({
|
||||
'streaming_url': f'file://{managed_stream.full_path}',
|
||||
})
|
||||
if output_exists:
|
||||
result.update({
|
||||
'file_name': managed_stream.file_name,
|
||||
'download_directory': managed_stream.download_directory,
|
||||
'download_path': managed_stream.full_path,
|
||||
})
|
||||
return result
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
@@ -1,11 +1,14 @@
|
||||
import asyncio
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
|
||||
import typing
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
import aiohttp
|
||||
from lbry import utils, __version__
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
@@ -14,6 +17,7 @@ class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
@@ -34,19 +38,22 @@ class JsonFormatter(logging.Formatter):
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
def __init__(self, loggly_token: str, config: 'Config'):
|
||||
super().__init__()
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
self.cookies = {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._session = aiohttp.ClientSession()
|
||||
self._config = config
|
||||
|
||||
def get_full_message(self, record):
|
||||
@property
|
||||
def enabled(self):
|
||||
return self._config.share_usage_data
|
||||
|
||||
@staticmethod
|
||||
def get_full_message(record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
@@ -59,13 +66,19 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except ClientError:
|
||||
if self._loop.is_running() and retry:
|
||||
if self._loop.is_running() and retry and self.enabled:
|
||||
await self._session.close()
|
||||
self._session = aiohttp.ClientSession()
|
||||
return await self._emit(record, retry=False)
|
||||
|
||||
def emit(self, record):
|
||||
asyncio.ensure_future(self._emit(record))
|
||||
if not self.enabled:
|
||||
return
|
||||
try:
|
||||
asyncio.ensure_future(self._emit(record), loop=self._loop)
|
||||
except RuntimeError: # TODO: use a second loop
|
||||
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
|
||||
f"{self.format(record)}")
|
||||
|
||||
def close(self):
|
||||
super().close()
|
||||
@@ -76,7 +89,7 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
pass
|
||||
|
||||
|
||||
def get_loggly_handler():
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
|
||||
def get_loggly_handler(config):
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -31,6 +31,10 @@ def migrate_db(conf, start, end):
|
||||
from .migrate10to11 import do_migration
|
||||
elif current == 11:
|
||||
from .migrate11to12 import do_migration
|
||||
elif current == 12:
|
||||
from .migrate12to13 import do_migration
|
||||
elif current == 13:
|
||||
from .migrate13to14 import do_migration
|
||||
else:
|
||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||
try:
|
||||
@@ -0,0 +1,80 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'bt_infohash' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.executescript("""
|
||||
pragma foreign_keys=off;
|
||||
|
||||
create table if not exists torrent (
|
||||
bt_infohash char(20) not null primary key,
|
||||
tracker text,
|
||||
length integer not null,
|
||||
name text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_node ( -- BEP-0005
|
||||
bt_infohash char(20) not null references torrent,
|
||||
host text not null,
|
||||
port integer not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_tracker ( -- BEP-0012
|
||||
bt_infohash char(20) not null references torrent,
|
||||
tracker text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_http_seed ( -- BEP-0017
|
||||
bt_infohash char(20) not null references torrent,
|
||||
http_seed text not null
|
||||
);
|
||||
|
||||
create table if not exists new_file (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text,
|
||||
added_on integer not null
|
||||
);
|
||||
|
||||
create table if not exists new_content_claim (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
claim_outpoint text unique not null references claim
|
||||
);
|
||||
|
||||
insert into new_file (stream_hash, bt_infohash, file_name, download_directory, blob_data_rate, status,
|
||||
saved_file, content_fee, added_on) select
|
||||
stream_hash, NULL, file_name, download_directory, blob_data_rate, status, saved_file, content_fee,
|
||||
added_on
|
||||
from file;
|
||||
|
||||
insert or ignore into new_content_claim (stream_hash, bt_infohash, claim_outpoint)
|
||||
select stream_hash, NULL, claim_outpoint from content_claim;
|
||||
|
||||
drop table file;
|
||||
drop table content_claim;
|
||||
alter table new_file rename to file;
|
||||
alter table new_content_claim rename to content_claim;
|
||||
|
||||
pragma foreign_keys=on;
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,21 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
create table if not exists peer (
|
||||
node_id char(96) not null primary key,
|
||||
address text not null,
|
||||
udp_port integer not null,
|
||||
tcp_port integer,
|
||||
unique (address, udp_port)
|
||||
);
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||