Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69e7566eaa | ||
|
|
399ac561c4 | ||
|
|
588568752b | ||
|
|
a30e2daf05 | ||
|
|
adb7edd3b2 | ||
|
|
06a9770504 | ||
|
|
0da43d3e05 | ||
|
|
077dbb3138 | ||
|
|
3b8fd65fa9 | ||
|
|
d0ddaa5518 | ||
|
|
a60ab97552 | ||
|
|
078d09976c | ||
|
|
3770558694 | ||
|
|
4409b8c4f8 | ||
|
|
f5b7aaac18 | ||
|
|
6dbb6b9079 | ||
|
|
16b774a69e | ||
|
|
72ea19532e | ||
|
|
ad24946af3 | ||
|
|
b676907fd7 | ||
|
|
6965971613 | ||
|
|
6651111522 | ||
|
|
d8630a02d9 | ||
|
|
69345b4b0e | ||
|
|
f23aea9951 | ||
|
|
f96c46e84a | ||
|
|
bae2939943 | ||
|
|
7063138e3c | ||
|
|
798ba9acf8 | ||
|
|
c8dd1987e6 | ||
|
|
8d2c9e5785 | ||
|
|
9dbf47916b | ||
|
|
565ef5e454 | ||
|
|
9f1f2b1781 | ||
|
|
76df239d1a | ||
|
|
86ee6a78f5 | ||
|
|
4fb6f2bb27 | ||
|
|
ec039825e8 | ||
|
|
a10f09d169 | ||
|
|
76409600e1 | ||
|
|
d06a0f876a | ||
|
|
8e857de62a | ||
|
|
3d37f7c735 | ||
|
|
552b94bc65 | ||
|
|
17164601a1 | ||
|
|
26dd6d2ac7 | ||
|
|
5d5e36a2c2 | ||
|
|
77afbcded6 | ||
|
|
ae11d72f66 | ||
|
|
d2773f9713 | ||
|
|
a7fba569cd | ||
|
|
0559c64038 | ||
|
|
5ab2754bc6 | ||
|
|
6bfb8384e7 | ||
|
|
0156aa8b00 | ||
|
|
ad411cc157 | ||
|
|
6d462ad6d8 | ||
|
|
1b35cef77a | ||
|
|
a357208a77 | ||
|
|
7e1f8a56d5 | ||
|
|
37665f2df7 | ||
|
|
8f3b596385 | ||
|
|
e45c6fe42c | ||
|
|
56edf62cb9 | ||
|
|
d367ff6ac1 | ||
|
|
7a78d49e13 | ||
|
|
4dae1ae05f | ||
|
|
9004f240e6 | ||
|
|
1848462ad9 | ||
|
|
786efa06dd | ||
|
|
b6811d55cf | ||
|
|
42dc0eccd6 | ||
|
|
29bd936181 | ||
|
|
9d97b90ff4 | ||
|
|
370b34f860 | ||
|
|
affa46e0f6 | ||
|
|
6b93705599 | ||
|
|
925e39c5a8 | ||
|
|
1efa2c9406 | ||
|
|
4f8bcfcd18 | ||
|
|
3ed72a215e | ||
|
|
85d94d4ca3 | ||
|
|
b922ea8e7e | ||
|
|
cb6ddaf276 | ||
|
|
1c2029cddf | ||
|
|
d8d136f1d8 | ||
|
|
fedc0e5bc3 | ||
|
|
4fbccb9336 | ||
|
|
ef5dd5593f | ||
|
|
baed974be0 | ||
|
|
b2b3509b61 | ||
|
|
51720d9b73 | ||
|
|
6abb569ec8 | ||
|
|
c2fbbadc69 | ||
|
|
b8897223ec | ||
|
|
c2c184b4ef | ||
|
|
07dbdef3fe | ||
|
|
b841fb2f0c | ||
|
|
2b75982446 | ||
|
|
021ca3efd2 | ||
|
|
e56b72cf04 | ||
|
|
342f485ddf | ||
|
|
8fe16fbfc9 | ||
|
|
1e3035029e | ||
|
|
5f4c02f836 | ||
|
|
03a1185e0f | ||
|
|
0e1705d19e | ||
|
|
82c739daa6 | ||
|
|
4702bea9f0 | ||
|
|
75211b7e84 | ||
|
|
9f9273bd82 | ||
|
|
d2608bdf43 | ||
|
|
091bef1ead | ||
|
|
998ee9feba | ||
|
|
587f0b2ea8 | ||
|
|
048aa07e80 | ||
|
|
7ca01511f4 | ||
|
|
8dee83aac0 | ||
|
|
879ae5cc72 | ||
|
|
74a9e17178 | ||
|
|
93d2173fad | ||
|
|
a459ee34be | ||
|
|
2e6297f834 | ||
|
|
9accbfcf8b | ||
|
|
6946860521 | ||
|
|
9b8be8682c | ||
|
|
03455310ae | ||
|
|
e5edaed677 | ||
|
|
5e97215eaa | ||
|
|
266f94bc95 | ||
|
|
d26caded53 | ||
|
|
2260608bb6 | ||
|
|
bb1f42057e | ||
|
|
695f6658a0 | ||
|
|
eca677a720 | ||
|
|
51f301785f | ||
|
|
41a6e288aa | ||
|
|
97d8d0ef4c | ||
|
|
5baeb5f538 | ||
|
|
cc49c8407c | ||
|
|
3f37963f7f | ||
|
|
12eb83f773 | ||
|
|
ca1d135ba4 | ||
|
|
107e57e3ff | ||
|
|
928689a10d | ||
|
|
6ecc22f2c7 | ||
|
|
079c81f298 | ||
|
|
8afe178d62 | ||
|
|
8777dcee7c | ||
|
|
1cca4a91dd | ||
|
|
41abfbdd9b | ||
|
|
46ee65c8f2 | ||
|
|
fd19daad9e | ||
|
|
26d183cbab | ||
|
|
71a4be02dd | ||
|
|
124fdb7d31 | ||
|
|
2439743804 | ||
|
|
6f79875135 | ||
|
|
b91d2190f4 | ||
|
|
d024433d1b | ||
|
|
7c4eb416d6 | ||
|
|
fb457c820a | ||
|
|
18af2dcd4e | ||
|
|
0075dcc2c0 | ||
|
|
6e327d729b | ||
|
|
20022d49c1 | ||
|
|
32d1563885 | ||
|
|
f5cef14d8d | ||
|
|
bac7d99b8a | ||
|
|
6f06026511 | ||
|
|
f02df86709 | ||
|
|
b7d76fd09f | ||
|
|
e675f1387c | ||
|
|
0da2827c78 | ||
|
|
309cfc840c | ||
|
|
68ee31b62a | ||
|
|
ad2da24d64 | ||
|
|
648dbcc36b | ||
|
|
e59e96190e | ||
|
|
7d2f592bb5 | ||
|
|
1a182bc9bd | ||
|
|
324cbdcdb2 | ||
|
|
6c0c0ee322 | ||
|
|
1e54271d8a | ||
|
|
54b9ed6cb2 | ||
|
|
558a02d187 | ||
|
|
3567b331c0 | ||
|
|
9e4cf3c995 | ||
|
|
db1d547b19 | ||
|
|
c31efd780b | ||
|
|
7cdb7987e3 | ||
|
|
ec07037e88 | ||
|
|
9bb7b0571e | ||
|
|
d98162a118 | ||
|
|
c5ae59bff7 | ||
|
|
efbf2f49a9 | ||
|
|
77c69f661d | ||
|
|
2e09d2adfa | ||
|
|
8620a0e424 | ||
|
|
e2c123456b | ||
|
|
c9be79f062 | ||
|
|
a2a1ef34ab | ||
|
|
761a2b235c | ||
|
|
3ca2fee592 | ||
|
|
a480b2d25f | ||
|
|
d7032b12d7 | ||
|
|
84381ff76c | ||
|
|
007dd43861 | ||
|
|
1bc4e4b702 | ||
|
|
b91f27219d | ||
|
|
dc4560cc9a | ||
|
|
84b471d486 | ||
|
|
c6a3c05a0c | ||
|
|
dd665a758d | ||
|
|
32dc1297ec | ||
|
|
bef244ba77 | ||
|
|
3c231d6215 | ||
|
|
a212cf6ba4 | ||
|
|
bc060f1378 | ||
|
|
22c701fd50 | ||
|
|
4f0e8fce6e | ||
|
|
f642cfe9dd | ||
|
|
24e073680b | ||
|
|
b2f63a1545 | ||
|
|
f506b3e6d4 | ||
|
|
f7412514b3 | ||
|
|
bd6a609b30 | ||
|
|
4e32b69d1d | ||
|
|
1116c7f29e | ||
|
|
d80c671cf2 | ||
|
|
14b12cbea2 | ||
|
|
1f7feafb67 | ||
|
|
cbe689ea7f | ||
|
|
9099ee2e8e | ||
|
|
b134e0c9c9 | ||
|
|
f8c0e80cfc | ||
|
|
1f266ebdad | ||
|
|
b09e1f8825 | ||
|
|
c496278a5b | ||
|
|
cacc65d45e | ||
|
|
2247b354b3 | ||
|
|
43111f0497 | ||
|
|
6c142b4477 | ||
|
|
e8683a24bd | ||
|
|
a4d17d1705 | ||
|
|
a030e13548 | ||
|
|
32b8601c38 | ||
|
|
f5bd28bfdb | ||
|
|
6cbbe23c7c | ||
|
|
d7c2e4e9b3 | ||
|
|
d61ddbb950 | ||
|
|
cf7bb6a391 | ||
|
|
68d7328918 | ||
|
|
ff2985cc80 | ||
|
|
ac69fcbc5f | ||
|
|
609cf42868 | ||
|
|
e43b29fcd1 | ||
|
|
6594b7c50c | ||
|
|
971f2a1cef | ||
|
|
fc88261cc6 | ||
|
|
895b3992c9 | ||
|
|
41951d8ec2 | ||
|
|
ae2121c5c4 | ||
|
|
771fe935f4 | ||
|
|
60f7766cf7 | ||
|
|
63bd983012 | ||
|
|
7ffce1040c | ||
|
|
ac6835fa60 | ||
|
|
3cf5c536c0 | ||
|
|
85cc1a4763 | ||
|
|
180cfcfb72 | ||
|
|
864c1dd901 | ||
|
|
aeaa7ba986 | ||
|
|
b8325c1f89 | ||
|
|
a36936b5ff | ||
|
|
7cf245a0e9 | ||
|
|
9f83cce574 | ||
|
|
30b113d64a | ||
|
|
ccd2023d37 | ||
|
|
3127cd11cf | ||
|
|
6e483cba8d | ||
|
|
46c03c0cb2 | ||
|
|
c2ca0a9467 | ||
|
|
b350e74aca | ||
|
|
de9473f75a | ||
|
|
bfc107fdb3 | ||
|
|
6a2efa54ab | ||
|
|
72f0e1ca14 | ||
|
|
9a1262d557 | ||
|
|
706a8e52dc | ||
|
|
f3c8776878 | ||
|
|
539ea04039 | ||
|
|
8f96b07559 | ||
|
|
0a1e97fe23 | ||
|
|
da637b9c0e | ||
|
|
3618de1227 | ||
|
|
3cb8a63bf9 | ||
|
|
bad99de5b3 | ||
|
|
3d1efbc9bb | ||
|
|
7bdef3146b | ||
|
|
2145bfcabf | ||
|
|
b88832d0a9 | ||
|
|
db08882f20 | ||
|
|
6f52d36b22 | ||
|
|
056a629e87 | ||
|
|
1ffb99f9f9 | ||
|
|
a8e612773d | ||
|
|
67088b2ca2 | ||
|
|
dab49e812f | ||
|
|
0f3f38d8a3 | ||
|
|
7ba433c2e5 | ||
|
|
bb7649e8ad | ||
|
|
832537a5cf | ||
|
|
3234d70270 | ||
|
|
432fe444f8 | ||
|
|
861c25716e | ||
|
|
9ba6653475 | ||
|
|
8107e97304 | ||
|
|
e2663f6d57 | ||
|
|
35479dcbd2 | ||
|
|
3a997277aa | ||
|
|
a7d94f4514 | ||
|
|
a0e3338177 | ||
|
|
d5a353066c | ||
|
|
bc29fc4465 | ||
|
|
987f175fcc | ||
|
|
5b1398b588 | ||
|
|
873b58efc9 | ||
|
|
01a81fceed | ||
|
|
a14b0aa573 | ||
|
|
a1abb7c8fb | ||
|
|
46b166952b | ||
|
|
1375153fda | ||
|
|
1a14157d37 | ||
|
|
40b9fa38b0 | ||
|
|
4d0e9e0428 | ||
|
|
3a53ef5690 | ||
|
|
33a68b5cef | ||
|
|
63bf084a6a | ||
|
|
a882f2e8c5 | ||
|
|
119e0ec408 | ||
|
|
7d782d516a | ||
|
|
d3eac48035 | ||
|
|
60c9ae64b4 | ||
|
|
2d314dce60 | ||
|
|
287b89db66 | ||
|
|
7a160f7335 | ||
|
|
04ddee7d89 | ||
|
|
3a916a8e8e | ||
|
|
f125468ebf | ||
|
|
f0e17cff9a | ||
|
|
93267efe0b | ||
|
|
c663e5a3cf | ||
|
|
676f0015aa | ||
|
|
d44a79ada2 | ||
|
|
d7ed8ffaa6 | ||
|
|
b230d8fbcb | ||
|
|
7d2428fd17 | ||
|
|
4157045a41 | ||
|
|
58987857cf | ||
|
|
fa8858325c | ||
|
|
3357a419ab | ||
|
|
a782adea75 | ||
|
|
e8991e6d52 | ||
|
|
13bd81c2b6 | ||
|
|
21241b98a4 | ||
|
|
bf467e06c5 | ||
|
|
886ed6415a | ||
|
|
97db241109 | ||
|
|
216379918b | ||
|
|
fcab4eb9bd | ||
|
|
19f300cc45 | ||
|
|
26fec098e4 | ||
|
|
7824de655e | ||
|
|
6ef1b1dd59 | ||
|
|
9fae8f5153 | ||
|
|
f660691e78 | ||
|
|
bf60683ba6 | ||
|
|
68215699f7 | ||
|
|
1644ffbf22 | ||
|
|
71f5061848 | ||
|
|
61cadb5cbe | ||
|
|
2034104cfc | ||
|
|
633f725634 | ||
|
|
8bf2e41c95 | ||
|
|
fa9a4a75c6 | ||
|
|
90bef98bc3 | ||
|
|
35c43647aa | ||
|
|
449a583286 | ||
|
|
b1f813782a | ||
|
|
60080cbd19 | ||
|
|
665dba62c7 | ||
|
|
76dea9e89b | ||
|
|
ebc9ed01c5 | ||
|
|
55116763e9 | ||
|
|
e1a6b4e3d6 | ||
|
|
911ca8c37e | ||
|
|
7f5deaf6c8 | ||
|
|
0200756218 | ||
|
|
a63442f843 | ||
|
|
74b3da79aa | ||
|
|
b6a4589da5 | ||
|
|
fbbb0a9931 | ||
|
|
a94d65d522 | ||
|
|
fa2ead002e | ||
|
|
8dfee4867a | ||
|
|
6614ab8e79 | ||
|
|
d32c1dfbb3 | ||
|
|
9f42b958fa | ||
|
|
574a912f7d | ||
|
|
2cfcc3cd45 | ||
|
|
9af625e627 | ||
|
|
1871c94003 | ||
|
|
93220c3874 | ||
|
|
362ed3f1c8 | ||
|
|
62a182b8bb | ||
|
|
6f76d9e24a | ||
|
|
55e83fe05c | ||
|
|
2f0c7d6a3f | ||
|
|
59e1df9efa | ||
|
|
fd56dc68cd | ||
|
|
a0a810edc3 | ||
|
|
c1d3b5984d | ||
|
|
1fb719dd61 | ||
|
|
b87316fb02 | ||
|
|
65f3d78084 | ||
|
|
47bc1526f7 | ||
|
|
3acabec4bb | ||
|
|
36a62d3756 | ||
|
|
2e22473ce3 | ||
|
|
0b0fd64642 | ||
|
|
ca17d8597d | ||
|
|
d16e2716c8 | ||
|
|
89c80cedfd | ||
|
|
021e30ffde | ||
|
|
eb2cd8fec1 | ||
|
|
6356818557 | ||
|
|
8bf724a78b | ||
|
|
65d0a924c1 | ||
|
|
ef7bd00f22 | ||
|
|
3c1fdb5114 | ||
|
|
83411acfbd | ||
|
|
4230812f82 | ||
|
|
7b57ba96dd | ||
|
|
668840921f | ||
|
|
a3bd335f7f | ||
|
|
4e6bca1389 | ||
|
|
c4b15920ff | ||
|
|
2a55e583e7 | ||
|
|
2c4fde06c5 | ||
|
|
eca83d7f47 | ||
|
|
87e818ac39 | ||
|
|
adab7bd7b1 | ||
|
|
750c0e4f4f | ||
|
|
5f8482bdb7 | ||
|
|
cd266477fb | ||
|
|
7860b956ff | ||
|
|
148481a5ca | ||
|
|
93faa276a5 | ||
|
|
53e67a6e81 | ||
|
|
3ad6808d87 | ||
|
|
071ab32ab7 | ||
|
|
ab87588250 | ||
|
|
22db29ee1d | ||
|
|
66bf6c61f0 | ||
|
|
9abf40d21f | ||
|
|
ea4444edbe | ||
|
|
51bb8fd283 | ||
|
|
183e3ac1c7 | ||
|
|
8087b457c6 | ||
|
|
fedb7c23d0 | ||
|
|
e2964f9728 | ||
|
|
e0cea60c87 | ||
|
|
c708c1db07 | ||
|
|
1cb696fb5c | ||
|
|
7cd0213eb9 | ||
|
|
2007c48af3 | ||
|
|
50721786d2 | ||
|
|
6d7eb259cb | ||
|
|
cc5852a9b9 | ||
|
|
0592cee567 | ||
|
|
814eaca524 | ||
|
|
e06916f5ae | ||
|
|
d61e1e9d8c | ||
|
|
25436893bc | ||
|
|
5b9d41fd5c | ||
|
|
b1720d1659 | ||
|
|
4b3a6ab875 | ||
|
|
67dcede2d8 | ||
|
|
0872a6fea3 | ||
|
|
a3aed3219f | ||
|
|
af73e1a2f5 | ||
|
|
55bbf5aa74 | ||
|
|
c1fbb02bfe | ||
|
|
d47575e8e0 | ||
|
|
8fc4f4aa73 | ||
|
|
fe9aa666e9 | ||
|
|
a9e0eeba7e | ||
|
|
0830917afb | ||
|
|
1ed086e90f | ||
|
|
56db631626 | ||
|
|
aa029955ee | ||
|
|
f2236eb7a4 | ||
|
|
df649341cd | ||
|
|
a3c3143868 | ||
|
|
d0201fe7c7 | ||
|
|
664a2a324d | ||
|
|
ea09aca175 | ||
|
|
ca92fddecf | ||
|
|
8259d7c680 | ||
|
|
6653da2973 | ||
|
|
e74998209d | ||
|
|
2bea0f697a | ||
|
|
1d68bef6f2 | ||
|
|
22bacc907b | ||
|
|
1e8ad900c7 | ||
|
|
cd2b535afb | ||
|
|
9dfe7bbcf6 | ||
|
|
120618ffd5 | ||
|
|
e8abcbb06e | ||
|
|
1ec8f0b0b4 | ||
|
|
323694fb12 | ||
|
|
d5e37acd8f | ||
|
|
129a886f2b | ||
|
|
cd15230a92 | ||
|
|
c1b7fdc68e | ||
|
|
9fd67aaad6 | ||
|
|
ce9dc1836e | ||
|
|
2e557d3a36 | ||
|
|
5df05cb37e | ||
|
|
326b62dc57 | ||
|
|
25cc7ba284 | ||
|
|
550f064302 | ||
|
|
11086b93f1 | ||
|
|
df5662136d | ||
|
|
291d65070d | ||
|
|
9b64ee33dc | ||
|
|
5404d26f2f | ||
|
|
26ade1f228 | ||
|
|
b49f1f1b54 | ||
|
|
3bafa2ad22 | ||
|
|
378ec44f3c | ||
|
|
9d0d17b422 | ||
|
|
f7c8145283 | ||
|
|
0de17730ca | ||
|
|
58743ba19b | ||
|
|
f2fefbe287 | ||
|
|
60a24f0e6e | ||
|
|
664f91bfab | ||
|
|
44f92a271f | ||
|
|
6565ca8558 | ||
|
|
3b0ba3e534 | ||
|
|
8106eb1fa5 | ||
|
|
c9b2829227 | ||
|
|
4f2f8b16f3 | ||
|
|
2ab2b846b1 | ||
|
|
2df7284c29 | ||
|
|
ab5d71ff61 | ||
|
|
99b289f135 | ||
|
|
6db02e4362 | ||
|
|
98d8b7d463 | ||
|
|
96f1883fc4 | ||
|
|
a914de155a | ||
|
|
2dee0ff0bf | ||
|
|
993cb43e5b | ||
|
|
5d212a0f82 | ||
|
|
0cc602ca53 | ||
|
|
f390ff144d | ||
|
|
b266cde511 | ||
|
|
d0a1541d0a | ||
|
|
4cbde1b08f | ||
|
|
c6740d8d70 | ||
|
|
46358142ed | ||
|
|
f0f651fc77 | ||
|
|
5d428138ed | ||
|
|
8d9da61c7d | ||
|
|
c326a9382c | ||
|
|
5bdd87c904 | ||
|
|
ad134fe8bb | ||
|
|
2f3c616acd | ||
|
|
3caa2a5039 | ||
|
|
e38a0f7a95 | ||
|
|
9de3f3ff09 | ||
|
|
df8934d472 | ||
|
|
be4a96c1b3 | ||
|
|
d97d978961 | ||
|
|
f52e44b56a | ||
|
|
10d73fe480 | ||
|
|
4fd2d994f2 | ||
|
|
d95903d219 | ||
|
|
591e66bfbb | ||
|
|
e047748961 | ||
|
|
9f071bbe36 | ||
|
|
2d8ebe25ed | ||
|
|
7a96e742f2 | ||
|
|
13c7026b20 | ||
|
|
876a3c68dd | ||
|
|
573617d6b6 | ||
|
|
e8c9abb944 | ||
|
|
8cee627dfa | ||
|
|
ab4a5cee01 | ||
|
|
5bb0a07fae | ||
|
|
5ca594b9fd | ||
|
|
3910964df1 | ||
|
|
faa1935203 | ||
|
|
ba518e2a0e | ||
|
|
2a82ff7cc5 | ||
|
|
673b958020 | ||
|
|
3f4ed0a896 | ||
|
|
f49578cdbc | ||
|
|
b834f29634 | ||
|
|
61132312dd | ||
|
|
fcdd9295b4 | ||
|
|
edb5083a5e | ||
|
|
4e7d88311f | ||
|
|
aa40337ded | ||
|
|
690e2e7e0c | ||
|
|
0c67469321 | ||
|
|
fc64d8d1a8 | ||
|
|
7fe9513857 | ||
|
|
e76107c318 | ||
|
|
3868dea819 | ||
|
|
c52541197f | ||
|
|
03cd6b7200 | ||
|
|
6915a7a553 | ||
|
|
7ad29b4bfb | ||
|
|
4294bf742d | ||
|
|
57ad9f1c52 | ||
|
|
e260de59f1 | ||
|
|
154702e6dd | ||
|
|
d06d998c81 | ||
|
|
8aff860bfe | ||
|
|
d7ba90bbea | ||
|
|
59325ad8c1 | ||
|
|
1f72e27272 | ||
|
|
aa41b8b558 | ||
|
|
2496e7ed44 | ||
|
|
ee12985135 | ||
|
|
4c7f357441 | ||
|
|
8944a36291 | ||
|
|
cb5d428d12 | ||
|
|
f175e8826c | ||
|
|
38ed212c01 | ||
|
|
d035c13883 | ||
|
|
82ce2ccee6 | ||
|
|
2cf9b2dd9e | ||
|
|
fcbdaa336f | ||
|
|
3ae6137edb | ||
|
|
0fdd2ae178 | ||
|
|
8c5007fe07 | ||
|
|
d2150cf142 | ||
|
|
ce2bb22929 | ||
|
|
1ce3adb6a0 | ||
|
|
c2dc1f3c1d | ||
|
|
b36c22e2f4 | ||
|
|
71b66ab337 | ||
|
|
461c3c8408 | ||
|
|
03396c1993 | ||
|
|
fa7e9923f4 | ||
|
|
3f7e466c97 | ||
|
|
4483380352 | ||
|
|
4a0a6b0243 | ||
|
|
a003e794ba | ||
|
|
051fabc180 | ||
|
|
ca686041e7 | ||
|
|
fc450eb164 | ||
|
|
652d7cbd24 | ||
|
|
43b372c0fe | ||
|
|
eaacf7e034 | ||
|
|
576da1bc30 | ||
|
|
5c217f1544 | ||
|
|
cfbf0dbe0f | ||
|
|
fd081496c0 | ||
|
|
32b4405a4c | ||
|
|
0d558bd552 | ||
|
|
ca835f3f80 | ||
|
|
dbb6ba6241 | ||
|
|
a228d20137 | ||
|
|
f1b60e3ef2 | ||
|
|
39737c790f | ||
|
|
dba4a09d38 | ||
|
|
918122e7cf | ||
|
|
a49f479a5f | ||
|
|
320aabbd70 | ||
|
|
19fdcad5af | ||
|
|
1734e08297 | ||
|
|
9d89f9b834 | ||
|
|
56c41b2fea | ||
|
|
9c8593d88e | ||
|
|
85f41887fd | ||
|
|
8eede6dfc7 | ||
|
|
0d8c37a63b | ||
|
|
d2ba3fdcfb | ||
|
|
be5ab0f307 | ||
|
|
04f70d1f31 | ||
|
|
60ba07cc17 | ||
|
|
dbc8cfe5ae | ||
|
|
d91e8f5996 | ||
|
|
b6df2b9d28 | ||
|
|
5c1ab42277 | ||
|
|
bbdc96e762 | ||
|
|
fc0c949187 | ||
|
|
3fce8e4023 | ||
|
|
37bb765a2e | ||
|
|
21ca4f60f7 | ||
|
|
119d498bce | ||
|
|
8108aa39c1 | ||
|
|
0a068011d8 | ||
|
|
1193d2cea4 | ||
|
|
76ecb5d50d | ||
|
|
dbc1164562 | ||
|
|
e2170ec83d | ||
|
|
266e777655 | ||
|
|
68905d82b9 | ||
|
|
396c49f7c6 | ||
|
|
5e509c9fd6 | ||
|
|
e6b6a3f55e | ||
|
|
019af119d9 | ||
|
|
58da2f16c2 | ||
|
|
8aa50aa14f | ||
|
|
65c6c83b51 | ||
|
|
f497989871 | ||
|
|
5366779b1b | ||
|
|
a3f59f041a | ||
|
|
d560b0c1cd | ||
|
|
e94229121a | ||
|
|
097291a498 | ||
|
|
afdc44ca13 | ||
|
|
6732b302bb | ||
|
|
c9d54bb049 | ||
|
|
f65e70df2f | ||
|
|
4541e754dd | ||
|
|
9f89c62e8a | ||
|
|
6386815a29 | ||
|
|
0c93090d2c | ||
|
|
45404b6ad4 | ||
|
|
aeb2891baa | ||
|
|
8abfdc82b2 | ||
|
|
da9b4e317a | ||
|
|
e414cc5c48 | ||
|
|
939edb2f18 | ||
|
|
868110a6f2 | ||
|
|
869a8b712b | ||
|
|
9ecec52412 | ||
|
|
8b25acff6e | ||
|
|
e2b06677b5 | ||
|
|
af1619ebfb | ||
|
|
3352e0e4f4 | ||
|
|
141d68a2cd | ||
|
|
c06ec6cd69 | ||
|
|
1be5dce30e | ||
|
|
73da223ae1 | ||
|
|
2d7eb83518 | ||
|
|
ad03f91d24 | ||
|
|
5586a226c2 | ||
|
|
a616582733 | ||
|
|
43ac928f0b | ||
|
|
a121110743 | ||
|
|
8bc5bb4e14 | ||
|
|
3a15ff4bcd | ||
|
|
3508da4993 | ||
|
|
dfdc8eda00 | ||
|
|
58f6cb71c6 | ||
|
|
ae11c5bb4b | ||
|
|
f22b5da170 | ||
|
|
c1c6d5bc99 | ||
|
|
0e972ec2ae | ||
|
|
f8dc0f298e | ||
|
|
8b5803c945 | ||
|
|
1314f75219 | ||
|
|
be90ec51e4 | ||
|
|
e06c903731 | ||
|
|
76c8447a56 | ||
|
|
82b53e108c | ||
|
|
7c8ad0fe30 | ||
|
|
073ffdd049 | ||
|
|
7fe211d134 | ||
|
|
43d424978d | ||
|
|
a4ced4dbd2 | ||
|
|
c6b910a5f3 | ||
|
|
63d1ec3c28 | ||
|
|
c31e61586d | ||
|
|
7b8f42e6d0 | ||
|
|
2c275efa48 | ||
|
|
30846f932b | ||
|
|
6ead932ccb | ||
|
|
e9b58577dd | ||
|
|
f766dbefe0 | ||
|
|
f8a7d2ebe2 | ||
|
|
af43b21ef6 | ||
|
|
0685bc5689 | ||
|
|
8cda3143f2 | ||
|
|
5af351a234 | ||
|
|
b9934a7eb7 | ||
|
|
b82d2c3120 | ||
|
|
8e607fcc2a | ||
|
|
2bce7cecfa | ||
|
|
e141cc9e22 | ||
|
|
8883587cc7 | ||
|
|
c85991704d | ||
|
|
377442d9f0 | ||
|
|
ad30b7df6d | ||
|
|
f0b49fc153 | ||
|
|
6033399f05 | ||
|
|
ae1162f5b0 | ||
|
|
7d7d4c35c3 | ||
|
|
ad71a9a89d | ||
|
|
13b3e4ba8e | ||
|
|
43799168ba | ||
|
|
736a949cb5 | ||
|
|
11bb1cc7ee | ||
|
|
dbce49e567 | ||
|
|
dcd78bd86e | ||
|
|
e5af041627 | ||
|
|
73f4e954e5 | ||
|
|
0a027cf2de | ||
|
|
3f98b16800 | ||
|
|
bbadece75d | ||
|
|
8f5a8ab8ba | ||
|
|
91e581422c | ||
|
|
b54d7185a1 | ||
|
|
f2968aab22 | ||
|
|
4e27eaf8e6 | ||
|
|
ccbe8aa0eb | ||
|
|
478f79353a | ||
|
|
c3b0151943 | ||
|
|
ba818354af | ||
|
|
e96b75a0d0 | ||
|
|
efe4afd09e | ||
|
|
c7cb6822cc | ||
|
|
c75665d3f0 | ||
|
|
744375b2c0 | ||
|
|
c1b4a012ec | ||
|
|
40da5b6cb6 | ||
|
|
b36cf3e36c | ||
|
|
4e3b529aec | ||
|
|
0a08f445a4 | ||
|
|
3eac99c509 | ||
|
|
f5bf1463af | ||
|
|
ca4a55ef28 | ||
|
|
832be0758b | ||
|
|
25050fdeeb | ||
|
|
df0635103e | ||
|
|
dabd9cbe1e | ||
|
|
42d7fbdbb3 | ||
|
|
2336015f9a | ||
|
|
5a28128b69 | ||
|
|
31445c7797 | ||
|
|
16d0ff8376 | ||
|
|
87f5d1bd4a | ||
|
|
be6e8744b0 | ||
|
|
2e9acd1baa | ||
|
|
607ccefe90 | ||
|
|
df86937f4e | ||
|
|
957d0290df | ||
|
|
40b90ce79b | ||
|
|
aa7f0845cc | ||
|
|
6e69d306de | ||
|
|
d85d3dfaa4 | ||
|
|
2b035009ef | ||
|
|
f9fd62c214 | ||
|
|
e8a1953815 | ||
|
|
34776e7eb9 | ||
|
|
a4419f562f | ||
|
|
c0160ab4e9 | ||
|
|
e60e5b3919 | ||
|
|
3589cc9977 | ||
|
|
2e978c00b2 | ||
|
|
2335243de9 | ||
|
|
6ad68eabd9 | ||
|
|
ca5c638124 | ||
|
|
262b9a624b | ||
|
|
05915c540a | ||
|
|
6aef6a80b7 | ||
|
|
16efe9ba95 | ||
|
|
fcda9500fe | ||
|
|
64dec821b7 | ||
|
|
d5f6e58aad | ||
|
|
087b65522e | ||
|
|
5ee4b2173a | ||
|
|
3e1c3d1e4b | ||
|
|
21f2b2dcd7 | ||
|
|
ffc7fc0793 | ||
|
|
15dbe5e54e | ||
|
|
2e633c0ee6 | ||
|
|
933d58d49b | ||
|
|
8f7cf0b38f | ||
|
|
adf378b6e9 | ||
|
|
1a4f083e76 | ||
|
|
7bde09dcf8 | ||
|
|
4630fd38aa | ||
|
|
418def56ac | ||
|
|
cefe3eb520 | ||
|
|
64dffa306f | ||
|
|
7d33b4f1f3 | ||
|
|
f507d95198 | ||
|
|
2a7d80f8b4 | ||
|
|
305034ffb2 | ||
|
|
bfaebf0085 | ||
|
|
a94cf04e87 | ||
|
|
71f9f8ae9c | ||
|
|
330862e487 | ||
|
|
778d3826ab | ||
|
|
4a11cf007f | ||
|
|
f6e7700c4c | ||
|
|
573285a5bb | ||
|
|
60ed4e503f | ||
|
|
aa1405887e | ||
|
|
cced217e43 | ||
|
|
8a3b8d2df1 | ||
|
|
be0bd3bdea | ||
|
|
e01c73a7f8 | ||
|
|
134338db4b | ||
|
|
f47a685734 | ||
|
|
6e29f7e99b | ||
|
|
026d93e92a | ||
|
|
a55802c1de | ||
|
|
894397a327 | ||
|
|
3e3f6c0e22 | ||
|
|
a7610e3d34 | ||
|
|
d0e6e74e1b | ||
|
|
126d71978b | ||
|
|
69bcb052de | ||
|
|
071af12d1a | ||
|
|
b7d62ca084 | ||
|
|
fa97f7bd38 | ||
|
|
4a35247016 | ||
|
|
d0526ad7d5 | ||
|
|
19b529cf38 | ||
|
|
143353be44 | ||
|
|
0373196707 | ||
|
|
7a601db23f | ||
|
|
ed61c7feec | ||
|
|
77ece96356 | ||
|
|
80440fd6bd | ||
|
|
ed6537db79 | ||
|
|
79e4767f69 | ||
|
|
0ee279bfd7 | ||
|
|
5249d2a3ca | ||
|
|
4d5823f473 | ||
|
|
c2b9909397 | ||
|
|
afff362544 | ||
|
|
64eb7dbc49 | ||
|
|
2dcd187d69 | ||
|
|
3d9ea92118 | ||
|
|
13a18d422e | ||
|
|
1bbbbd289c | ||
|
|
e33b74f60c | ||
|
|
cd76e0c296 | ||
|
|
3b61222e5a | ||
|
|
2ca438c36e | ||
|
|
8948d5be86 | ||
|
|
a3e64f9cef | ||
|
|
09afec1961 | ||
|
|
f9c465a636 | ||
|
|
89efe3d1b9 | ||
|
|
94cec1b86d | ||
|
|
d62ac0b608 | ||
|
|
5596eac930 | ||
|
|
59f5381b0a | ||
|
|
65ae1b3f08 | ||
|
|
977a7b34f4 | ||
|
|
c01728a6a7 | ||
|
|
5bb94d744e | ||
|
|
494917158c | ||
|
|
ea15674e62 | ||
|
|
34a409ade5 | ||
|
|
273e614715 | ||
|
|
34a725f7ad | ||
|
|
bb6112276f | ||
|
|
db75f0baef | ||
|
|
30689bbf8d | ||
|
|
4bf7742c19 | ||
|
|
69b259c285 | ||
|
|
2fa5233796 | ||
|
|
a5524d490c | ||
|
|
c713fac2d9 | ||
|
|
7a038bbb98 | ||
|
|
eba88c1df7 | ||
|
|
881e5aba68 | ||
|
|
368f0683ec | ||
|
|
94dc8c8520 | ||
|
|
8593cd362a | ||
|
|
2dbf96da81 | ||
|
|
83c7797187 | ||
|
|
6606deafa5 | ||
|
|
19b0d59159 | ||
|
|
9ccb3fa2a3 | ||
|
|
48533df523 | ||
|
|
1ae1137005 | ||
|
|
d5f2832b2f | ||
|
|
d0230b4893 | ||
|
|
a6cd53b97c | ||
|
|
7a28171a72 | ||
|
|
150bcb1116 | ||
|
|
6fe9c5bf00 | ||
|
|
3a86fc6680 | ||
|
|
2805cbb331 | ||
|
|
68d7a7014c | ||
|
|
883fa267a5 | ||
|
|
5f2c3ac965 | ||
|
|
bf8e32ebd2 | ||
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 | ||
|
|
6aa709e34e | ||
|
|
e73f121c08 | ||
|
|
3b9d6c4f8e | ||
|
|
69b60c19c3 | ||
|
|
04278cb1ab | ||
|
|
e8dec5b440 | ||
|
|
75f82b107e | ||
|
|
0e90c4b611 | ||
|
|
fb66b68bc8 | ||
|
|
c601677386 | ||
|
|
1fbd1cd04e | ||
|
|
4a71b91fbf | ||
|
|
b8b0d8b1ac | ||
|
|
8ef73d3985 | ||
|
|
05e556a5e3 | ||
|
|
d1da42bea8 | ||
|
|
90685e8357 | ||
|
|
a471d5d167 | ||
|
|
16122299e4 | ||
|
|
9f18da4bae | ||
|
|
1abce52867 | ||
|
|
072e2b65d3 | ||
|
|
f6d11e5467 | ||
|
|
458bddcb65 | ||
|
|
5fad43ff69 | ||
|
|
6992d0d947 | ||
|
|
d1b0180437 | ||
|
|
ba59e126cd | ||
|
|
4e3467bb7e | ||
|
|
4d4c705eb9 | ||
|
|
ce7b3184c4 | ||
|
|
b71faddaac | ||
|
|
1328cb4cce | ||
|
|
946488bce9 | ||
|
|
7713881e71 | ||
|
|
26fcc48c5a | ||
|
|
bf12fb68a2 | ||
|
|
3a4c9ff4f9 | ||
|
|
6180a2917d | ||
|
|
014cbb4623 | ||
|
|
ee98251d79 | ||
|
|
e3d08ec773 | ||
|
|
f03e68e63e | ||
|
|
72090b45dd | ||
|
|
93f9622798 | ||
|
|
16ff4b4e2c | ||
|
|
c90cca4b20 | ||
|
|
98d83ea74f | ||
|
|
a234f92d2c | ||
|
|
8ba3575901 | ||
|
|
a857c83c59 | ||
|
|
a6128ac8a2 | ||
|
|
2693301ad0 | ||
|
|
f7c7cb9535 | ||
|
|
b36a5f9b6b | ||
|
|
4826eb9969 | ||
|
|
c34b150794 | ||
|
|
c4c7670a7c | ||
|
|
0918bac86e | ||
|
|
a60a921706 | ||
|
|
b3ee29b66d | ||
|
|
3a1a9b988c | ||
|
|
12f2bebc46 | ||
|
|
31a3812c8a | ||
|
|
70ec08e30b | ||
|
|
23e8f0d849 | ||
|
|
1cdfcd5b4e | ||
|
|
dc2786fbc0 | ||
|
|
7e39b0c4fe | ||
|
|
4af830c405 | ||
|
|
fe844581fd | ||
|
|
443ce6352b | ||
|
|
158c354a27 | ||
|
|
92d1230a90 | ||
|
|
d64916a061 | ||
|
|
20a0568253 | ||
|
|
5a1f42ee54 | ||
|
|
ab95f14e21 | ||
|
|
bae2676b69 | ||
|
|
ea6869a63e | ||
|
|
953327c150 | ||
|
|
e01ad6bc85 | ||
|
|
47a33009ff | ||
|
|
bc844013a1 | ||
|
|
2fd2319cdb | ||
|
|
a50657c472 | ||
|
|
ef5f6e06ba | ||
|
|
977cf9f506 | ||
|
|
ffae10d11f | ||
|
|
221107e76e | ||
|
|
e74c78b607 | ||
|
|
434c7b57b1 | ||
|
|
9dc4608a62 | ||
|
|
fc69366092 | ||
|
|
9f0b1f3e25 | ||
|
|
3f03a845ec | ||
|
|
bc28976e12 | ||
|
|
8f826098f9 | ||
|
|
0a765e6207 | ||
|
|
f83aa61923 | ||
|
|
f9ff3e5d14 | ||
|
|
b3fde9d78d | ||
|
|
0c999083cc | ||
|
|
561215db8a | ||
|
|
8dbb7f2c78 | ||
|
|
6f7f3bb316 | ||
|
|
8d507c4b73 | ||
|
|
2beb0eda94 | ||
|
|
b7e07b72ca | ||
|
|
fde8718c8c | ||
|
|
b2d1bddcfe | ||
|
|
dd1d1be994 | ||
|
|
a93957178a | ||
|
|
5026d0b2f3 | ||
|
|
cc8b47d882 | ||
|
|
db0b5961b3 | ||
|
|
cb6f7ada42 | ||
|
|
59464099b4 | ||
|
|
aed00b53fd | ||
|
|
cdb0f97cb4 | ||
|
|
33bcc8256c | ||
|
|
0a8d44f12f | ||
|
|
1b74594f5b | ||
|
|
b0718d295e | ||
|
|
9bb91b126d | ||
|
|
6dbb00d8d5 | ||
|
|
3a27a4509a | ||
|
|
1d438f5d23 | ||
|
|
3840f19869 | ||
|
|
ec0483b975 | ||
|
|
2fba1f417f | ||
|
|
5b419b55c5 | ||
|
|
b31379e51c | ||
|
|
8394dcb663 | ||
|
|
457a56ce04 | ||
|
|
7c8016d807 | ||
|
|
d704336f20 | ||
|
|
1b4230b06e | ||
|
|
88095af1cd | ||
|
|
e9efd9f68a | ||
|
|
7b32e2ff7c | ||
|
|
572777305e | ||
|
|
7e27a6e37d | ||
|
|
b98621d7ea | ||
|
|
bc5824b8bc | ||
|
|
a6b643bb34 | ||
|
|
488b976734 | ||
|
|
253912b52e | ||
|
|
0bf8416d9f | ||
|
|
2e921437d8 | ||
|
|
b6f9b60683 | ||
|
|
57430e9f5f | ||
|
|
1e13fbee8d | ||
|
|
3d8b9e2b66 | ||
|
|
64898d43f1 | ||
|
|
b2117fddab | ||
|
|
5658623f29 | ||
|
|
934682bd0f | ||
|
|
daecc582f3 | ||
|
|
3cb4012dbf | ||
|
|
f7bd44a4f5 | ||
|
|
52317b05bb | ||
|
|
83c0baa3cb | ||
|
|
340c5addab | ||
|
|
e3fd94dbf0 | ||
|
|
923fcfb9e9 | ||
|
|
0ae5f1a4c6 | ||
|
|
90e06abba2 | ||
|
|
6ecfc94ff1 | ||
|
|
60c4a727d3 | ||
|
|
9fbb9f2e98 | ||
|
|
f1feec66c0 | ||
|
|
73b9b983b1 | ||
|
|
588779e3a4 | ||
|
|
b7ae446059 | ||
|
|
db898307b5 | ||
|
|
7e3512cbf2 | ||
|
|
cdd65c6783 | ||
|
|
320ad717cd | ||
|
|
264e6744e1 | ||
|
|
64e2b79c07 | ||
|
|
9255bb5abe | ||
|
|
d7525dacc3 | ||
|
|
5cb07100f8 | ||
|
|
171c080d33 | ||
|
|
cfc3ec2a05 | ||
|
|
1392b8f83b | ||
|
|
8b265801df | ||
|
|
6c4fb8dab1 | ||
|
|
c1a9cd93ee | ||
|
|
0d4e83e167 | ||
|
|
1292eb345d | ||
|
|
f4c6dc90c9 | ||
|
|
8c8f5d8f58 | ||
|
|
b6d986cb40 | ||
|
|
6cc134c7c8 | ||
|
|
eee964cfa0 | ||
|
|
a03c94bfad | ||
|
|
f1f0917063 | ||
|
|
fe2476b5aa | ||
|
|
9c5fa8bf76 | ||
|
|
cd43a71172 | ||
|
|
e2c6de76f4 | ||
|
|
05aeadc9fe | ||
|
|
7153417372 | ||
|
|
fceb2ae0a2 | ||
|
|
0cef2fdd82 | ||
|
|
9c91b135bd | ||
|
|
e3b127192c | ||
|
|
ee1738da6e | ||
|
|
8ca2b75c80 | ||
|
|
dc81b6def1 | ||
|
|
22f5aaee70 | ||
|
|
4bd6569c9c | ||
|
|
dbf39baeb1 | ||
|
|
3adf3c26f2 | ||
|
|
fa0d13e630 | ||
|
|
227a82d6d3 | ||
|
|
65cfc1f63c | ||
|
|
e46f0e2444 | ||
|
|
00bc7995c4 | ||
|
|
2dcebe48b7 | ||
|
|
d12b6cddb3 | ||
|
|
699c057f2d | ||
|
|
3ce9f31f4b | ||
|
|
485d5d55e8 | ||
|
|
6ee9f2513b | ||
|
|
81a6e14a5c | ||
|
|
496ffe0096 | ||
|
|
216c760179 | ||
|
|
01eca3004f | ||
|
|
2f47a3758e | ||
|
|
73bcc1796f | ||
|
|
e03e41ad31 | ||
|
|
6cb30f8e05 | ||
|
|
d9b1be6214 | ||
|
|
5f2f06c269 | ||
|
|
e1dcdcf4db | ||
|
|
416472c5d8 | ||
|
|
196f3ba92d | ||
|
|
6feae93c58 | ||
|
|
203d30d4f7 | ||
|
|
d0a7ca841b | ||
|
|
2e6c848c39 | ||
|
|
18a0204105 | ||
|
|
01baa193d5 | ||
|
|
ba57af5502 | ||
|
|
8139f93ff1 | ||
|
|
4596d5c26f | ||
|
|
810a63782d | ||
|
|
ea83827d57 | ||
|
|
e19f8f3d95 | ||
|
|
7af501e0bc | ||
|
|
a692b6ff7d | ||
|
|
3364ac0976 | ||
|
|
cf1157f02a | ||
|
|
0ed56bbe1a | ||
|
|
213bdac55f | ||
|
|
6d8ab97ca9 | ||
|
|
fb34476cd7 | ||
|
|
07d353a947 | ||
|
|
3c5fb6490d | ||
|
|
a5d21081b9 | ||
|
|
ed98a3cc21 | ||
|
|
45c2cd2986 | ||
|
|
75f21950da | ||
|
|
3f704be85c | ||
|
|
67a15ea788 | ||
|
|
55bcd2dfee | ||
|
|
d9757fdae8 | ||
|
|
a4ae8d7c53 | ||
|
|
78078b4c9f | ||
|
|
1331e2bbe6 | ||
|
|
4c4d652f1b | ||
|
|
73bd76b631 | ||
|
|
59a0f9f602 | ||
|
|
1b1847bc31 | ||
|
|
21a569edcc | ||
|
|
2ddf1a08f6 | ||
|
|
8754c6fabe | ||
|
|
f049109a38 | ||
|
|
8db7c15aa1 | ||
|
|
90e0694d2e | ||
|
|
4748cbf654 | ||
|
|
74eb991983 | ||
|
|
e11bae2dda | ||
|
|
8462f56b13 | ||
|
|
0bec3c7294 | ||
|
|
2bb4052193 | ||
|
|
101a3f2b22 | ||
|
|
04a2bde8dd | ||
|
|
a1368b05da | ||
|
|
99ea7a26b0 | ||
|
|
8b3a457925 | ||
|
|
e02e2a249a | ||
|
|
3019a1c222 | ||
|
|
1f3a19c100 | ||
|
|
5eac4247e1 | ||
|
|
1ca82b9d52 | ||
|
|
21702ec43c | ||
|
|
d2083bc17e | ||
|
|
52edf880e0 | ||
|
|
1f45af991f | ||
|
|
8afdd0b2e9 | ||
|
|
99efc70ba3 | ||
|
|
64e306801d | ||
|
|
330db61b36 | ||
|
|
357763ff61 | ||
|
|
8a6e6b4d77 | ||
|
|
64913c4288 | ||
|
|
f01ad0e4bd | ||
|
|
d9fcc458dc | ||
|
|
24a11d3f21 | ||
|
|
234eaf8890 | ||
|
|
9e2e53147e | ||
|
|
fc41af5889 | ||
|
|
72ce0cd0b0 | ||
|
|
e4497c1327 | ||
|
|
a1bf832ea6 | ||
|
|
d4e76500fb | ||
|
|
f898bb3d0e | ||
|
|
ad13282494 | ||
|
|
73519f8c6a | ||
|
|
b2c95f35c7 | ||
|
|
0bfccb86ff | ||
|
|
5118240897 | ||
|
|
427fc179ac | ||
|
|
55bcc520e4 | ||
|
|
bc8251696e | ||
|
|
677ba25ccd | ||
|
|
95436b233c | ||
|
|
82871b8f6b | ||
|
|
9525b87224 | ||
|
|
f06098978c | ||
|
|
141491f2da | ||
|
|
38006f7d29 | ||
|
|
3664c25d98 | ||
|
|
1747c9d866 | ||
|
|
1066fbaa7c | ||
|
|
7f7961f4cc | ||
|
|
9d9916548b | ||
|
|
ada5b55f22 | ||
|
|
a2f28e7788 | ||
|
|
c6092a4c79 | ||
|
|
315a557019 | ||
|
|
20221e1186 | ||
|
|
50003d2600 | ||
|
|
ea6b2b98fb | ||
|
|
4890fdfb50 | ||
|
|
2f95c3a9d1 | ||
|
|
6039159ecc | ||
|
|
8e4d8345bd | ||
|
|
22df26eeb6 | ||
|
|
7b9ff3e8b5 | ||
|
|
1a5b2c08ee | ||
|
|
b6babb1c9d | ||
|
|
a88485c5a0 | ||
|
|
0cefa07e97 | ||
|
|
ca9d48d574 | ||
|
|
e10e1de92f | ||
|
|
bdf6887791 | ||
|
|
295f2c1115 | ||
|
|
78fcf6a44a | ||
|
|
7dcebf99e0 | ||
|
|
463cbde4b1 | ||
|
|
1adce8b227 | ||
|
|
673d259d7a | ||
|
|
d0b739a0b9 | ||
|
|
b4aa51444c | ||
|
|
9178ca701c | ||
|
|
7eb9f344f4 | ||
|
|
9f8d3e69ae | ||
|
|
e1130147b5 | ||
|
|
0fb6f50aba | ||
|
|
91fc772e3c | ||
|
|
30b3d31af0 | ||
|
|
1a9f81ea14 | ||
|
|
5813df74a5 | ||
|
|
dc6d803fed | ||
|
|
f9cba69d03 | ||
|
|
17f9d7cfb2 | ||
|
|
42c43e86ef | ||
|
|
8cd230eb6e | ||
|
|
dcc71713c3 | ||
|
|
3d72b2adf3 | ||
|
|
655ece18de | ||
|
|
cda14f7c69 | ||
|
|
3d36e36173 | ||
|
|
2dd0221711 | ||
|
|
1fec9f52ef | ||
|
|
ca9ca09a00 | ||
|
|
b16c36b55f | ||
|
|
1c2849235c | ||
|
|
202a957a1a | ||
|
|
66f33202c7 | ||
|
|
f310bb6825 | ||
|
|
1fb7ffd759 | ||
|
|
3fcca4bc47 | ||
|
|
12c73cf50c | ||
|
|
02d398613d | ||
|
|
65e13898c2 | ||
|
|
559afa465a | ||
|
|
c753a5825a | ||
|
|
4185a3b9e3 | ||
|
|
2f4e6911d6 | ||
|
|
bcf7218b66 | ||
|
|
e21ade7360 | ||
|
|
53d4c149bc | ||
|
|
0adcaa6a5d | ||
|
|
ce4d734250 | ||
|
|
04e0933d36 | ||
|
|
bcf52c3f24 | ||
|
|
a702afd010 | ||
|
|
ad492a09f7 | ||
|
|
790991ecad | ||
|
|
f5789775a2 | ||
|
|
5cc0d365c2 | ||
|
|
5430a0f289 | ||
|
|
cc7656faa4 | ||
|
|
03415415ed | ||
|
|
1e47ff5a83 | ||
|
|
88bad1f868 | ||
|
|
91405ef232 | ||
|
|
570c1163c1 | ||
|
|
17827036ce | ||
|
|
338e6c2116 | ||
|
|
cdf6155102 | ||
|
|
e47ef741f7 | ||
|
|
2878716381 | ||
|
|
6d9947779e | ||
|
|
c7e5bdd3ae | ||
|
|
46dcf8a899 | ||
|
|
c050231ac5 | ||
|
|
ee5be5adc8 | ||
|
|
d99ce5f3d6 | ||
|
|
e7634b74c8 | ||
|
|
26af01a5cc | ||
|
|
36989f1974 | ||
|
|
1d40ec34a2 | ||
|
|
f0f30fd75c | ||
|
|
be2ef747dc | ||
|
|
268a01dc23 | ||
|
|
8399f2ab33 | ||
|
|
7698ab7053 | ||
|
|
eff66bdf80 | ||
|
|
70e68c6361 | ||
|
|
2a53afbfd8 | ||
|
|
a42bf67800 | ||
|
|
b783b4daed | ||
|
|
c38b463bf6 | ||
|
|
8696cd3c27 | ||
|
|
0abbc20899 | ||
|
|
2fca29144e | ||
|
|
1675cc4580 | ||
|
|
93d134f0ce | ||
|
|
1c6ce49b7b | ||
|
|
a7f34e2913 | ||
|
|
2a285d139a | ||
|
|
5e371e8e6a | ||
|
|
fdc98fc746 | ||
|
|
0d8e55bc95 | ||
|
|
69851a6588 | ||
|
|
b101fafd39 | ||
|
|
d2aaf7bd09 | ||
|
|
5a1f551ce5 | ||
|
|
584433e16c | ||
|
|
0c9c6c2705 | ||
|
|
afae649a9e | ||
|
|
2238e1f802 | ||
|
|
518c447fef | ||
|
|
cb9b348828 | ||
|
|
eab95a6246 | ||
|
|
593d0046bd | ||
|
|
c5d1478c02 | ||
|
|
1c57293e9c | ||
|
|
14a067a4c4 | ||
|
|
707ab831fd | ||
|
|
ceb25f917a | ||
|
|
002faf7a58 | ||
|
|
d1bbb7af74 | ||
|
|
ea603dce13 | ||
|
|
f2a768c9c4 | ||
|
|
12384a89a1 | ||
|
|
c21956b4f3 | ||
|
|
4a8776be10 | ||
|
|
d6d69a46d4 | ||
|
|
4384a5f022 | ||
|
|
aa83bbd637 | ||
|
|
daae0b2836 | ||
|
|
5afd446bb3 | ||
|
|
2b5e3204c0 | ||
|
|
84c91c480f | ||
|
|
cbc98e72a7 | ||
|
|
d1f00255c1 | ||
|
|
434d98c6a6 | ||
|
|
e51c98ca8c | ||
|
|
f719aa1db0 | ||
|
|
3686b1d970 | ||
|
|
e6fd8cc0f2 | ||
|
|
8739dc5d2c | ||
|
|
d6c04d3586 | ||
|
|
7236203f1e | ||
|
|
8c6c442fdd | ||
|
|
49c659b832 | ||
|
|
2590523fa1 | ||
|
|
08b4fd6942 | ||
|
|
6a8963d807 | ||
|
|
083bf4f61d | ||
|
|
ab3b0134b5 | ||
|
|
3b88d2465d | ||
|
|
0841c90e6c | ||
|
|
3f6e928cc2 | ||
|
|
a32a0c6401 | ||
|
|
95cb29d3c8 | ||
|
|
e85f8b245b | ||
|
|
04836ea0d9 | ||
|
|
451823f33e | ||
|
|
8c3552dd34 | ||
|
|
a204f0d3e6 | ||
|
|
892758be65 | ||
|
|
3594c8976d | ||
|
|
23ede44bff | ||
|
|
0006a68bf2 | ||
|
|
0349e68201 | ||
|
|
be248dc448 | ||
|
|
d0d5d0340d | ||
|
|
f79a49bbf4 | ||
|
|
6a5d88a0d5 | ||
|
|
4165881d5f | ||
|
|
8dc4e3be43 | ||
|
|
0badea874d | ||
|
|
0dd6193eaa | ||
|
|
d10778430f | ||
|
|
da8e09846d | ||
|
|
310fe4a42c | ||
|
|
a7ef8889dd | ||
|
|
f41229cb5b | ||
|
|
8ab4e3ca49 | ||
|
|
ff8d37443e | ||
|
|
fcd46629c4 | ||
|
|
9ad9eb083b | ||
|
|
4669507880 | ||
|
|
9348f4f366 | ||
|
|
47bb634035 | ||
|
|
10b34d6b33 | ||
|
|
f061ca2b15 | ||
|
|
bc24dbea29 | ||
|
|
2d4bf73632 | ||
|
|
c21d8cc8dc | ||
|
|
cc6a91a477 | ||
|
|
21a2725215 | ||
|
|
411b8c74cc | ||
|
|
91bf312e4d | ||
|
|
277b8b122c | ||
|
|
59c49d2d31 | ||
|
|
16024c95aa | ||
|
|
923ed80fcf | ||
|
|
2ee2916f78 | ||
|
|
79bf1b427f | ||
|
|
c21c329aa0 | ||
|
|
6b3af4bc21 | ||
|
|
ffbdf03847 | ||
|
|
8d08bfb3f9 | ||
|
|
1af6326b97 | ||
|
|
31630a84be | ||
|
|
0601bf3197 | ||
|
|
826d2866fa | ||
|
|
ba00498d83 | ||
|
|
4da40e8d52 | ||
|
|
d54d989c40 | ||
|
|
18ecbba87c | ||
|
|
9b417486a9 | ||
|
|
6d208ac496 | ||
|
|
8990c145fb | ||
|
|
e5b1c3c829 | ||
|
|
6c7128c0f6 | ||
|
|
f31f1fa40b | ||
|
|
2ceba79b9f | ||
|
|
1c8a59c1a9 | ||
|
|
298bf91293 | ||
|
|
b25d592d99 | ||
|
|
875edb5f76 | ||
|
|
84d97ab323 | ||
|
|
aeaffd620e | ||
|
|
3f7091cbf2 | ||
|
|
a95f791215 | ||
|
|
af2aeaa66b | ||
|
|
855fd8bf9a | ||
|
|
a937aff80f | ||
|
|
5597d45aed | ||
|
|
473d2eabfa | ||
|
|
78c8c8e64d | ||
|
|
7335d012ef | ||
|
|
99be38604a | ||
|
|
d35d340613 | ||
|
|
f05ca137be | ||
|
|
e718caca77 | ||
|
|
693a3346d2 | ||
|
|
1f75b6a49e | ||
|
|
4ece422f48 | ||
|
|
607677f98f | ||
|
|
e1e7be63b8 | ||
|
|
e1314a9d1e | ||
|
|
1ee682f06f | ||
|
|
ba80c0e594 | ||
|
|
fbdbcc8070 | ||
|
|
24a872885a | ||
|
|
005a131440 | ||
|
|
0b1cb16353 | ||
|
|
1a3eaf7e6a | ||
|
|
971252d5d1 | ||
|
|
c5ba9b263e | ||
|
|
c01716a6c0 | ||
|
|
0fbf131df3 | ||
|
|
4bd0806e58 | ||
|
|
4cbb71705e | ||
|
|
8ab4dd6b2c | ||
|
|
318c369752 | ||
|
|
cea3b7630c | ||
|
|
19211d4417 | ||
|
|
c312d1b3a6 | ||
|
|
e4ea1ccbfb | ||
|
|
5b35c4e8f0 | ||
|
|
9967857a57 | ||
|
|
78c560a3be | ||
|
|
07a12b66e8 | ||
|
|
a28c9d09c8 | ||
|
|
b2ab13187f | ||
|
|
5ec696fbfd | ||
|
|
44bd0ae847 | ||
|
|
5809ba093d | ||
|
|
17aee92dc9 | ||
|
|
6f1d2e7690 | ||
|
|
75f57fc517 | ||
|
|
f589410496 | ||
|
|
aecc7c6643 | ||
|
|
c5c1b62939 | ||
|
|
adc92a2b52 | ||
|
|
9636ca22e7 | ||
|
|
0442770c9d | ||
|
|
3b6a879fc9 | ||
|
|
a1b4c9acd6 | ||
|
|
0f90dee224 | ||
|
|
159e84468c | ||
|
|
59e4ac30c2 | ||
|
|
b5ce948bc1 | ||
|
|
076af7ef43 | ||
|
|
43bef9447c | ||
|
|
b1c5fe0b4d | ||
|
|
39d7f2e46e | ||
|
|
315661208d | ||
|
|
e175e43082 | ||
|
|
cb9b320ed7 | ||
|
|
1d906fb0c7 | ||
|
|
c544e26206 | ||
|
|
1eaac35a3e | ||
|
|
cc7962fc2e | ||
|
|
3638984d67 | ||
|
|
62e77c69f5 | ||
|
|
774d813576 | ||
|
|
688caf4453 | ||
|
|
eeb5b3c139 | ||
|
|
b5e6dd1060 | ||
|
|
272d818050 | ||
|
|
0fc1bd7a8c | ||
|
|
2b234e0713 | ||
|
|
9e8cb17ecb | ||
|
|
e0230864ce | ||
|
|
9e89b3e40e | ||
|
|
036663ae62 | ||
|
|
b62321689d | ||
|
|
bdd271e78f | ||
|
|
4a56d38782 | ||
|
|
70a7ca95fe | ||
|
|
f85e61d8ed | ||
|
|
ac8c641d64 | ||
|
|
d81502e191 | ||
|
|
8c35d355e5 | ||
|
|
72b004664f | ||
|
|
49a7408968 | ||
|
|
17738ad248 | ||
|
|
425c952293 | ||
|
|
03d2d0e237 | ||
|
|
edfc2c6124 | ||
|
|
545451a829 | ||
|
|
12ff7015cd | ||
|
|
f2f212edba | ||
|
|
69446491b8 | ||
|
|
d5beaa0937 | ||
|
|
cc970c499b | ||
|
|
0087b16912 | ||
|
|
5520d518b5 | ||
|
|
b3e9240aa8 | ||
|
|
137a270488 | ||
|
|
e36610d63c | ||
|
|
c4273fbef0 | ||
|
|
a4e9f3100f | ||
|
|
2668ceac0e | ||
|
|
fb2fb5c38c | ||
|
|
56175df121 | ||
|
|
7d3daa9fe6 | ||
|
|
1d2de27536 | ||
|
|
aab42e56bd | ||
|
|
1368a6e074 | ||
|
|
80990b959c | ||
|
|
053b6d1c80 | ||
|
|
692d8e8e1a | ||
|
|
5e71dcbaf0 | ||
|
|
83958604d5 | ||
|
|
dcd8a6bb0e | ||
|
|
7161a0edb9 | ||
|
|
0fd160e6e6 | ||
|
|
ca8b2dd83e | ||
|
|
79d1da5ff8 | ||
|
|
d73252b11a | ||
|
|
1f5c56cac3 | ||
|
|
18db2aa970 | ||
|
|
a7a53f9381 | ||
|
|
f4b2a05fff | ||
|
|
cdd2b6bfc1 | ||
|
|
26bd27cd32 | ||
|
|
613feb0808 | ||
|
|
d4849443e5 | ||
|
|
db70f7e98a | ||
|
|
37ddaeca12 | ||
|
|
2c546b2cde | ||
|
|
c48d195951 | ||
|
|
149fa625e0 | ||
|
|
16cda60f31 | ||
|
|
0d54f6215f | ||
|
|
a9774a293f | ||
|
|
ee8cf08c0e | ||
|
|
ea481e6c5a | ||
|
|
36df9e8283 | ||
|
|
0467b65e69 | ||
|
|
68f1dda7b0 | ||
|
|
5b7f7c87bf | ||
|
|
153022a1a7 | ||
|
|
1098ca0494 | ||
|
|
d4a7dbf805 | ||
|
|
56ee815eec | ||
|
|
fe43764bcb | ||
|
|
93be8b9e31 | ||
|
|
2448d51ed5 | ||
|
|
2804fa4448 | ||
|
|
90a119bb2c | ||
|
|
69ae60b4bb | ||
|
|
dca4af942f | ||
|
|
2b7e135839 | ||
|
|
d565aa0b3f | ||
|
|
8b53140581 | ||
|
|
917bae568d | ||
|
|
efb6af4d51 | ||
|
|
2361d8bdc1 | ||
|
|
5fe8022b15 | ||
|
|
7931397595 | ||
|
|
0501f52805 | ||
|
|
108d2a033b | ||
|
|
17a9cfd65c | ||
|
|
7069ffa37d | ||
|
|
8eb740cf67 | ||
|
|
7803eb2e5a | ||
|
|
b73e50e64b | ||
|
|
c73148a1fb | ||
|
|
65a02190dc | ||
|
|
74b49fe69f | ||
|
|
26e32472d5 | ||
|
|
20d8b2963c | ||
|
|
d21313e590 | ||
|
|
81e9c9bf74 | ||
|
|
382226ff9a | ||
|
|
9511040d5a | ||
|
|
049936f4a6 | ||
|
|
15161dd429 | ||
|
|
efc5982860 | ||
|
|
acb95fdcdf | ||
|
|
f1b19f10cc | ||
|
|
6eaf1ac2a0 | ||
|
|
01ee1e1468 | ||
|
|
83dc295b7b | ||
|
|
4a28c620a4 | ||
|
|
e50ac69358 | ||
|
|
aef561d78b | ||
|
|
4ef4a4816f | ||
|
|
8533bfa7c5 | ||
|
|
6389e961b9 | ||
|
|
6359fc608e | ||
|
|
314400a1bd | ||
|
|
a3de065c93 | ||
|
|
99207b7221 | ||
|
|
e3c3fafa1e | ||
|
|
5b3103e41b | ||
|
|
9742da348b | ||
|
|
94a31922f7 | ||
|
|
1937eb17c4 | ||
|
|
f291ea515a | ||
|
|
56522f39c0 | ||
|
|
ff4aba9423 | ||
|
|
9ab256df30 | ||
|
|
88b8ca8a49 | ||
|
|
09f637496c | ||
|
|
826d269e0e | ||
|
|
5163baf9c1 | ||
|
|
ab27203100 | ||
|
|
88c2051605 | ||
|
|
f510c2a433 | ||
|
|
66982c86a9 | ||
|
|
ec140d5d8a | ||
|
|
ad96b006f9 | ||
|
|
32507d74f7 | ||
|
|
2a00d00e44 | ||
|
|
40c5f6e3ab | ||
|
|
ae67918c8a | ||
|
|
96e73f3476 | ||
|
|
c1b05402ef | ||
|
|
1dd1bad9c2 | ||
|
|
9b873d2f48 | ||
|
|
5984ae7ce9 | ||
|
|
903cd86cdd | ||
|
|
5d9a187659 | ||
|
|
a9c94ca22d | ||
|
|
a285db1b08 | ||
|
|
13bf8125e3 | ||
|
|
b06dcf0a0d | ||
|
|
a800f6ddf0 | ||
|
|
a89306b6bf | ||
|
|
c3120e93cf | ||
|
|
defe9506bb | ||
|
|
55d3bb0ec3 | ||
|
|
005a8b3008 | ||
|
|
a741fdfc44 | ||
|
|
944b94aae8 | ||
|
|
37eaf634c4 | ||
|
|
edcb06a415 | ||
|
|
62b50dc0ae | ||
|
|
e7c57dcabc | ||
|
|
094d9c6497 | ||
|
|
3dc5a9de7b | ||
|
|
4e29041291 | ||
|
|
6b6a29fdb7 | ||
|
|
2d61ba629f | ||
|
|
5a2075019c | ||
|
|
944200ca8c | ||
|
|
40d8e96811 | ||
|
|
7e8ca842a2 | ||
|
|
b808d08eb3 | ||
|
|
eb11da9b19 | ||
|
|
68b31a09b4 | ||
|
|
cab8416596 | ||
|
|
94f4e6714a | ||
|
|
2ddd6b051d | ||
|
|
c506fc5c75 | ||
|
|
d306ece0d4 | ||
|
|
d29f103910 | ||
|
|
2386a3a79b | ||
|
|
ec51333b56 | ||
|
|
75829812f3 | ||
|
|
4d720370b1 | ||
|
|
7d7dd665aa | ||
|
|
b6bee47481 | ||
|
|
0ec8739d5a | ||
|
|
7d09d1402f | ||
|
|
5492ab9081 | ||
|
|
7f6bbc0648 | ||
|
|
b1d4072c0b | ||
|
|
562d23df11 | ||
|
|
2c13c04f86 | ||
|
|
7365bfad9f | ||
|
|
1af1cf212c | ||
|
|
5c7d279104 | ||
|
|
cfe8e17223 | ||
|
|
91d4bf3cd9 | ||
|
|
df9642c325 | ||
|
|
af98b0eb4d | ||
|
|
c589e89bf0 | ||
|
|
284264962d | ||
|
|
6952c2c07e | ||
|
|
6a15b51ac3 | ||
|
|
75a6ff269e | ||
|
|
148cc96025 | ||
|
|
14efa94382 | ||
|
|
da7e7503b9 | ||
|
|
67bedfafcb |
No files matched your search
@@ -1,48 +0,0 @@
|
||||
version: 1.0.{build}
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
|
||||
environment:
|
||||
GH_TOKEN:
|
||||
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
|
||||
key_pass:
|
||||
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
|
||||
pfx_key:
|
||||
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
|
||||
AWS_ACCESS_KEY_ID:
|
||||
secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
|
||||
|
||||
#notifications:
|
||||
# - provider: Slack
|
||||
# incoming_webhook:
|
||||
# secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
|
||||
|
||||
|
||||
clone_folder: c:\projects\lbry
|
||||
|
||||
test_script:
|
||||
- cd C:\projects\lbry\
|
||||
- pip install cython
|
||||
- pip install mock pylint unqlite Faker
|
||||
- pip install .
|
||||
- pylint lbrynet
|
||||
# disable tests for now so that appveyor can build the app
|
||||
- set PYTHONPATH=.
|
||||
- trial lbrynet.tests
|
||||
# TODO: integration tests do not work
|
||||
#- python -m unittest discover tests/integration
|
||||
#- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
|
||||
|
||||
build_script:
|
||||
- cd C:\projects\lbry\build\
|
||||
- ps: .\build.ps1
|
||||
|
||||
|
||||
artifacts:
|
||||
- path: build\dist\*.zip
|
||||
name: lbrynet-daemon
|
||||
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
@@ -4,10 +4,10 @@ Thanks for reporting an issue to LBRY and helping us improve!
|
||||
To make it possible for us to help you, please fill out below information carefully.
|
||||
|
||||
Before reporting any issues, please make sure that you're using the latest version.
|
||||
- App: https://github.com/lbryio/lbry-app/releases
|
||||
- App: https://github.com/lbryio/lbry-desktop/releases
|
||||
- Daemon: https://github.com/lbryio/lbry/releases
|
||||
|
||||
We are also available on Discord at https://chat.lbry.io
|
||||
We are also available on Discord at https://chat.lbry.com
|
||||
-->
|
||||
|
||||
|
||||
@@ -51,3 +51,24 @@ Tell us what happens instead
|
||||
|
||||
## Screenshots
|
||||
<!-- If a screenshot would help explain the bug, please include one or two here -->
|
||||
|
||||
|
||||
|
||||
## Internal Use
|
||||
|
||||
### Acceptance Criteria
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
### Definition of Done
|
||||
- [ ] Tested against acceptance criteria
|
||||
- [ ] Tested against the assumptions of user story
|
||||
- [ ] The project builds without errors
|
||||
- [ ] Unit tests are written and passing
|
||||
- [ ] Tests on devices/browsers listed in the issue have passed
|
||||
- [ ] QA performed & issues resolved
|
||||
- [ ] Refactoring completed
|
||||
- [ ] Any configuration or build changes documented
|
||||
- [ ] Documentation updated
|
||||
- [ ] Peer Code Review performed
|
||||
@@ -0,0 +1,38 @@
|
||||
## PR Checklist
|
||||
Please check all that apply to this PR using "x":
|
||||
|
||||
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
|
||||
- [ ] I have checked that this PR does not introduce a breaking change
|
||||
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
|
||||
|
||||
|
||||
## PR Type
|
||||
What kind of change does this PR introduce?
|
||||
|
||||
Why is this change necessary?
|
||||
|
||||
<!-- Please check all that apply to this PR using "x". -->
|
||||
|
||||
- [ ] Bugfix
|
||||
- [ ] Feature
|
||||
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
|
||||
- [ ] Code style update (formatting)
|
||||
- [ ] Refactoring (no functional changes)
|
||||
- [ ] Documentation changes
|
||||
- [ ] Other - Please describe:
|
||||
|
||||
## Fixes
|
||||
|
||||
Issue Number: N/A
|
||||
|
||||
|
||||
## What is the current behavior?
|
||||
|
||||
|
||||
## What is the new behavior?
|
||||
|
||||
|
||||
## Other information
|
||||
|
||||
|
||||
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
|
||||
@@ -1,25 +1,13 @@
|
||||
*.pyc
|
||||
*.egg
|
||||
*.so
|
||||
*.xml
|
||||
*.iml
|
||||
*.log
|
||||
*.pem
|
||||
*.decTest
|
||||
*.prof
|
||||
.#*
|
||||
|
||||
/build/build
|
||||
/build/dist
|
||||
/bulid/requirements_base.txt
|
||||
|
||||
/lbrynet.egg-info
|
||||
/docs_build
|
||||
.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
/lbry-venv
|
||||
|
||||
.idea/
|
||||
.coverage
|
||||
.DS_Store
|
||||
|
||||
# temporary files from the twisted.trial test runner
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
ignore=CVS,schema
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
@@ -33,9 +33,7 @@ unsafe-load-any-extension=no
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
miniupnpc,
|
||||
unqlite
|
||||
# extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
@@ -62,7 +60,7 @@ confidence=
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# disable everything first and then re-enable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
@@ -121,7 +119,11 @@ disable=
|
||||
unidiomatic-typecheck,
|
||||
global-at-module-level,
|
||||
inconsistent-return-statements,
|
||||
keyword-arg-before-vararg
|
||||
keyword-arg-before-vararg,
|
||||
assignment-from-no-return,
|
||||
useless-return,
|
||||
assignment-from-none,
|
||||
stop-iteration-return
|
||||
|
||||
|
||||
[REPORTS]
|
||||
@@ -350,13 +352,13 @@ ignore-mixin-members=yes
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb,distutils
|
||||
ignored-modules=leveldb,distutils
|
||||
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet.reactor,RequestMessage
|
||||
# ignored-classes=
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
@@ -386,7 +388,7 @@ int-import-graph=
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
max-args=10
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
@@ -405,7 +407,7 @@ max-branches=12
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
max-parents=8
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
@@ -1,46 +1,145 @@
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
dist: xenial
|
||||
language: python
|
||||
python: 2.7
|
||||
python: "3.7"
|
||||
|
||||
branches:
|
||||
except:
|
||||
- gh-pages
|
||||
env:
|
||||
global:
|
||||
# must also be updated in wine_build.sh
|
||||
- TORBA=master
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
|
||||
jobs:
|
||||
include:
|
||||
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
- stage: test
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
services:
|
||||
- docker
|
||||
install:
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-windows.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-windows.zip
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- &build
|
||||
name: "Linux"
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
|
||||
- ./dist/lbrynet --version
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- <<: *build
|
||||
name: "Mac"
|
||||
os: osx
|
||||
osx_image: xcode8.3
|
||||
language: generic
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
addons:
|
||||
#srcclr:
|
||||
# debug: true
|
||||
apt:
|
||||
packages:
|
||||
- libgmp3-dev
|
||||
- build-essential
|
||||
- git
|
||||
- libssl-dev
|
||||
- libffi-dev
|
||||
|
||||
before_install:
|
||||
- virtualenv venv
|
||||
- source venv/bin/activate
|
||||
|
||||
install:
|
||||
- pip install -U pip==9.0.3
|
||||
- pip install -r requirements.txt
|
||||
- pip install -r requirements_testing.txt
|
||||
- pip install .
|
||||
|
||||
script:
|
||||
- pip install mock pylint
|
||||
- pylint lbrynet
|
||||
- PYTHONPATH=. trial lbrynet.tests
|
||||
- rvm install ruby-2.3.1
|
||||
- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
- $TRAVIS_BUILD_DIR/.tox
|
||||
@@ -4,37 +4,215 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/) with
|
||||
regard to the json-rpc api. As we're currently pre-1.0 release, we
|
||||
can and probably will change functionality and break backwards compatability
|
||||
can and probably will change functionality and break backwards compatibility
|
||||
at anytime.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.30.1] - 2018-12-09
|
||||
|
||||
### API
|
||||
* add `aioupnp_version` to `upnp` field in the response to `status`
|
||||
* fixed uncaught claim signature errors
|
||||
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
|
||||
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
|
||||
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
|
||||
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
|
||||
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
|
||||
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
|
||||
|
||||
### Schema
|
||||
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
|
||||
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
|
||||
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
|
||||
|
||||
### Wallet
|
||||
* fixed resolving names with invalid channel signatures
|
||||
* fixed wallet sync issues
|
||||
* fixed wallet locking/encryption issues
|
||||
|
||||
### DHT
|
||||
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
|
||||
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
|
||||
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
|
||||
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
|
||||
|
||||
### Database
|
||||
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
|
||||
|
||||
### Misc
|
||||
* switch to Python 3.7
|
||||
* brought back icon for windows binary
|
||||
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
|
||||
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
|
||||
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
|
||||
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
|
||||
|
||||
## [0.30.0] - 2018-10-22
|
||||
This release is the most ambitious upgrade to lbrynet yet; including a brand
|
||||
new wallet implementation, switch to Python 3 of the entire code base and
|
||||
many changes to the API such as pagination and a more consistent output from
|
||||
most commands.
|
||||
|
||||
|
||||
### Security
|
||||
*
|
||||
*
|
||||
* upgraded `cryptography` package.
|
||||
* switch to `coincurve` for private / public key generation (addresses)
|
||||
* support for connecting to daemon via SSL
|
||||
* improved workflow for encrypt/decrypt of wallet
|
||||
|
||||
### API
|
||||
* unified all command line executables into a single `lbrynet` executable.
|
||||
* added `use_https` configuration setting (off by default), if true the api
|
||||
server will generate a self signed ssl certificate when started and only
|
||||
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
|
||||
* deprecated `daemon_stop` command, use `stop` instead.
|
||||
* deprecated `wallet_balance` command, use `account_balance` instead.
|
||||
* deprecated `wallet_unlock` command, use `account_unlock` instead.
|
||||
* deprecated `wallet_decrypt` command, use `account_decrypt` instead.
|
||||
* deprecated `wallet_encrypt` command, use `account_encrypt` instead.
|
||||
* deprecated `wallet_prefill_addresses` command, use `account_fund` instead.
|
||||
* deprecated `wallet_list` command, use `address_list` instead.
|
||||
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
|
||||
* deprecated `wallet_public_key` command.
|
||||
* deprecated `wallet_new_address` command.
|
||||
* deprecated `wallet_unused_address` command, use `address_unused` instead.
|
||||
* deprecated `claim_renew` command.
|
||||
* added `account_list` command to list accounts including their balance.
|
||||
* added `account_add` command to add a previously created account from seed or private key.
|
||||
* added `account_create` command to generate a new account.
|
||||
* added `account_remove` command to remove an account from wallet.
|
||||
* added `account_set` command to change a setting on an account.
|
||||
* added `account_balance` command to get just the account balance.
|
||||
* added `account_unlock` command to unlock an encrypted account.
|
||||
* added `account_lock` command to lock an encrypted account.
|
||||
* added `account_encrypt` command to encrypt an account.
|
||||
* added `account_decrypt` command to decrypt an account.
|
||||
* added `account_fund` command to move funds between or within an account in various ways.
|
||||
* added `account_max_address_gap` command to find large gaps of unused addresses.
|
||||
* added `address_list` command to list addresses.
|
||||
* added `address_is_mine` command to check if an address is one of your addresses.
|
||||
* added `address_unused` command to get existing or generate a new unused address.
|
||||
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
|
||||
`transaction_list` and `utxo_list`.
|
||||
* added `upnp` field to `status` response
|
||||
* removed `send_amount_to_address` command previously marked as deprecated
|
||||
* removed `channel_list_mine` command previously marked as deprecated
|
||||
* removed `get_availability` command previously marked as deprecated
|
||||
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
|
||||
|
||||
### Wallet
|
||||
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
|
||||
* changed wallet file format to support multiple accounts in one wallet.
|
||||
* moved transaction data from wallet file into an sqlite database.
|
||||
* changed channel certificates to be keyed by txid:nout instead of claim_id which
|
||||
makes it possible to recover old certificates.
|
||||
|
||||
### File Manager
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### DHT
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### P2P
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### Reflector
|
||||
* No major changes to reflector.
|
||||
|
||||
### Database
|
||||
* No changes to existing storage module.
|
||||
|
||||
### Known Issues
|
||||
* Channel certificate import/export is currently not working.
|
||||
|
||||
## [0.21.2] - 2018-08-23
|
||||
### Fixed
|
||||
* `blob_list` raising an error when blobs in a stream haven't yet been created
|
||||
* stopping a download from raising `NoneType object has no attribute finished_deferred`
|
||||
* file manager startup locking up when there are many files for some channels
|
||||
* improper sorting when getting the closest peers to a hash
|
||||
* issue in dht ping queue where enqueued pings that aren't yet due wouldn't be rescheduled
|
||||
* blob mirror downloader not finishing streams that were partially uploaded at the time of the download attempt (https://github.com/lbryio/lbry/issues/1376)
|
||||
|
||||
### Deprecated
|
||||
*
|
||||
*
|
||||
|
||||
### Changed
|
||||
* default for `peer_search_timeout` raised from 3 to 30 and being logged on console
|
||||
* change iterative find stop condition on find value to allow it to continue until a value is found (or it times out)
|
||||
* include all of our own blobs in the local dht datastore (as if we had announced them to ourselves)
|
||||
* ignore dht `store` token validation errors for the first expiration-time after startup (fixes failed `store` requests after a restart)
|
||||
## [0.21.1] - 2018-08-13
|
||||
### Fixed
|
||||
* `download_progress` field in `blockchain_headers` section of `status` not initializing correctly when resuming a download (https://github.com/lbryio/lbry/issues/1355)
|
||||
* `wallet_send` not accepting decimal amounts (https://github.com/lbryio/lbry/issues/1356 commit https://github.com/lbryio/lbry/commit/1098ca0494ece420c70fc57f69d6d388715a99b8)
|
||||
|
||||
### Added
|
||||
*
|
||||
*
|
||||
* `is_locked` to `wallet` in `status` response (https://github.com/lbryio/lbry/issues/1354, commit https://github.com/lbryio/lbry/commit/153022a1a7122ab6d31d3db433dccbe469bbcb3c)
|
||||
|
||||
### Changed
|
||||
* Bumped `lbryum` requirement to 3.2.4 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#324---2018-08-13)
|
||||
|
||||
## [0.21.0] - 2018-08-09
|
||||
### Fixed
|
||||
* check `claim_address` and `change_address` earlier on publishing, to avoid hard to understand errors later in the process (pr https://github.com/lbryio/lbry/pull/1347)
|
||||
* loggly error reporting not following `share_usage_data` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* improper error handling when data is not valid JSON (pr https://github.com/lbryio/lbry/pull/1326)
|
||||
* blob mirroring being set in unrelated tests, making them fail (pr https://github.com/lbryio/lbry/pull/1348)
|
||||
* http blob mirroring edge cases (pr https://github.com/lbryio/lbry/pull/1315)
|
||||
* external ports in log messages not showing the correct external port from the upnp redirects (https://github.com/lbryio/lbry/issues/1338) (pr https://github.com/lbryio/lbry/pull/1349)
|
||||
* miniupnpc fallback issues in txupnp (https://github.com/lbryio/lbry/issues/1341) (pr https://github.com/lbryio/lbry/pull/1349)
|
||||
* upnp error when disabled on router and a non-gateway is found, such as chromecast (https://github.com/lbryio/lbry/issues/1352) (https://github.com/lbryio/lbry/commit/dca4af942fbe95547794213775f0a62cd04a393f)
|
||||
|
||||
### Deprecated
|
||||
* automatic claim renew, this is no longer needed
|
||||
|
||||
### Changed
|
||||
* api server class to use components, and for all JSONRPC API commands to be callable so long as the required components are available. (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* return error messages when required conditions on components are not met for API calls (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* `status` to no longer return a base58 encoded `lbry_id`, instead return this as the hex encoded `node_id` in a new `dht` field. (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* `startup_status` field in the response to `status` to be a dict of component names to status booleans (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* renamed the `blockchain_status` field in the response to `status` to `wallet` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* moved and renamed `wallet_is_encrypted` to `is_encrypted` in the `wallet` field in the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* moved wallet, upnp and dht startup code from `Session` to `Components` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* attempt blob downloads from http mirror sources (by default) concurrently to p2p sources (pr https://github.com/lbryio/lbry/pull/1233)
|
||||
* replace miniupnpc with [txupnp](https://github.com/lbryio/txupnp). Since txupnp is still under development, it will internally fall back to miniupnpc. (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* simplified test_misc.py in the functional tests (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* update `cryptography` requirement to 2.3 (pr https://github.com/lbryio/lbry/pull/1333)
|
||||
|
||||
### Added
|
||||
* `skipped_components` list to the response from `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* component statuses (`blockchain_headers`, `dht`, `wallet`, `blob_manager` `hash_announcer`, and `file_manager`) to the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* `skipped_components` config setting, accepts a list of names of components to not run (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* `ComponentManager` for managing the life-cycles of dependencies (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* `requires` decorator to register the components required by a `jsonrpc_` command, to facilitate commands registering asynchronously (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* unit tests for `ComponentManager` (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* script to generate docs/api.json file (https://github.com/lbryio/lbry.tech/issues/42)
|
||||
* additional information to the balance error message when editing a claim (pr https://github.com/lbryio/lbry/pull/1309)
|
||||
* `address` and `port` arguments to `peer_ping` (https://github.com/lbryio/lbry/issues/1313) (pr https://github.com/lbryio/lbry/pull/1299)
|
||||
* ability to download from HTTP mirrors by setting `download_mirrors` (prs https://github.com/lbryio/lbry/pull/1233 and https://github.com/lbryio/lbry/pull/1315)
|
||||
* ability to filter peers from an iterative find value operation (finding peers for a blob). This is used to filter peers we've already found for a blob when accumulating the list of peers. (pr https://github.com/lbryio/lbry/pull/1287)
|
||||
|
||||
### Removed
|
||||
* `jsonrpclib` as a requirement for the project
|
||||
*
|
||||
* `session_status` argument and response field from `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* most of the internal attributes from `Daemon` (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
|
||||
|
||||
## [0.20.4] - 2018-07-18
|
||||
### Fixed
|
||||
* spelling errors in messages printed by `lbrynet-cli`
|
||||
* high CPU usage when a stream is incomplete and the peers we're requesting from have no more blobs to send us (https://github.com/lbryio/lbry/pull/1301)
|
||||
|
||||
### Changed
|
||||
* keep track of failures for DHT peers for up to ten minutes instead of indefinitely (https://github.com/lbryio/lbry/pull/1300)
|
||||
* skip ignored peers from iterative lookups instead of blocking the peer who returned them to us too (https://github.com/lbryio/lbry/pull/1300)
|
||||
* if a node becomes ignored during an iterative find cycle remove it from the shortlist so that we can't return it as a result nor try to probe it anyway (https://github.com/lbryio/lbry/pull/1303)
|
||||
|
||||
|
||||
## [0.20.3] - 2018-07-03
|
||||
### Fixed
|
||||
* `blob_list` raising an error when blobs in a stream haven't yet been created (8a0d0b44ddf9cbeb2a9074eed39d6064ce21df64)
|
||||
* stopping a download potentially raising an attribute error (https://github.com/lbryio/lbry/pull/1269)
|
||||
* file manager startup locking up when there are many files for some channels (https://github.com/lbryio/lbry/pull/1281)
|
||||
* improper sorting when getting the closest peers to a hash (https://github.com/lbryio/lbry/pull/1282)
|
||||
|
||||
### Changed
|
||||
* raised the default `peer_search_timeout` setting from 3 to 30 and added logging for when it happens (https://github.com/lbryio/lbry/pull/1283)
|
||||
* change iterative find stop condition on find value to allow it to continue until a value is found or it times out (https://github.com/lbryio/lbry/pull/1283)
|
||||
* include all of our own blobs in the local dht datastore (as if we had announced them to ourselves) (https://github.com/lbryio/lbry/pull/1280)
|
||||
* ignore dht `store` token validation errors for the first expiration-time after startup (fixes failed `store` requests after a restart) (https://github.com/lbryio/lbry/pull/1280)
|
||||
|
||||
### Removed
|
||||
* `jsonrpclib` as a requirement for the project (https://github.com/lbryio/lbry/pull/1274)
|
||||
|
||||
|
||||
## [0.20.2] - 2018-06-23
|
||||
@@ -308,7 +486,7 @@ at anytime.
|
||||
* Fixed https://github.com/lbryio/lbry/issues/923
|
||||
* Fixed concurrent reflects opening too many files
|
||||
* Fixed cases when reflecting would fail on error conditions
|
||||
* Fixed deadlocks from occuring during blob writes
|
||||
* Fixed deadlocks from occurring during blob writes
|
||||
* Fixed and updated`lbrynet.tests.dht`
|
||||
* Fixed redundant dht id
|
||||
* Fixed dht `ping` method
|
||||
@@ -695,7 +873,7 @@ at anytime.
|
||||
* Removed check_pending logic from Daemon
|
||||
* Switched to txrequests so requests can use twisted event loop
|
||||
* Renamed API command file_seed to file_set_status
|
||||
* Dont add expected payment to wallet when payment rate is 0
|
||||
* Don't add expected payment to wallet when payment rate is 0
|
||||
### Fixed
|
||||
* Fix restart procedure in DaemonControl
|
||||
* Create download directory if it doesn't exist
|
||||
@@ -726,7 +904,7 @@ at anytime.
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
* Made dictionary key names in API commmand outputs to be more consistent
|
||||
* Made dictionary key names in API command outputs to be more consistent
|
||||
### Added
|
||||
* Add file filters: `claim_id`, `outpoint`, and `rowid`
|
||||
* Make loggly logs less verbose
|
||||
@@ -778,7 +956,7 @@ at anytime.
|
||||
* Include download url in version check
|
||||
|
||||
### Fixed
|
||||
* add misssing traceback to logging
|
||||
* add missing traceback to logging
|
||||
|
||||
## [0.8.3] - 2017-02-15
|
||||
### Fixed
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
## Contributing to LBRY
|
||||
|
||||
https://lbry.io/faq/contributing
|
||||
https://lbry.tech/contribute
|
||||
@@ -1,70 +1,45 @@
|
||||
# Installing LBRY
|
||||
|
||||
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
|
||||
[](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, you'll need to install some additional libraries depending on your operating system.
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
|
||||
### Python
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
LBRY requires Python 2.7. Python 3 is not supported.
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
### OS X Prerequisites
|
||||
|
||||
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
These environment variables also need to be set
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install mpfr libmpc protobuf
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
brew install python protobuf
|
||||
```
|
||||
|
||||
### Linux Prerequisites
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
|
||||
|
||||
On Ubuntu (we recommend 16.04), install the following:
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux flavor, install the equivalent of the above packages for your system.
|
||||
|
||||
### Windows Prerequisites
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
|
||||
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- requests
|
||||
|
||||
Install the above binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -73,34 +48,33 @@ To install:
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
Creating venv:
|
||||
python -m venv lbry-venv
|
||||
|
||||
virtualenv lbry-venv
|
||||
Activating lbry-venv on Linux/Mac:
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
python --version # Python 3 is not supported. Make sure you're on Python 2.7
|
||||
|
||||
pip install -U -r requirements.txt
|
||||
pip install --editable .
|
||||
Activating lbry-venv on Windows:
|
||||
lbry-venv\Scripts\activate
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
## Run the tests
|
||||
To run the unit tests from the repo directory:
|
||||
```
|
||||
trial --reactor=asyncio tests.unit
|
||||
```
|
||||
|
||||
## Making a Change
|
||||
## Usage
|
||||
|
||||
If you are not already inside the virtual environment, run:
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
|
||||
```
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
This will install `lbrynet-daemon` in such a way that the changes you make to the code will be used as soon as you restart the daemon.
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2018 LBRY Inc
|
||||
Copyright (c) 2015-2019 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -1,52 +1,56 @@
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
[](https://coveralls.io/github/lbryio/lbry)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
# LBRY
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network.
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
This repo is a reference implementation of the [LBRY API](https://lbry.io/api).
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
It provides a daemon that can interact with the network via a json-rpc interface over HTTP.
|
||||
## Installation
|
||||
|
||||
## Installing
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
|
||||
**Note**: This project no longer bundles a graphic interface (browser). If you want to use LBRY via a GUI, [use the LBRY App](https://github.com/lbryio/lbry-app).
|
||||
## Usage
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows.
|
||||
Run `lbrynet start` to launch the API server.
|
||||
|
||||
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
|
||||
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
## Running
|
||||
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
|
||||
|
||||
Run `lbrynet-daemon` to launch the daemon.
|
||||
With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
|
||||
## Using
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
|
||||
By default, `lbrynet-daemon` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
## Running from source
|
||||
|
||||
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
The full API is documented [here](https://lbryio.github.io/lbry/cli).
|
||||
## Contributing
|
||||
|
||||
## What is LBRY?
|
||||
|
||||
LBRY is a protocol that provides a fully decentralized network for the discovery, distribution, and payment of data.
|
||||
|
||||
It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules.
|
||||
|
||||
LBRY also provides a data network that consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
|
||||
|
||||
## Contributions
|
||||
|
||||
To contribute, [join us on Discord](https://chat.lbry.io/) or contact jeremy@lbry.io. Pull requests are also welcome.
|
||||
|
||||
## Support
|
||||
|
||||
Please open an issue and describe your situation in detail. We will respond as soon as we can.
|
||||
|
||||
For private issues, contact jeremy@lbry.io.
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
|
||||
## License
|
||||
|
||||
See [LICENSE](LICENSE)
|
||||
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
|
||||
@@ -1,33 +0,0 @@
|
||||
$env:Path += ";C:\MinGW\bin\"
|
||||
|
||||
$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
|
||||
gcc --version
|
||||
mingw32-make --version
|
||||
|
||||
# build/install miniupnpc manually
|
||||
tar zxf miniupnpc-1.9.tar.gz
|
||||
cd miniupnpc-1.9
|
||||
mingw32-make -f Makefile.mingw
|
||||
python setupmingw32.py build --compiler=mingw32
|
||||
python setupmingw32.py install
|
||||
cd ..\
|
||||
Remove-Item -Recurse -Force miniupnpc-1.9
|
||||
|
||||
# copy requirements from lbry, but remove miniupnpc (installed manually)
|
||||
Get-Content ..\requirements.txt | Select-String -Pattern 'miniupnpc' -NotMatch | Out-File requirements_base.txt
|
||||
|
||||
python set_build.py
|
||||
|
||||
pip install -r requirements.txt
|
||||
pip install ..\.
|
||||
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
pyinstaller -y console.onefile.spec
|
||||
|
||||
nuget install secure-file -ExcludeVersion
|
||||
secure-file\tools\secure-file -decrypt .\lbry2.pfx.enc -secret "$env:pfx_key"
|
||||
signtool.exe sign /f .\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
|
||||
|
||||
python zip_daemon.py
|
||||
python upload_assets.py
|
||||
@@ -1,59 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
|
||||
cd "$ROOT"
|
||||
BUILD_DIR="$ROOT/build"
|
||||
|
||||
FULL_BUILD="${FULL_BUILD:-false}"
|
||||
if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then
|
||||
FULL_BUILD="true"
|
||||
fi
|
||||
|
||||
[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build"
|
||||
[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# install dependencies
|
||||
$BUILD_DIR/prebuild.sh
|
||||
|
||||
VENV="$BUILD_DIR/venv"
|
||||
if [ -d "$VENV" ]; then
|
||||
rm -rf "$VENV"
|
||||
fi
|
||||
virtualenv "$VENV"
|
||||
set +u
|
||||
source "$VENV/bin/activate"
|
||||
set -u
|
||||
|
||||
# must set build before installing lbrynet. otherwise it has no effect
|
||||
python "$BUILD_DIR/set_build.py"
|
||||
fi
|
||||
|
||||
cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt"
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pip install -r requirements.txt
|
||||
)
|
||||
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
pyinstaller -y console.onefile.spec
|
||||
)
|
||||
|
||||
python "$BUILD_DIR/zip_daemon.py"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# electron-build has a publish feature, but I had a hard time getting
|
||||
# it to reliably work and it also seemed difficult to configure. Not proud of
|
||||
# this, but it seemed better to write my own.
|
||||
python "$BUILD_DIR/upload_assets.py"
|
||||
|
||||
deactivate
|
||||
fi
|
||||
|
||||
echo 'Build complete.'
|
||||
@@ -1,42 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-cli', pathex=[cwd])
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-cli',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
import lbryum
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
datas = [
|
||||
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
|
||||
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
|
||||
]
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-console', pathex=[cwd], datas=datas)
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-console',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
import lbryum
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
datas = [
|
||||
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
|
||||
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
|
||||
]
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-daemon', pathex=[cwd], datas=datas)
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-daemon',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,47 +0,0 @@
|
||||
# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point
|
||||
def Entrypoint(dist, group, name,
|
||||
scripts=None, pathex=None, binaries=None, datas=None,
|
||||
hiddenimports=None, hookspath=None, excludes=None, runtime_hooks=None,
|
||||
cipher=None, win_no_prefer_redirects=False, win_private_assemblies=False):
|
||||
import pkg_resources
|
||||
|
||||
# get toplevel packages of distribution from metadata
|
||||
def get_toplevel(dist):
|
||||
distribution = pkg_resources.get_distribution(dist)
|
||||
if distribution.has_metadata('top_level.txt'):
|
||||
return list(distribution.get_metadata('top_level.txt').split())
|
||||
else:
|
||||
return []
|
||||
|
||||
hiddenimports = hiddenimports or []
|
||||
packages = []
|
||||
for distribution in hiddenimports:
|
||||
packages += get_toplevel(distribution)
|
||||
|
||||
scripts = scripts or []
|
||||
pathex = pathex or []
|
||||
# get the entry point
|
||||
ep = pkg_resources.get_entry_info(dist, group, name)
|
||||
# insert path of the egg at the verify front of the search path
|
||||
pathex = [ep.dist.location] + pathex
|
||||
# script name must not be a valid module name to avoid name clashes on import
|
||||
script_path = os.path.join(workpath, name + '-script.py')
|
||||
print "creating script for entry point", dist, group, name
|
||||
with open(script_path, 'w') as fh:
|
||||
fh.write("import {0}\n".format(ep.module_name))
|
||||
fh.write("{0}.{1}()\n".format(ep.module_name, '.'.join(ep.attrs)))
|
||||
for package in packages:
|
||||
fh.write("import {0}\n".format(package))
|
||||
|
||||
return Analysis([script_path] + scripts,
|
||||
pathex=pathex,
|
||||
binaries=binaries,
|
||||
datas=datas,
|
||||
hiddenimports=hiddenimports,
|
||||
hookspath=hookspath,
|
||||
excludes=excludes,
|
||||
runtime_hooks=runtime_hooks,
|
||||
cipher=cipher,
|
||||
win_no_prefer_redirects=win_no_prefer_redirects,
|
||||
win_private_assemblies=win_private_assemblies
|
||||
)
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
|
||||
LINUX=false
|
||||
OSX=false
|
||||
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
OSX=true
|
||||
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
|
||||
LINUX=true
|
||||
else
|
||||
echo "Platform detection failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
SUDO=''
|
||||
if $LINUX && (( $EUID != 0 )); then
|
||||
SUDO='sudo'
|
||||
fi
|
||||
|
||||
cmd_exists() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
return $?
|
||||
}
|
||||
|
||||
set +eu
|
||||
GITUSERNAME=$(git config --global --get user.name)
|
||||
if [ -z "$GITUSERNAME" ]; then
|
||||
git config --global user.name "$(whoami)"
|
||||
fi
|
||||
GITEMAIL=$(git config --global --get user.email)
|
||||
if [ -z "$GITEMAIL" ]; then
|
||||
git config --global user.email "$(whoami)@lbry.io"
|
||||
fi
|
||||
set -eu
|
||||
|
||||
|
||||
if $LINUX; then
|
||||
INSTALL="$SUDO apt-get install --no-install-recommends -y"
|
||||
$INSTALL build-essential libssl-dev libffi-dev python2.7-dev wget
|
||||
elif $OSX && ! cmd_exists brew ; then
|
||||
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
|
||||
fi
|
||||
|
||||
|
||||
if ! cmd_exists python; then
|
||||
if $LINUX; then
|
||||
$INSTALL python2.7
|
||||
elif $OSX; then
|
||||
brew install python
|
||||
fi
|
||||
fi
|
||||
|
||||
PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
|
||||
if [ "$PYTHON_VERSION" != "2.7" ]; then
|
||||
echo "Python 2.7 required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! cmd_exists pip; then
|
||||
if $LINUX; then
|
||||
$INSTALL python-pip
|
||||
$SUDO pip install --upgrade pip
|
||||
else
|
||||
echo "Pip required"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
|
||||
#$INSTALL python-setuptools
|
||||
$SUDO pip install setuptools
|
||||
fi
|
||||
|
||||
if ! cmd_exists virtualenv; then
|
||||
$SUDO pip install virtualenv
|
||||
fi
|
||||
@@ -1,11 +0,0 @@
|
||||
# install daemon requirements (created by build script. see build.sh, build.ps1)
|
||||
-r requirements_base.txt
|
||||
|
||||
# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
|
||||
file:../.
|
||||
|
||||
# install other build requirements
|
||||
PyInstaller==3.2.1
|
||||
requests[security]==2.13.0
|
||||
uritemplate==3.0.0
|
||||
boto3==1.4.4
|
||||
@@ -1,29 +0,0 @@
|
||||
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
|
||||
|
||||
import os.path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
build = get_build()
|
||||
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f:
|
||||
f.write("BUILD = '{}'\n".format(build))
|
||||
|
||||
|
||||
def get_build():
|
||||
try:
|
||||
tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip()
|
||||
if re.match('v\d+\.\d+\.\d+rc\d+', tag):
|
||||
return 'rc'
|
||||
else:
|
||||
return 'release'
|
||||
except subprocess.CalledProcessError:
|
||||
# if the build doesn't have a tag
|
||||
return 'qa'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,143 +0,0 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import github
|
||||
import uritemplate
|
||||
import boto3
|
||||
|
||||
|
||||
def main():
|
||||
upload_to_github_if_tagged('lbryio/lbry')
|
||||
upload_to_s3('daemon')
|
||||
|
||||
|
||||
def get_asset_filename():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
return glob.glob(this_dir + '/dist/*.zip')[0]
|
||||
|
||||
|
||||
def upload_to_s3(folder):
|
||||
tag = subprocess.check_output(['git', 'describe', '--always', '--abbrev=8', 'HEAD']).strip()
|
||||
commit_date = subprocess.check_output([
|
||||
'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
bucket = 'releases.lbry.io'
|
||||
key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
|
||||
|
||||
print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
|
||||
|
||||
if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
|
||||
print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
|
||||
return 1
|
||||
|
||||
s3 = boto3.resource(
|
||||
's3',
|
||||
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
|
||||
config=boto3.session.Config(signature_version='s3v4')
|
||||
)
|
||||
s3.meta.client.upload_file(asset_path, bucket, key)
|
||||
|
||||
|
||||
def upload_to_github_if_tagged(repo_name):
|
||||
try:
|
||||
current_tag = subprocess.check_output(
|
||||
['git', 'describe', '--exact-match', 'HEAD']).strip()
|
||||
except subprocess.CalledProcessError:
|
||||
print 'Not uploading to GitHub as we are not currently on a tag'
|
||||
return 1
|
||||
|
||||
print "Current tag: " + current_tag
|
||||
|
||||
if 'GH_TOKEN' not in os.environ:
|
||||
print 'Must set GH_TOKEN in order to publish assets to a release'
|
||||
return 1
|
||||
|
||||
gh_token = os.environ['GH_TOKEN']
|
||||
auth = github.Github(gh_token)
|
||||
repo = auth.get_repo(repo_name)
|
||||
|
||||
if not check_repo_has_tag(repo, current_tag):
|
||||
print 'Tag {} is not in repo {}'.format(current_tag, repo)
|
||||
# TODO: maybe this should be an error
|
||||
return 1
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
print "Uploading " + asset_path + " to Github tag " + current_tag
|
||||
release = get_github_release(repo, current_tag)
|
||||
upload_asset_to_github(release, asset_path, gh_token)
|
||||
|
||||
|
||||
def check_repo_has_tag(repo, target_tag):
|
||||
tags = repo.get_tags().get_page(0)
|
||||
for tag in tags:
|
||||
if tag.name == target_tag:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_github_release(repo, current_tag):
|
||||
for release in repo.get_releases():
|
||||
if release.tag_name == current_tag:
|
||||
return release
|
||||
raise Exception('No release for {} was found'.format(current_tag))
|
||||
|
||||
|
||||
def upload_asset_to_github(release, asset_to_upload, token):
|
||||
basename = os.path.basename(asset_to_upload)
|
||||
for asset in release.raw_data['assets']:
|
||||
if asset['name'] == basename:
|
||||
print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
|
||||
return
|
||||
|
||||
upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
|
||||
count = 0
|
||||
while count < 10:
|
||||
try:
|
||||
output = _curl_uploader(upload_uri, asset_to_upload, token)
|
||||
if 'errors' in output:
|
||||
raise Exception(output)
|
||||
else:
|
||||
print 'Successfully uploaded to {}'.format(output['browser_download_url'])
|
||||
except Exception:
|
||||
print 'Failed uploading on attempt {}'.format(count + 1)
|
||||
count += 1
|
||||
|
||||
|
||||
def _curl_uploader(upload_uri, asset_to_upload, token):
|
||||
# using requests.post fails miserably with SSL EPIPE errors. I spent
|
||||
# half a day trying to debug before deciding to switch to curl.
|
||||
#
|
||||
# TODO: actually set the content type
|
||||
print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
|
||||
cmd = [
|
||||
'curl',
|
||||
'-sS',
|
||||
'-X', 'POST',
|
||||
'-u', ':{}'.format(os.environ['GH_TOKEN']),
|
||||
'--header', 'Content-Type: application/octet-stream',
|
||||
'--data-binary', '@-',
|
||||
upload_uri
|
||||
]
|
||||
# '-d', '{"some_key": "some_value"}',
|
||||
print 'Calling curl:'
|
||||
print cmd
|
||||
print
|
||||
with open(asset_to_upload, 'rb') as fp:
|
||||
p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||
stdout, stderr = p.communicate()
|
||||
print 'curl return code:', p.returncode
|
||||
if stderr:
|
||||
print 'stderr output from curl:'
|
||||
print stderr
|
||||
print 'stdout from curl:'
|
||||
print stdout
|
||||
return json.loads(stdout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,29 +0,0 @@
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
|
||||
def main():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
tag = subprocess.check_output(['git', 'describe']).strip()
|
||||
zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
|
||||
full_filename = os.path.join(this_dir, 'dist', zipfilename)
|
||||
executables = ['lbrynet-daemon', 'lbrynet-cli', 'lbrynet-console']
|
||||
ext = '.exe' if platform.system() == 'Windows' else ''
|
||||
with zipfile.ZipFile(full_filename, 'w') as myzip:
|
||||
for executable in executables:
|
||||
myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext)
|
||||
|
||||
|
||||
def get_system_label():
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
return 'macos'
|
||||
else:
|
||||
return system.lower()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -100,7 +100,7 @@
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
|
||||
|
||||
</a>
|
||||
</div>
|
||||
@@ -196,7 +196,7 @@
|
||||
<label class="md-nav__title md-nav__title--site" for="drawer">
|
||||
<span class="md-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
|
||||
|
||||
</span>
|
||||
LBRY
|
||||
@@ -304,4 +304,4 @@
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -1409,7 +1409,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1454,7 +1454,7 @@ Returns:
|
||||
'is_spent': (bool) true if claim is abandoned, false otherwise
|
||||
'name': (str) name of the claim
|
||||
'permanent_url': (str) permanent url of the claim,
|
||||
'txid': (str) txid of the cliam
|
||||
'txid': (str) txid of the claim
|
||||
'nout': (int) nout of the claim
|
||||
'value': (str) value of the claim
|
||||
},
|
||||
@@ -1579,12 +1579,12 @@ Usage:
|
||||
|
||||
|
||||
Options:
|
||||
--a_arg : a arg
|
||||
--b_arg : b arg
|
||||
--pos_arg=<pos_arg> : pos arg
|
||||
--pos_args=<pos_args> : pos args
|
||||
--pos_arg2=<pos_arg2> : pos arg 2
|
||||
--pos_arg3=<pos_arg3> : pos arg 3
|
||||
--a_arg : (bool) a arg
|
||||
--b_arg : (bool) b arg
|
||||
--pos_arg=<pos_arg> : (int) pos arg
|
||||
--pos_args=<pos_args> : (int) pos args
|
||||
--pos_arg2=<pos_arg2> : (int) pos arg 2
|
||||
--pos_arg3=<pos_arg3> : (int) pos arg 3
|
||||
|
||||
Returns:
|
||||
pos args
|
||||
@@ -1828,14 +1828,16 @@ Returns:
|
||||
</code></pre>
|
||||
|
||||
<h2 id="peer_ping">peer_ping</h2>
|
||||
<pre><code class="text">Find and ping a peer by node id
|
||||
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
|
||||
if not provided the peer is located first.
|
||||
|
||||
Usage:
|
||||
peer_ping (<node_id> | --node_id=<node_id>)
|
||||
peer_ping (<node_id> | --node_id=<node_id>) [<address> | --address=<address>] [<port> | --port=<port>]
|
||||
|
||||
|
||||
Options:
|
||||
None
|
||||
--address=<address> : (str) ip address of the peer
|
||||
--port=<port> : (int) udp port of the peer
|
||||
|
||||
Returns:
|
||||
(str) pong, or {'error': <error message>} if an error is encountered
|
||||
@@ -1893,7 +1895,7 @@ Options:
|
||||
--license_url=<license_url> : (str) publication license url
|
||||
--thumbnail=<thumbnail> : (str) thumbnail url
|
||||
--preview=<preview> : (str) preview url
|
||||
--nsfw=<nsfw> : (bool) title of the publication
|
||||
--nsfw=<nsfw> : (bool) whether the content is nsfw
|
||||
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
|
||||
@@ -1901,7 +1903,7 @@ Options:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
--claim_address=<claim_address> : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
@@ -1952,7 +1954,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1977,7 +1978,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2110,41 +2110,62 @@ Returns:
|
||||
<pre><code class="text">Get daemon status
|
||||
|
||||
Usage:
|
||||
status [--session_status]
|
||||
status
|
||||
|
||||
|
||||
Options:
|
||||
--session_status : (bool) include session status in results
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) lbrynet-daemon status
|
||||
{
|
||||
'lbry_id': lbry peer id, base58,
|
||||
'installation_id': installation id, base58,
|
||||
'is_running': bool,
|
||||
'installation_id': (str) installation id - base58,
|
||||
'is_running': (bool),
|
||||
'is_first_run': bool,
|
||||
'startup_status': {
|
||||
'code': status code,
|
||||
'message': status message
|
||||
'skipped_components': (list) [names of skipped components (str)],
|
||||
'startup_status': { Does not include components which have been skipped
|
||||
'database': (bool),
|
||||
'wallet': (bool),
|
||||
'session': (bool),
|
||||
'dht': (bool),
|
||||
'hash_announcer': (bool),
|
||||
'stream_identifier': (bool),
|
||||
'file_manager': (bool),
|
||||
'blob_manager': (bool),
|
||||
'blockchain_headers': (bool),
|
||||
'peer_protocol_server': (bool),
|
||||
'reflector': (bool),
|
||||
'upnp': (bool),
|
||||
'exchange_rate_manager': (bool),
|
||||
},
|
||||
'connection_status': {
|
||||
'code': connection status code,
|
||||
'message': connection status message
|
||||
'code': (str) connection status code,
|
||||
'message': (str) connection status message
|
||||
},
|
||||
'blockchain_status': {
|
||||
'blocks': local blockchain height,
|
||||
'blocks_behind': remote_height - local_height,
|
||||
'best_blockhash': block hash of most recent block,
|
||||
'blockchain_headers': {
|
||||
'downloading_headers': (bool),
|
||||
'download_progress': (float) 0-100.0
|
||||
},
|
||||
'wallet_is_encrypted': bool,
|
||||
|
||||
If given the session status option:
|
||||
'session_status': {
|
||||
'managed_blobs': count of blobs in the blob manager,
|
||||
'managed_streams': count of streams in the file manager
|
||||
'announce_queue_size': number of blobs currently queued to be announced
|
||||
'should_announce_blobs': number of blobs that should be announced
|
||||
}
|
||||
'wallet': {
|
||||
'blocks': (int) local blockchain height,
|
||||
'blocks_behind': (int) remote_height - local_height,
|
||||
'best_blockhash': (str) block hash of most recent block,
|
||||
'is_encrypted': (bool),
|
||||
'is_locked': (bool),
|
||||
},
|
||||
'dht': {
|
||||
'node_id': (str) lbry dht node id - hex encoded,
|
||||
'peers_in_routing_table': (int) the number of peers in the routing table,
|
||||
},
|
||||
'blob_manager': {
|
||||
'finished_blobs': (int) number of finished blobs in the blob manager,
|
||||
},
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
},
|
||||
'file_manager': {
|
||||
'managed_files': (int) count of files in the file manager,
|
||||
}
|
||||
}
|
||||
</code></pre>
|
||||
|
||||
@@ -2452,9 +2473,9 @@ Usage:
|
||||
|
||||
|
||||
Options:
|
||||
--amount=<amount> : (float) amount of credit to send
|
||||
--address=<address> : (str) address to send credits to
|
||||
--claim_id=<claim_id> : (float) claim_id of the claim to send to tip to
|
||||
--amount=<amount> : (Decimal) amount of credit to send
|
||||
--address=<address> : (str) address to send credits to
|
||||
--claim_id=<claim_id> : (str) claim_id of the claim to send to tip to
|
||||
|
||||
Returns:
|
||||
If sending to an address:
|
||||
@@ -2563,4 +2584,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -1339,7 +1339,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1380,7 +1380,7 @@ Returns:
|
||||
'is_spent': (bool) true if claim is abandoned, false otherwise
|
||||
'name': (str) name of the claim
|
||||
'permanent_url': (str) permanent url of the claim,
|
||||
'txid': (str) txid of the cliam
|
||||
'txid': (str) txid of the claim
|
||||
'nout': (int) nout of the claim
|
||||
'value': (str) value of the claim
|
||||
},
|
||||
@@ -1479,12 +1479,12 @@ Returns:
|
||||
<h2 id="cli_test_command">cli_test_command</h2>
|
||||
<pre><code class="text">This command is only for testing the CLI argument parsing
|
||||
Args:
|
||||
'a_arg' : a arg
|
||||
'b_arg' : b arg
|
||||
'pos_arg' (required) : pos arg
|
||||
'pos_args' : pos args
|
||||
'pos_arg2' : pos arg 2
|
||||
'pos_arg3' : pos arg 3
|
||||
'a_arg' : (bool) a arg
|
||||
'b_arg' : (bool) b arg
|
||||
'pos_arg' (required) : (int) pos arg
|
||||
'pos_args' : (int) pos args
|
||||
'pos_arg2' : (int) pos arg 2
|
||||
'pos_arg3' : (int) pos arg 3
|
||||
|
||||
Returns:
|
||||
pos args
|
||||
@@ -1682,10 +1682,12 @@ Returns:
|
||||
</code></pre>
|
||||
|
||||
<h2 id="peer_ping">peer_ping</h2>
|
||||
<pre><code class="text">Find and ping a peer by node id
|
||||
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
|
||||
if not provided the peer is located first.
|
||||
|
||||
Args:
|
||||
None
|
||||
'address' : (str) ip address of the peer
|
||||
'port' : (int) udp port of the peer
|
||||
|
||||
Returns:
|
||||
(str) pong, or {'error': <error message>} if an error is encountered
|
||||
@@ -1733,7 +1735,7 @@ Args:
|
||||
'license_url' : (str) publication license url
|
||||
'thumbnail' : (str) thumbnail url
|
||||
'preview' : (str) preview url
|
||||
'nsfw' : (bool) title of the publication
|
||||
'nsfw' : (bool) whether the content is nsfw
|
||||
'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
|
||||
'channel_name' : (str) name of the publisher channel name in the wallet
|
||||
'channel_id' : (str) claim id of the publisher channel, does not check
|
||||
@@ -1741,7 +1743,7 @@ Args:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
'claim_address' : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
@@ -1784,7 +1786,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1809,7 +1810,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1913,37 +1913,58 @@ Returns:
|
||||
<pre><code class="text">Get daemon status
|
||||
|
||||
Args:
|
||||
'session_status' : (bool) include session status in results
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) lbrynet-daemon status
|
||||
{
|
||||
'lbry_id': lbry peer id, base58,
|
||||
'installation_id': installation id, base58,
|
||||
'is_running': bool,
|
||||
'installation_id': (str) installation id - base58,
|
||||
'is_running': (bool),
|
||||
'is_first_run': bool,
|
||||
'startup_status': {
|
||||
'code': status code,
|
||||
'message': status message
|
||||
'skipped_components': (list) [names of skipped components (str)],
|
||||
'startup_status': { Does not include components which have been skipped
|
||||
'database': (bool),
|
||||
'wallet': (bool),
|
||||
'session': (bool),
|
||||
'dht': (bool),
|
||||
'hash_announcer': (bool),
|
||||
'stream_identifier': (bool),
|
||||
'file_manager': (bool),
|
||||
'blob_manager': (bool),
|
||||
'blockchain_headers': (bool),
|
||||
'peer_protocol_server': (bool),
|
||||
'reflector': (bool),
|
||||
'upnp': (bool),
|
||||
'exchange_rate_manager': (bool),
|
||||
},
|
||||
'connection_status': {
|
||||
'code': connection status code,
|
||||
'message': connection status message
|
||||
'code': (str) connection status code,
|
||||
'message': (str) connection status message
|
||||
},
|
||||
'blockchain_status': {
|
||||
'blocks': local blockchain height,
|
||||
'blocks_behind': remote_height - local_height,
|
||||
'best_blockhash': block hash of most recent block,
|
||||
'blockchain_headers': {
|
||||
'downloading_headers': (bool),
|
||||
'download_progress': (float) 0-100.0
|
||||
},
|
||||
'wallet_is_encrypted': bool,
|
||||
|
||||
If given the session status option:
|
||||
'session_status': {
|
||||
'managed_blobs': count of blobs in the blob manager,
|
||||
'managed_streams': count of streams in the file manager
|
||||
'announce_queue_size': number of blobs currently queued to be announced
|
||||
'should_announce_blobs': number of blobs that should be announced
|
||||
}
|
||||
'wallet': {
|
||||
'blocks': (int) local blockchain height,
|
||||
'blocks_behind': (int) remote_height - local_height,
|
||||
'best_blockhash': (str) block hash of most recent block,
|
||||
'is_encrypted': (bool),
|
||||
'is_locked': (bool),
|
||||
},
|
||||
'dht': {
|
||||
'node_id': (str) lbry dht node id - hex encoded,
|
||||
'peers_in_routing_table': (int) the number of peers in the routing table,
|
||||
},
|
||||
'blob_manager': {
|
||||
'finished_blobs': (int) number of finished blobs in the blob manager,
|
||||
},
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
},
|
||||
'file_manager': {
|
||||
'managed_files': (int) count of files in the file manager,
|
||||
}
|
||||
}
|
||||
</code></pre>
|
||||
|
||||
@@ -2186,9 +2207,9 @@ to the owner of a claim specified by uri. A tip is a claim support where the rec
|
||||
of the support is the claim address for the claim being supported.
|
||||
|
||||
Args:
|
||||
'amount' (required) : (float) amount of credit to send
|
||||
'address' (required) : (str) address to send credits to
|
||||
'claim_id' (required) : (float) claim_id of the claim to send to tip to
|
||||
'amount' (required) : (Decimal) amount of credit to send
|
||||
'address' (required) : (str) address to send credits to
|
||||
'claim_id' (required) : (str) claim_id of the claim to send to tip to
|
||||
|
||||
Returns:
|
||||
If sending to an address:
|
||||
@@ -2289,4 +2310,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -2,7 +2,7 @@
|
||||
"docs": [
|
||||
{
|
||||
"location": "/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"title": "API"
|
||||
},
|
||||
{
|
||||
@@ -87,7 +87,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#claim_list_mine",
|
||||
"text": "List my name claims\n\nArgs:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the cliam\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
|
||||
"text": "List my name claims\n\nArgs:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
|
||||
"title": "claim_list_mine"
|
||||
},
|
||||
{
|
||||
@@ -112,7 +112,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#cli_test_command",
|
||||
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : a arg\n 'b_arg' : b arg\n 'pos_arg' (required) : pos arg\n 'pos_args' : pos args\n 'pos_arg2' : pos arg 2\n 'pos_arg3' : pos arg 3\n\nReturns:\n pos args",
|
||||
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : (bool) a arg\n 'b_arg' : (bool) b arg\n 'pos_arg' (required) : (int) pos arg\n 'pos_args' : (int) pos args\n 'pos_arg2' : (int) pos arg 2\n 'pos_arg3' : (int) pos arg 3\n\nReturns:\n pos args",
|
||||
"title": "cli_test_command"
|
||||
},
|
||||
{
|
||||
@@ -162,12 +162,12 @@
|
||||
},
|
||||
{
|
||||
"location": "/#peer_ping",
|
||||
"text": "Find and ping a peer by node id\n\nArgs:\n None\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nArgs:\n 'address' : (str) ip address of the peer\n 'port' : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"title": "peer_ping"
|
||||
},
|
||||
{
|
||||
"location": "/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) title of the publication\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
@@ -202,7 +202,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#status",
|
||||
"text": "Get daemon status\n\nArgs:\n 'session_status' : (bool) include session status in results\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n }",
|
||||
"text": "Get daemon status\n\nArgs:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
|
||||
"title": "status"
|
||||
},
|
||||
{
|
||||
@@ -277,7 +277,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#wallet_send",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (float) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (Decimal) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"title": "wallet_send"
|
||||
},
|
||||
{
|
||||
@@ -292,7 +292,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"title": "CLI"
|
||||
},
|
||||
{
|
||||
@@ -377,7 +377,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#claim_list_mine",
|
||||
"text": "List my name claims\n\nUsage:\n claim_list_mine\n\n\nOptions:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the cliam\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
|
||||
"text": "List my name claims\n\nUsage:\n claim_list_mine\n\n\nOptions:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
|
||||
"title": "claim_list_mine"
|
||||
},
|
||||
{
|
||||
@@ -402,7 +402,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#cli_test_command",
|
||||
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : a arg\n --b_arg : b arg\n --pos_arg= pos_arg : pos arg\n --pos_args= pos_args : pos args\n --pos_arg2= pos_arg2 : pos arg 2\n --pos_arg3= pos_arg3 : pos arg 3\n\nReturns:\n pos args",
|
||||
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : (bool) a arg\n --b_arg : (bool) b arg\n --pos_arg= pos_arg : (int) pos arg\n --pos_args= pos_args : (int) pos args\n --pos_arg2= pos_arg2 : (int) pos arg 2\n --pos_arg3= pos_arg3 : (int) pos arg 3\n\nReturns:\n pos args",
|
||||
"title": "cli_test_command"
|
||||
},
|
||||
{
|
||||
@@ -452,12 +452,12 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#peer_ping",
|
||||
"text": "Find and ping a peer by node id\n\nUsage:\n peer_ping ( node_id | --node_id= node_id )\n\n\nOptions:\n None\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nUsage:\n peer_ping ( node_id | --node_id= node_id ) [ address | --address= address ] [ port | --port= port ]\n\n\nOptions:\n --address= address : (str) ip address of the peer\n --port= port : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"title": "peer_ping"
|
||||
},
|
||||
{
|
||||
"location": "/cli/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) title of the publication\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
@@ -492,7 +492,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#status",
|
||||
"text": "Get daemon status\n\nUsage:\n status [--session_status]\n\n\nOptions:\n --session_status : (bool) include session status in results\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n }",
|
||||
"text": "Get daemon status\n\nUsage:\n status\n\n\nOptions:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
|
||||
"title": "status"
|
||||
},
|
||||
{
|
||||
@@ -567,7 +567,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#wallet_send",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (float) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (Decimal) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"title": "wallet_send"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# This is an example daemon_settings.yml file.
|
||||
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
|
||||
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.com:50001
|
||||
- lbryumx2.lbry.com:50001
|
||||
- lbryumx4.lbry.com:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: true
|
||||
|
||||
#components_to_skip:
|
||||
# - hash_announcer
|
||||
# - blob_server
|
||||
# - dht
|
||||
|
Before Width: | Height: | Size: 7.4 KiB After Width: | Height: | Size: 7.4 KiB |
|
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 19 KiB |
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 2.6 KiB |
|
Before Width: | Height: | Size: 6.1 KiB After Width: | Height: | Size: 6.1 KiB |
|
Before Width: | Height: | Size: 97 KiB After Width: | Height: | Size: 97 KiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 361 KiB After Width: | Height: | Size: 361 KiB |
|
Before Width: | Height: | Size: 5.3 KiB After Width: | Height: | Size: 5.3 KiB |
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 31 KiB After Width: | Height: | Size: 31 KiB |
@@ -1,6 +1,4 @@
|
||||
import logging
|
||||
|
||||
__version__ = "0.20.3rc2"
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.38.0rc6"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -1,274 +0,0 @@
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer, task
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import looping_call_manager, utils, system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
SERVER_STARTUP_ERROR = 'Server Startup Error'
|
||||
DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
self.session_id = session_id or conf.settings.get_session_id()
|
||||
self.is_started = False
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
api = Api.new_instance(enabled)
|
||||
return cls(api)
|
||||
|
||||
# Things We Track
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 60),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 300),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self._tracked_data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash,
|
||||
'error': error_name(error),
|
||||
'reason': error.message,
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
context = {
|
||||
'app': {
|
||||
'name': 'lbrynet',
|
||||
'version': platform['lbrynet_version'],
|
||||
'python_version': platform['python_version'],
|
||||
'build': platform['build'],
|
||||
'wallet': {
|
||||
'name': wallet,
|
||||
'version': platform['lbryum_version'] if wallet == conf.LBRYUM_WALLET else None
|
||||
},
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
'library': {
|
||||
'name': 'lbrynet-analytics',
|
||||
'version': '1.0.0'
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api(object):
|
||||
def __init__(self, cookies, url, write_key, enabled):
|
||||
self.cookies = cookies
|
||||
self.url = url
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
def update_cookies(response):
|
||||
self.cookies.update(response.cookies())
|
||||
return response
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
headers = {b"Connection": b"close"}
|
||||
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
|
||||
headers=headers, cookies=self.cookies)
|
||||
d.addCallback(update_cookies)
|
||||
return d
|
||||
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('Analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if enabled is None:
|
||||
enabled = conf.settings['share_usage_data']
|
||||
return cls(
|
||||
{},
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
|
||||
enabled,
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
import paths
|
||||
@@ -1,26 +0,0 @@
|
||||
# App root files dir
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
|
||||
def android_files_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base internal storage path
|
||||
def android_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base external (SD card, if present) storage path
|
||||
def android_external_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Internal device storage (private app folder)
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# External (app folder on SD card, if present) storage
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_external_storage_dir():
|
||||
return None
|
||||
@@ -1,4 +1,6 @@
|
||||
from blob_file import BlobFile
|
||||
from creator import BlobFileCreator
|
||||
from writer import HashBlobWriter
|
||||
from reader import HashBlobReader
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
@@ -1,209 +1,335 @@
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
import re
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
import contextlib
|
||||
from io import BytesIO
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
|
||||
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
|
||||
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
|
||||
|
||||
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.blob.reader import HashBlobReader
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
class BlobFile(object):
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
|
||||
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we already know the blob hash before hand (i.e., when downloading blobs)
|
||||
Also can be used for reading from blobs on the local filesystem
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
|
||||
digest = get_lbry_hash_obj()
|
||||
digest.update(encrypted)
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
|
||||
if len(data) != length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
|
||||
class AbstractBlob:
|
||||
"""
|
||||
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
|
||||
|
||||
def __str__(self):
|
||||
return self.blob_hash[:16]
|
||||
This class is non-io specific
|
||||
"""
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_hash',
|
||||
'length',
|
||||
'blob_completed_callback',
|
||||
'blob_directory',
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
]
|
||||
|
||||
def __repr__(self):
|
||||
return '<{}({})>'.format(self.__class__.__name__, str(self))
|
||||
|
||||
def __init__(self, blob_dir, blob_hash, length=None):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.writers = {} # {Peer: writer, finished_deferred}
|
||||
self._verified = False
|
||||
self.readers = 0
|
||||
self.blob_dir = blob_dir
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.blob_write_lock = defer.DeferredLock()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
self.set_length(os.path.getsize(self.file_path))
|
||||
# This assumes that the hash of the blob has already been
|
||||
# checked as part of the blob creation process. It might
|
||||
# be worth having a function that checks the actual hash;
|
||||
# its probably too expensive to have that check be part of
|
||||
# this call.
|
||||
self._verified = True
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
self.blob_directory = blob_directory
|
||||
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
"""
|
||||
open a blob file to be written by peer, supports concurrent
|
||||
writers, as long as they are from differnt peers.
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
|
||||
returns tuple of (writer, finished_deferred)
|
||||
def __del__(self):
|
||||
if self.writers or self.readers:
|
||||
log.warning("%s not closed before being garbage collected", self.blob_hash)
|
||||
self.close()
|
||||
|
||||
writer - a file like object with a write() function, close() when finished
|
||||
finished_deferred - deferred that is fired when write is finished and returns
|
||||
a instance of itself as HashBlob
|
||||
"""
|
||||
if not peer in self.writers:
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(self.get_length, self.writer_finished)
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return (writer, finished_deferred)
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
raise NotImplementedError()
|
||||
|
||||
def open_for_reading(self):
|
||||
"""
|
||||
open blob for reading
|
||||
@contextlib.contextmanager
|
||||
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
|
||||
with self._reader_context() as reader:
|
||||
try:
|
||||
self.readers.append(reader)
|
||||
yield reader
|
||||
finally:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
returns a file like object that can be read() from, and closed() when
|
||||
finished
|
||||
"""
|
||||
if self._verified is True:
|
||||
f = open(self.file_path, 'rb')
|
||||
reader = HashBlobReader(f, self.reader_finished)
|
||||
self.readers += 1
|
||||
return reader
|
||||
return None
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
raise NotImplementedError()
|
||||
|
||||
def delete(self):
|
||||
"""
|
||||
delete blob file from file system, prevent deletion
|
||||
if a blob is being read from or written to
|
||||
|
||||
returns a deferred that firesback when delete is completed
|
||||
"""
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
self.saved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
"""
|
||||
Protect verified from being modified by other classes.
|
||||
verified is True if a write to a blob has completed succesfully,
|
||||
or a blob has been read to have the same length as specified
|
||||
in init
|
||||
"""
|
||||
return self._verified
|
||||
|
||||
def set_length(self, length):
|
||||
def set_length(self, length) -> None:
|
||||
if self.length is not None and length == self.length:
|
||||
return True
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
self.length = length
|
||||
return True
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
|
||||
self.length, length)
|
||||
return False
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self):
|
||||
def get_length(self) -> typing.Optional[int]:
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self):
|
||||
return self.verified
|
||||
def get_is_verified(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_downloading(self):
|
||||
if self.writers:
|
||||
return True
|
||||
return False
|
||||
def is_readable(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def reader_finished(self, reader):
|
||||
self.readers -= 1
|
||||
return defer.succeed(True)
|
||||
def is_writeable(self) -> bool:
|
||||
return not self.writing.is_set()
|
||||
|
||||
def writer_finished(self, writer, err=None):
|
||||
def fire_finished_deferred():
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.callback(self)
|
||||
return True
|
||||
log.warning(
|
||||
"Somehow, the writer that was accepted as being valid was already removed: %s",
|
||||
writer)
|
||||
return False
|
||||
def write_blob(self, blob_bytes: bytes):
|
||||
if not self.is_writeable():
|
||||
raise OSError("cannot open blob for writing")
|
||||
try:
|
||||
self.writing.set()
|
||||
self._write_blob(blob_bytes)
|
||||
finally:
|
||||
self.writing.clear()
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.errback(err)
|
||||
def close(self):
|
||||
while self.writers:
|
||||
peer, writer = self.writers.popitem()
|
||||
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
|
||||
writer.finished.cancel()
|
||||
while self.readers:
|
||||
reader = self.readers.pop()
|
||||
if reader:
|
||||
reader.close()
|
||||
|
||||
def cancel_other_downloads():
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
w.close()
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.verified.clear()
|
||||
self.length = None
|
||||
|
||||
if err is None:
|
||||
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
|
||||
if self._verified is False:
|
||||
d = self.save_verified_blob(writer)
|
||||
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
|
||||
d.addCallback(lambda _: cancel_other_downloads())
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
fire_finished_deferred()
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with self.reader_context() as reader:
|
||||
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
|
||||
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers[(peer_address, peer_port)] = writer
|
||||
|
||||
def remove_writer(_):
|
||||
if (peer_address, peer_port) in self.writers:
|
||||
del self.writers[(peer_address, peer_port)]
|
||||
|
||||
fut.add_done_callback(remove_writer)
|
||||
|
||||
def writer_finished_callback(finished: asyncio.Future):
|
||||
try:
|
||||
err = finished.exception()
|
||||
if err:
|
||||
raise err
|
||||
verified_bytes = finished.result()
|
||||
while self.writers:
|
||||
_, other = self.writers.popitem()
|
||||
if other is not writer:
|
||||
other.close_handle()
|
||||
self.save_verified_blob(verified_bytes)
|
||||
except (InvalidBlobHashError, InvalidDataError) as error:
|
||||
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
|
||||
fut.add_done_callback(writer_finished_callback)
|
||||
return writer
|
||||
|
||||
|
||||
class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError("cannot open blob for reading")
|
||||
try:
|
||||
yield self._verified_bytes
|
||||
finally:
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
return super().delete()
|
||||
|
||||
def __del__(self):
|
||||
super().__del__()
|
||||
if self._verified_bytes:
|
||||
self.delete()
|
||||
|
||||
|
||||
class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
if self.file_exists:
|
||||
file_size = int(os.stat(self.file_path).st_size)
|
||||
if length and length != file_size:
|
||||
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
|
||||
self.delete()
|
||||
else:
|
||||
if writer.len_so_far != self.length:
|
||||
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
|
||||
else:
|
||||
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(None)
|
||||
else:
|
||||
errback_finished_deferred(err)
|
||||
d = defer.succeed(None)
|
||||
d.addBoth(lambda _: writer.close_handle())
|
||||
return d
|
||||
self.length = file_size
|
||||
self.verified.set()
|
||||
|
||||
def save_verified_blob(self, writer):
|
||||
# we cannot have multiple _save_verified_blob interrupting
|
||||
# each other, can happen since startProducing is a deferred
|
||||
return self.blob_write_lock.run(self._save_verified_blob, writer)
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _save_verified_blob(self, writer):
|
||||
if self.saved_verified_blob is False:
|
||||
writer.write_handle.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(writer.write_handle)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self.saved_verified_blob = True
|
||||
defer.returnValue(True)
|
||||
else:
|
||||
raise DownloadCanceledError()
|
||||
def is_writeable(self) -> bool:
|
||||
return super().is_writeable() and not os.path.isfile(self.file_path)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
return super().get_blob_writer(peer_address, peer_port)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
handle = open(self.file_path, 'rb')
|
||||
try:
|
||||
yield handle
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
__slots__ = [
|
||||
'blob_hash',
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.iv = iv
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
'length': self.length,
|
||||
'blob_num': self.blob_num,
|
||||
'iv': self.iv,
|
||||
}
|
||||
if self.blob_hash: # non-terminator blobs have a blob hash
|
||||
d['blob_hash'] = self.blob_hash
|
||||
return d
|
||||
@@ -0,0 +1,135 @@
|
||||
import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbrynet.utils import LRUCache
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
|
||||
blob_dir - directory where blobs are stored
|
||||
storage - SQLiteStorage object
|
||||
"""
|
||||
self.loop = loop
|
||||
self.blob_dir = blob_dir
|
||||
self.storage = storage
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs:
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
else:
|
||||
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise ValueError(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
return self.blobs[blob_hash].get_is_verified()
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
return self._get_blob(blob_hash, length).get_is_verified()
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
if not self.blob_dir:
|
||||
return set()
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
self.connection_manager.start()
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
self.connection_manager.stop()
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
|
||||
for blob_hash in blob_hashes:
|
||||
self.delete_blob(blob_hash)
|
||||
|
||||
if delete_from_db:
|
||||
await self.storage.delete_blobs_from_db(blob_hashes)
|
||||
@@ -1,51 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileCreator(object):
|
||||
"""
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we do not know the blob hash beforehand (i.e, when creating
|
||||
a new stream)
|
||||
"""
|
||||
def __init__(self, blob_dir):
|
||||
self.blob_dir = blob_dir
|
||||
self.buffer = BytesIO()
|
||||
self._is_open = True
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.blob_hash = None
|
||||
self.length = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def close(self):
|
||||
self.length = self.len_so_far
|
||||
self.blob_hash = self._hashsum.hexdigest()
|
||||
if self.blob_hash and self._is_open and self.length > 0:
|
||||
# do not save 0 length files (empty tail blob in streams)
|
||||
# or if its been closed already
|
||||
self.buffer.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(self.buffer)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self._is_open = False
|
||||
if self.length > 0:
|
||||
defer.returnValue(self.blob_hash)
|
||||
else:
|
||||
# 0 length files (empty tail blob in streams )
|
||||
# must return None as their blob_hash for
|
||||
# it to be saved properly by EncryptedFileMetadataManagers
|
||||
defer.returnValue(None)
|
||||
|
||||
def write(self, data):
|
||||
if not self._is_open:
|
||||
raise IOError
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
self.buffer.write(data)
|
||||
@@ -1,32 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
"""
|
||||
This is a file like reader class that supports
|
||||
read(size) and close()
|
||||
"""
|
||||
def __init__(self, read_handle, finished_cb):
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
self.read_handle = read_handle
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warn("Garbage collection was called, but reader for %s was not closed yet",
|
||||
self.read_handle.name)
|
||||
self.close()
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.read_handle.read(size)
|
||||
|
||||
def close(self):
|
||||
# if we've already closed and called finished_cb, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
self.read_handle.close()
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
|
||||
|
||||
@@ -1,58 +1,68 @@
|
||||
import typing
|
||||
import logging
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobWriter(object):
|
||||
def __init__(self, length_getter, finished_cb):
|
||||
self.write_handle = BytesIO()
|
||||
self.length_getter = length_getter
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
class HashBlobWriter:
|
||||
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
|
||||
finished: asyncio.Future):
|
||||
self.expected_blob_hash = expected_blob_hash
|
||||
self.get_length = get_length
|
||||
self.buffer = BytesIO()
|
||||
self.finished = finished
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warn("Garbage collection was called, but writer was not closed yet")
|
||||
self.close()
|
||||
if self.buffer is not None:
|
||||
log.warning("Garbage collection was called, but writer was not closed yet")
|
||||
self.close_handle()
|
||||
|
||||
@property
|
||||
def blob_hash(self):
|
||||
def calculate_blob_hash(self) -> str:
|
||||
return self._hashsum.hexdigest()
|
||||
|
||||
def write(self, data):
|
||||
if self.write_handle is None:
|
||||
log.exception("writer has already been closed")
|
||||
def closed(self):
|
||||
return self.buffer is None or self.buffer.closed
|
||||
|
||||
def write(self, data: bytes):
|
||||
expected_length = self.get_length()
|
||||
if not expected_length:
|
||||
raise IOError("unknown blob length")
|
||||
if self.buffer is None:
|
||||
log.warning("writer has already been closed")
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
return
|
||||
raise IOError('I/O operation on closed file')
|
||||
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(
|
||||
self,
|
||||
Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (self.len_so_far,
|
||||
self.length_getter()))))
|
||||
else:
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
if self.len_so_far > expected_length:
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
self.close_handle()
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
self.close_handle()
|
||||
|
||||
def close_handle(self):
|
||||
if self.write_handle is not None:
|
||||
self.write_handle.close()
|
||||
self.write_handle = None
|
||||
|
||||
def close(self, reason=None):
|
||||
# if we've already called finished_cb because we either finished writing
|
||||
# or closed already, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
if reason is None:
|
||||
reason = Failure(DownloadCanceledError())
|
||||
self.finished_cb_d = self.finished_cb(self, reason)
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
@@ -0,0 +1,245 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
from lbrynet.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.connection_manager = connection_manager
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: typing.Optional[asyncio.Future] = None
|
||||
self.buf = b''
|
||||
|
||||
# this is here to handle the race when the downloader is closed right as response_fut gets a result
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.connection_manager:
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
# assert self.peer_address is not None
|
||||
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
|
||||
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if not self._response_fut:
|
||||
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
|
||||
return self.close()
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
response = BlobResponse.deserialize(self.buf + data)
|
||||
if not response.responses and not self._response_fut.done():
|
||||
self.buf += data
|
||||
return
|
||||
else:
|
||||
self.buf = b''
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
|
||||
def _write(self, data: bytes):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
else:
|
||||
data = data
|
||||
self._blob_bytes_received += len(data)
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except (asyncio.TimeoutError) as err: # TODO: is this needed?
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
if self.connection_manager:
|
||||
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if self.closed.is_set():
|
||||
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
|
||||
log.warning(msg)
|
||||
raise asyncio.CancelledError(msg)
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return self._blob_bytes_received, self.close()
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
log.info(msg)
|
||||
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
|
||||
return self._blob_bytes_received, self.transport
|
||||
except asyncio.TimeoutError:
|
||||
return self._blob_bytes_received, self.close()
|
||||
except (InvalidBlobHashError, InvalidDataError):
|
||||
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
def close(self):
|
||||
self.closed.set()
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
self.closed.clear()
|
||||
blob_hash = blob.blob_hash
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self._blob_bytes_received = 0
|
||||
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
self.close()
|
||||
return self._blob_bytes_received, None
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
finally:
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
addr = transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr[0], addr[1]
|
||||
self.transport = transport
|
||||
if self.connection_manager:
|
||||
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
if self.connection_manager:
|
||||
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
|
||||
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(
|
||||
loop, blob_download_timeout, connection_manager
|
||||
)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.transport = connected_transport
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
try:
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return 0, None
|
||||
@@ -0,0 +1,130 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import cache_concurrent
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
|
||||
connection_manager=self.blob_manager.connection_manager
|
||||
|
||||
)
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = self.failures.get(peer, 0) + 1
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = 0
|
||||
self.connections[peer] = transport
|
||||
elapsed = self.loop.time() - start
|
||||
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
|
||||
|
||||
async def new_peer_or_finished(self):
|
||||
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
|
||||
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
|
||||
def cleanup_active(self):
|
||||
if not self.active_connections and not self.connections:
|
||||
self.clearbanned()
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
self.ignored = dict((
|
||||
(peer, when) for (peer, when) in self.ignored.items()
|
||||
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
|
||||
))
|
||||
|
||||
@cache_concurrent
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
|
||||
connection_id: int = 0) -> 'AbstractBlob':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
try:
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set()
|
||||
while not self.peer_queue.empty():
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
blob.close()
|
||||
|
||||
def close(self):
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
self.is_running.clear()
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
|
||||
blob_hash: str) -> 'AbstractBlob':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
downloader.close()
|
||||
@@ -0,0 +1,283 @@
|
||||
import typing
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobMessage:
|
||||
key = ''
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class BlobPriceRequest(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobPriceResponse(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
rate_accepted = 'RATE_ACCEPTED'
|
||||
rate_too_low = 'RATE_TOO_LOW'
|
||||
rate_unset = 'RATE_UNSET'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
|
||||
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
|
||||
raise ValueError(blob_data_payment_rate)
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityRequest(BlobMessage):
|
||||
key = 'requested_blobs'
|
||||
|
||||
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
|
||||
**kwargs) -> None:
|
||||
assert len(requested_blobs)
|
||||
self.requested_blobs = requested_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blobs,
|
||||
'lbrycrd_address': self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityResponse(BlobMessage):
|
||||
key = 'available_blobs'
|
||||
|
||||
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
|
||||
**kwargs) -> None:
|
||||
self.available_blobs = available_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
self.key: self.available_blobs
|
||||
}
|
||||
if self.lbrycrd_address:
|
||||
d['lbrycrd_address'] = self.lbrycrd_address
|
||||
return d
|
||||
|
||||
|
||||
class BlobDownloadRequest(BlobMessage):
|
||||
key = 'requested_blob'
|
||||
|
||||
def __init__(self, requested_blob: str, **kwargs) -> None:
|
||||
self.requested_blob = requested_blob
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blob
|
||||
}
|
||||
|
||||
|
||||
class BlobDownloadResponse(BlobMessage):
|
||||
key = 'incoming_blob'
|
||||
|
||||
def __init__(self, **response: typing.Dict) -> None:
|
||||
incoming_blob = response[self.key]
|
||||
self.error = None
|
||||
self.incoming_blob = None
|
||||
if 'error' in incoming_blob:
|
||||
self.error = incoming_blob['error']
|
||||
else:
|
||||
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
|
||||
self.length = None if not self.incoming_blob else self.incoming_blob['length']
|
||||
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key if not self.error else 'error': self.incoming_blob or self.error,
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressRequest(BlobMessage):
|
||||
key = 'lbrycrd_address'
|
||||
|
||||
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
|
||||
pass
|
||||
|
||||
|
||||
class BlobErrorResponse(BlobMessage):
|
||||
key = 'error'
|
||||
|
||||
def __init__(self, error: str, **kwargs) -> None:
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.error
|
||||
}
|
||||
|
||||
|
||||
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest]
|
||||
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse]
|
||||
|
||||
|
||||
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
|
||||
# scenarios:
|
||||
# <json>
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
return None, response_msg
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
except ValueError:
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
def __init__(self, requests: typing.List[blob_request_types]) -> None:
|
||||
self.requests = requests
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for request in self.requests:
|
||||
d.update(request.to_dict())
|
||||
return d
|
||||
|
||||
def _get_request(self, request_type: blob_request_types):
|
||||
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
|
||||
if request:
|
||||
return request[0]
|
||||
|
||||
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
|
||||
response = self._get_request(BlobAvailabilityRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
|
||||
response = self._get_request(BlobPriceRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
|
||||
response = self._get_request(BlobDownloadRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
|
||||
response = self._get_request(BlobPaymentAddressRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobRequest':
|
||||
request = json.loads(data)
|
||||
return cls([
|
||||
request_type(**request)
|
||||
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest)
|
||||
if request_type.key in request
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
|
||||
return cls(
|
||||
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
|
||||
)
|
||||
|
||||
|
||||
class BlobResponse:
|
||||
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
|
||||
self.responses = responses
|
||||
self.blob_data = blob_data
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for response in self.responses:
|
||||
d.update(response.to_dict())
|
||||
return d
|
||||
|
||||
def _get_response(self, response_type: blob_response_types):
|
||||
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
|
||||
if response:
|
||||
return response[0]
|
||||
|
||||
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
|
||||
error = self._get_response(BlobErrorResponse)
|
||||
if error:
|
||||
log.error(error)
|
||||
return error
|
||||
|
||||
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
|
||||
response = self._get_response(BlobAvailabilityResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
|
||||
response = self._get_response(BlobPriceResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
|
||||
response = self._get_response(BlobDownloadResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
|
||||
response = self._get_response(BlobPaymentAddressResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobResponse':
|
||||
response, extra = _parse_blob_response(data)
|
||||
requests = []
|
||||
if response:
|
||||
requests.extend([
|
||||
response_type(**response)
|
||||
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse)
|
||||
if response_type.key in response
|
||||
])
|
||||
return cls(requests, extra)
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
from json.decoder import JSONDecodeError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
|
||||
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.buf = b''
|
||||
self.transport = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.peer_address_and_port: typing.Optional[str] = None
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
|
||||
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
|
||||
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
serialized = BlobResponse(to_send).serialize()
|
||||
self.transport.write(serialized)
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
|
||||
responses = []
|
||||
address_request = request.get_address_request()
|
||||
if address_request:
|
||||
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
|
||||
availability_request = request.get_availability_request()
|
||||
if availability_request:
|
||||
responses.append(BlobAvailabilityResponse(available_blobs=list(set((
|
||||
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
|
||||
availability_request.requested_blobs)
|
||||
)))))
|
||||
price_request = request.get_price_request()
|
||||
if price_request:
|
||||
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
|
||||
download_request = request.get_blob_request()
|
||||
|
||||
if download_request:
|
||||
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
||||
if blob.get_is_verified():
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
return
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
# self.transport.close()
|
||||
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if data:
|
||||
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.error("failed to decode blob request from %s:%i (%i bytes): %s", peer_address, peer_port,
|
||||
len(data), '' if not data else binascii.hexlify(data).decode())
|
||||
if not request:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.warning("failed to decode blob request from %s:%i", peer_address, peer_port)
|
||||
self.transport.close()
|
||||
return
|
||||
self.loop.create_task(self.handle_request(request))
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
log.info("Blob server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
def stop_server(self):
|
||||
if self.server_task:
|
||||
self.server_task.cancel()
|
||||
self.server_task = None
|
||||
log.info("Stopped blob server")
|
||||
@@ -1,2 +1,3 @@
|
||||
# dont touch this. CI server changes this during build/deployment
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
BUILD_COMMIT = "source installation"
|
||||
@@ -0,0 +1,90 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import collections
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CONNECTED_EVENT = "connected"
|
||||
DISCONNECTED_EVENT = "disconnected"
|
||||
TRANSFERRED_EVENT = "transferred"
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop):
|
||||
self.loop = loop
|
||||
self.incoming_connected: typing.Set[str] = set()
|
||||
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self.outgoing_connected: typing.Set[str] = set()
|
||||
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self._status = {}
|
||||
|
||||
self._task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return self._status
|
||||
|
||||
def sent_data(self, host_and_port: str, size: int):
|
||||
self.outgoing[host_and_port] += size
|
||||
|
||||
def received_data(self, host_and_port: str, size: int):
|
||||
self.incoming[host_and_port] += size
|
||||
|
||||
def connection_made(self, host_and_port: str):
|
||||
self.outgoing_connected.add(host_and_port)
|
||||
|
||||
def connection_received(self, host_and_port: str):
|
||||
# self.incoming_connected.add(host_and_port)
|
||||
pass
|
||||
|
||||
def outgoing_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.outgoing_connected:
|
||||
self.outgoing_connected.remove(host_and_port)
|
||||
|
||||
def incoming_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.incoming_connected:
|
||||
self.incoming_connected.remove(host_and_port)
|
||||
|
||||
async def _update(self):
|
||||
|
||||
self._status = {
|
||||
'incoming_bps': {},
|
||||
'outgoing_bps': {},
|
||||
'total_incoming_mbs': 0.0,
|
||||
'total_outgoing_mbs': 0.0,
|
||||
'time': self.loop.time()
|
||||
}
|
||||
|
||||
while True:
|
||||
last = self.loop.time()
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
self._status['incoming_bps'].clear()
|
||||
self._status['outgoing_bps'].clear()
|
||||
while self.outgoing:
|
||||
k, v = self.outgoing.popitem()
|
||||
self._status['outgoing_bps'][k] = v
|
||||
while self.incoming:
|
||||
k, v = self.incoming.popitem()
|
||||
self._status['incoming_bps'][k] = v
|
||||
now = self.loop.time()
|
||||
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
|
||||
) / (now - last)) / 1000000.0
|
||||
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
|
||||
) / (now - last)) / 1000000.0
|
||||
self._status['time'] = now
|
||||
|
||||
def stop(self):
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
self.outgoing.clear()
|
||||
self.outgoing_connected.clear()
|
||||
self.incoming.clear()
|
||||
self.incoming_connected.clear()
|
||||
self._status.clear()
|
||||
|
||||
def start(self):
|
||||
self.stop()
|
||||
self._task = self.loop.create_task(self._update())
|
||||
@@ -1,96 +0,0 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
from decimal import Decimal
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityTracker(object):
|
||||
"""
|
||||
Class to track peer counts for known blobs, and to discover new popular blobs
|
||||
|
||||
Attributes:
|
||||
availability (dict): dictionary of peers for known blobs
|
||||
"""
|
||||
|
||||
def __init__(self, blob_manager, peer_finder, dht_node):
|
||||
self.availability = {}
|
||||
self._last_mean_availability = Decimal(0.0)
|
||||
self._blob_manager = blob_manager
|
||||
self._peer_finder = peer_finder
|
||||
self._dht_node = dht_node
|
||||
self._check_mine = LoopingCall(self._update_mine)
|
||||
|
||||
def start(self):
|
||||
log.info("Starting blob availability tracker.")
|
||||
self._check_mine.start(600)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping blob availability tracker.")
|
||||
if self._check_mine.running:
|
||||
self._check_mine.stop()
|
||||
|
||||
def get_blob_availability(self, blob, timeout=None):
|
||||
def _get_peer_count(peers):
|
||||
have_blob = sum(1 for peer in peers if peer.is_available())
|
||||
return {blob: have_blob}
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob, timeout)
|
||||
d.addCallback(_get_peer_count)
|
||||
return d
|
||||
|
||||
def get_availability_for_blobs(self, blobs, timeout=None):
|
||||
dl = [self.get_blob_availability(blob, timeout) for blob in blobs if blob]
|
||||
d = defer.DeferredList(dl)
|
||||
d.addCallback(lambda results: [val for success, val in results if success])
|
||||
return d
|
||||
|
||||
@property
|
||||
def last_mean_availability(self):
|
||||
return max(Decimal(0.01), self._last_mean_availability)
|
||||
|
||||
def _update_peers_for_blob(self, blob):
|
||||
def _save_peer_info(blob_hash, peers):
|
||||
v = {blob_hash: peers}
|
||||
self.availability.update(v)
|
||||
return v
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob)
|
||||
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
|
||||
d.addCallback(lambda peers: _save_peer_info(blob, peers))
|
||||
return d
|
||||
|
||||
def _update_mine(self):
|
||||
def _get_peers(blobs):
|
||||
dl = []
|
||||
for hash in blobs:
|
||||
dl.append(self._update_peers_for_blob(hash))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
def sample(blobs):
|
||||
return random.sample(blobs, min(len(blobs), 10))
|
||||
|
||||
start = time.time()
|
||||
log.debug('==> Updating the peers for my blobs')
|
||||
d = self._blob_manager.get_all_verified_blobs()
|
||||
# as far as I can tell, this only is used to set _last_mean_availability
|
||||
# which... seems like a very expensive operation for such little payoff.
|
||||
# so taking a sample should get about the same effect as querying the entire
|
||||
# list of blobs
|
||||
d.addCallback(sample)
|
||||
d.addCallback(_get_peers)
|
||||
d.addCallback(lambda _: self._set_mean_peers())
|
||||
d.addCallback(lambda _: log.debug('<== Done updating peers for my blobs. Took %s seconds',
|
||||
time.time() - start))
|
||||
# although unused, need to return or else the looping call
|
||||
# could overrun on a previous call
|
||||
return d
|
||||
|
||||
def _set_mean_peers(self):
|
||||
num_peers = [len(self.availability[blob]) for blob in self.availability]
|
||||
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
|
||||
self._last_mean_availability = mean
|
||||
@@ -1,19 +0,0 @@
|
||||
class BlobInfo(object):
|
||||
"""
|
||||
This structure is used to represent the metadata of a blob.
|
||||
|
||||
@ivar blob_hash: The sha384 hashsum of the blob's data.
|
||||
@type blob_hash: string, hex-encoded
|
||||
|
||||
@ivar blob_num: For streams, the position of the blob in the stream.
|
||||
@type blob_num: integer
|
||||
|
||||
@ivar length: The length of the blob in bytes.
|
||||
@type length: integer
|
||||
"""
|
||||
|
||||
def __init__(self, blob_hash, blob_num, length):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
from sqlite3 import IntegrityError
|
||||
from twisted.internet import threads, defer, task
|
||||
from lbrynet import conf
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.creator import BlobFileCreator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DiskBlobManager(object):
|
||||
def __init__(self, blob_dir, storage, node_datastore=None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
|
||||
blob_dir - directory where blobs are stored
|
||||
storage - SQLiteStorage object
|
||||
"""
|
||||
self.storage = storage
|
||||
self.blob_dir = blob_dir
|
||||
self._node_datastore = node_datastore
|
||||
self.blob_creator_type = BlobFileCreator
|
||||
# TODO: consider using an LRU for blobs as there could potentially
|
||||
# be thousands of blobs loaded up, many stale
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
|
||||
self.check_should_announce_lc = None
|
||||
if conf.settings['run_reflector_server']: # TODO: move this looping call to SQLiteStorage
|
||||
self.check_should_announce_lc = task.LoopingCall(self.storage.verify_will_announce_all_head_and_sd_blobs)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
if self.check_should_announce_lc and not self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.start(600)
|
||||
if self._node_datastore is not None:
|
||||
raw_blob_hashes = yield self.storage.get_all_finished_blobs()
|
||||
self._node_datastore.completed_blobs.update(raw_blob_hashes)
|
||||
defer.returnValue(True)
|
||||
|
||||
def stop(self):
|
||||
if self.check_should_announce_lc and self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.stop()
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a
|
||||
blob that is already on the hard disk
|
||||
"""
|
||||
if length is not None and not isinstance(length, int):
|
||||
raise Exception("invalid length type: %s (%s)" % (length, str(type(length))))
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
return self.blob_creator_type(self.blob_dir)
|
||||
|
||||
def _make_new_blob(self, blob_hash, length=None):
|
||||
log.debug('Making a new blob for %s', blob_hash)
|
||||
blob = BlobFile(self.blob_dir, blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
return defer.succeed(blob)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def blob_completed(self, blob, should_announce=False, next_announce_time=None):
|
||||
yield self.storage.add_completed_blob(
|
||||
blob.blob_hash, blob.length, next_announce_time, should_announce
|
||||
)
|
||||
if self._node_datastore is not None:
|
||||
self._node_datastore.completed_blobs.add(blob.blob_hash.decode('hex'))
|
||||
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
return self._completed_blobs(blobhashes_to_check)
|
||||
|
||||
def count_should_announce_blobs(self):
|
||||
return self.storage.count_should_announce_blobs()
|
||||
|
||||
def set_should_announce(self, blob_hash, should_announce):
|
||||
now = self.storage.clock.seconds()
|
||||
return self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
|
||||
def get_should_announce(self, blob_hash):
|
||||
return self.storage.should_announce(blob_hash)
|
||||
|
||||
def creator_finished(self, blob_creator, should_announce):
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
if blob_creator.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if blob_creator.blob_hash in self.blobs:
|
||||
raise Exception("Creator finished for blob that is already marked as completed")
|
||||
if blob_creator.length is None:
|
||||
raise Exception("Blob has a length of 0")
|
||||
new_blob = BlobFile(self.blob_dir, blob_creator.blob_hash, blob_creator.length)
|
||||
self.blobs[blob_creator.blob_hash] = new_blob
|
||||
return self.blob_completed(new_blob, should_announce)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_blobs(self, blob_hashes):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
if self._node_datastore is not None:
|
||||
try:
|
||||
self._node_datastore.completed_blobs.remove(blob_hash.decode('hex'))
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
blob = yield self.get_blob(blob_hash)
|
||||
yield blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
del self.blobs[blob_hash]
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
try:
|
||||
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
except IntegrityError as err:
|
||||
if err.message != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _completed_blobs(self, blobhashes_to_check):
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
|
||||
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
|
||||
defer.returnValue(blob_hashes)
|
||||
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
d = self.storage.get_all_blob_hashes()
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
@@ -1,21 +0,0 @@
|
||||
class DownloadOptionChoice(object):
|
||||
"""A possible choice that can be picked for some option.
|
||||
|
||||
An option can have one or more choices that can be picked from.
|
||||
"""
|
||||
def __init__(self, value, short_description, long_description, bool_options_description=None):
|
||||
self.value = value
|
||||
self.short_description = short_description
|
||||
self.long_description = long_description
|
||||
self.bool_options_description = bool_options_description
|
||||
|
||||
|
||||
class DownloadOption(object):
|
||||
"""An option for a user to select a value from several different choices."""
|
||||
def __init__(self, option_types, long_description, short_description, default_value,
|
||||
default_value_description):
|
||||
self.option_types = option_types
|
||||
self.long_description = long_description
|
||||
self.short_description = short_description
|
||||
self.default_value = default_value
|
||||
self.default_value_description = default_value_description
|
||||
@@ -1,62 +0,0 @@
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class Offer(object):
|
||||
"""A rate offer to download blobs from a host."""
|
||||
|
||||
RATE_ACCEPTED = "RATE_ACCEPTED"
|
||||
RATE_TOO_LOW = "RATE_TOO_LOW"
|
||||
RATE_UNSET = "RATE_UNSET"
|
||||
|
||||
def __init__(self, offer):
|
||||
self._state = None
|
||||
self.rate = None
|
||||
if isinstance(offer, Decimal):
|
||||
self.rate = round(offer, 5)
|
||||
elif isinstance(offer, float):
|
||||
self.rate = round(Decimal(offer), 5)
|
||||
if self.rate is None or self.rate < Decimal(0.0):
|
||||
self.unset()
|
||||
|
||||
@property
|
||||
def is_accepted(self):
|
||||
return self._state is Offer.RATE_ACCEPTED
|
||||
|
||||
@property
|
||||
def is_too_low(self):
|
||||
return self._state is Offer.RATE_TOO_LOW
|
||||
|
||||
@property
|
||||
def is_unset(self):
|
||||
return self._state is Offer.RATE_UNSET
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
if self.is_accepted:
|
||||
return Offer.RATE_ACCEPTED
|
||||
elif self.is_too_low:
|
||||
return Offer.RATE_TOO_LOW
|
||||
elif self.is_unset:
|
||||
return Offer.RATE_UNSET
|
||||
return None
|
||||
|
||||
def accept(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_ACCEPTED
|
||||
|
||||
def reject(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_TOO_LOW
|
||||
|
||||
def unset(self):
|
||||
self._state = Offer.RATE_UNSET
|
||||
|
||||
def handle(self, reply_message):
|
||||
if reply_message == Offer.RATE_TOO_LOW:
|
||||
self.reject()
|
||||
elif reply_message == Offer.RATE_ACCEPTED:
|
||||
self.accept()
|
||||
elif reply_message == Offer.RATE_UNSET:
|
||||
self.unset()
|
||||
else:
|
||||
raise Exception("Unknown offer reply %s" % str(reply_message))
|
||||
@@ -1,128 +0,0 @@
|
||||
from lbrynet.core.Strategy import get_default_strategy, OnlyFreeStrategy
|
||||
from lbrynet import conf
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class BasePaymentRateManager(object):
|
||||
def __init__(self, rate=None, info_rate=None):
|
||||
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
|
||||
self.min_blob_info_payment_rate = (
|
||||
info_rate if info_rate is not None else conf.settings['min_info_rate'])
|
||||
|
||||
|
||||
class PaymentRateManager(object):
|
||||
def __init__(self, base, rate=None):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
|
||||
@param rate: the min blob data payment rate
|
||||
"""
|
||||
self.base = base
|
||||
self.min_blob_data_payment_rate = rate
|
||||
self.points_paid = 0.0
|
||||
|
||||
def get_rate_blob_data(self, peer):
|
||||
return self.get_effective_min_blob_data_payment_rate()
|
||||
|
||||
def accept_rate_blob_data(self, peer, payment_rate):
|
||||
return payment_rate >= self.get_effective_min_blob_data_payment_rate()
|
||||
|
||||
def get_effective_min_blob_data_payment_rate(self):
|
||||
if self.min_blob_data_payment_rate is None:
|
||||
return self.base.min_blob_data_payment_rate
|
||||
return self.min_blob_data_payment_rate
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
|
||||
class NegotiatedPaymentRateManager(object):
|
||||
def __init__(self, base, availability_tracker, generous=None):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
@param availability_tracker: a BlobAvailabilityTracker
|
||||
@param rate: the min blob data payment rate
|
||||
"""
|
||||
|
||||
self.base = base
|
||||
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
|
||||
self.points_paid = 0.0
|
||||
self.blob_tracker = availability_tracker
|
||||
self.generous = generous if generous is not None else conf.settings['is_generous_host']
|
||||
self.strategy = get_default_strategy(self.blob_tracker,
|
||||
base_price=self.base.min_blob_data_payment_rate,
|
||||
is_generous=generous)
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
return (offer.is_too_low and
|
||||
round(Decimal.from_float(offer.rate), 5) >= round(self.strategy.max_rate, 5))
|
||||
return False
|
||||
|
||||
|
||||
class OnlyFreePaymentsManager(object):
|
||||
def __init__(self, **kwargs):
|
||||
"""
|
||||
A payment rate manager that will only ever accept and offer a rate of 0.0,
|
||||
Used for testing
|
||||
"""
|
||||
|
||||
self.base = BasePaymentRateManager(0.0, 0.0)
|
||||
self.points_paid = 0.0
|
||||
self.min_blob_data_payment_rate = 0.0
|
||||
self.generous = True
|
||||
self.strategy = OnlyFreeStrategy()
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
if offer.rate > 0.0:
|
||||
return True
|
||||
return False
|
||||
@@ -1,46 +0,0 @@
|
||||
import datetime
|
||||
from collections import defaultdict
|
||||
from lbrynet.core import utils
|
||||
|
||||
# Do not create this object except through PeerManager
|
||||
class Peer(object):
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
# If a peer is reported down, we wait till this time till reattempting connection
|
||||
self.attempt_connection_at = None
|
||||
# Number of times this Peer has been reported to be down, resets to 0 when it is up
|
||||
self.down_count = 0
|
||||
# Number of succesful connections (with full protocol completion) to this peer
|
||||
self.success_count = 0
|
||||
self.score = 0
|
||||
self.stats = defaultdict(float) # {string stat_type, float count}
|
||||
|
||||
def is_available(self):
|
||||
if self.attempt_connection_at is None or utils.today() > self.attempt_connection_at:
|
||||
return True
|
||||
return False
|
||||
|
||||
def report_up(self):
|
||||
self.down_count = 0
|
||||
self.attempt_connection_at = None
|
||||
|
||||
def report_success(self):
|
||||
self.success_count += 1
|
||||
|
||||
def report_down(self):
|
||||
self.down_count += 1
|
||||
timeout_time = datetime.timedelta(seconds=60 * self.down_count)
|
||||
self.attempt_connection_at = utils.today() + timeout_time
|
||||
|
||||
def update_score(self, score_change):
|
||||
self.score += score_change
|
||||
|
||||
def update_stats(self, stat_type, count):
|
||||
self.stats[stat_type] += count
|
||||
|
||||
def __str__(self):
|
||||
return '{}:{}'.format(self.host, self.port)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Peer({!r}, {!r})'.format(self.host, self.port)
|
||||
@@ -1,14 +0,0 @@
|
||||
from lbrynet.core.Peer import Peer
|
||||
|
||||
|
||||
class PeerManager(object):
|
||||
def __init__(self):
|
||||
self.peers = []
|
||||
|
||||
def get_peer(self, host, port):
|
||||
for peer in self.peers:
|
||||
if peer.host == host and peer.port == port:
|
||||
return peer
|
||||
peer = Peer(host, port)
|
||||
self.peers.append(peer)
|
||||
return peer
|
||||
@@ -1,62 +0,0 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.interfaces import IBlobPriceModel
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
def get_default_price_model(blob_tracker, base_price, **kwargs):
|
||||
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
|
||||
|
||||
|
||||
class ZeroPrice(object):
|
||||
def __init__(self):
|
||||
self.base_price = 0.0
|
||||
|
||||
def calculate_price(self, blob):
|
||||
return 0.0
|
||||
|
||||
|
||||
class MeanAvailabilityWeightedPrice(object):
|
||||
"""Calculate mean-blob-availability and stream-position weighted price for a blob
|
||||
|
||||
Attributes:
|
||||
base_price (float): base price
|
||||
alpha (float): constant, > 0.0 and <= 1.0, used to more highly
|
||||
value blobs at the beginning of a stream.
|
||||
alpha defaults to 1.0, which has a null effect
|
||||
blob_tracker (BlobAvailabilityTracker): blob availability tracker
|
||||
|
||||
"""
|
||||
implementer(IBlobPriceModel)
|
||||
|
||||
def __init__(self, tracker, base_price=None, alpha=1.0):
|
||||
self.blob_tracker = tracker
|
||||
base_price = base_price if base_price is not None else conf.settings['data_rate']
|
||||
self.base_price = Decimal(base_price)
|
||||
self.alpha = Decimal(alpha)
|
||||
|
||||
def calculate_price(self, blob):
|
||||
mean_availability = self.blob_tracker.last_mean_availability
|
||||
availability = self.blob_tracker.availability.get(blob, [])
|
||||
index = 0 # blob.index
|
||||
availability_mult = self._get_availability_multiplier(mean_availability, availability)
|
||||
price = self.base_price * availability_mult / self._frontload(index)
|
||||
return round(price, 5)
|
||||
|
||||
def _get_availability_multiplier(self, mean_availability, availability):
|
||||
return Decimal(max(1, mean_availability) / Decimal(max(1, len(availability))))
|
||||
|
||||
def _frontload(self, index):
|
||||
"""Get front-load multiplier, used to weight prices of blobs in a
|
||||
stream towards the front of the stream.
|
||||
|
||||
At index 0, returns 1.0
|
||||
As index increases, return value approaches 2.0
|
||||
|
||||
@param index: blob position in stream
|
||||
@return: front-load multiplier
|
||||
|
||||
"""
|
||||
|
||||
return Decimal(2.0) - (self.alpha ** index)
|
||||
@@ -1,156 +0,0 @@
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IRateLimiter
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyRateLimiter(object):
|
||||
def __init__(self):
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.target_dl = 0
|
||||
self.target_ul = 0
|
||||
self.tick_call = None
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(1)
|
||||
|
||||
def tick(self):
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
|
||||
def stop(self):
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
pass
|
||||
|
||||
def set_ul_limit(self, limit):
|
||||
pass
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_second += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
|
||||
|
||||
class RateLimiter(object):
|
||||
"""This class ensures that upload and download rates don't exceed specified maximums"""
|
||||
|
||||
implements(IRateLimiter)
|
||||
|
||||
#called by main application
|
||||
|
||||
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
|
||||
self.max_dl_bytes = max_dl_bytes
|
||||
self.max_ul_bytes = max_ul_bytes
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.tick_call = None
|
||||
self.tick_interval = 0.1
|
||||
|
||||
self.dl_throttled = False
|
||||
self.ul_throttled = False
|
||||
|
||||
self.protocols = []
|
||||
|
||||
def start(self):
|
||||
log.info("Starting rate limiter.")
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(self.tick_interval)
|
||||
|
||||
def tick(self):
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.unthrottle_dl()
|
||||
self.unthrottle_ul()
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping rate limiter.")
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
self.max_dl_bytes = limit
|
||||
|
||||
def set_ul_limit(self, limit):
|
||||
self.max_ul_bytes = limit
|
||||
|
||||
#throttling
|
||||
|
||||
def check_dl(self):
|
||||
need_throttle = (self.max_dl_bytes is not None and
|
||||
self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_dl)
|
||||
|
||||
def check_ul(self):
|
||||
need_throttle = (self.max_ul_bytes is not None and
|
||||
self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_ul)
|
||||
|
||||
def throttle_dl(self):
|
||||
if self.dl_throttled is False:
|
||||
for protocol in self.protocols:
|
||||
protocol.throttle_download()
|
||||
self.dl_throttled = True
|
||||
|
||||
def throttle_ul(self):
|
||||
if self.ul_throttled is False:
|
||||
for protocol in self.protocols:
|
||||
protocol.throttle_upload()
|
||||
self.ul_throttled = True
|
||||
|
||||
def unthrottle_dl(self):
|
||||
if self.dl_throttled is True:
|
||||
for protocol in self.protocols:
|
||||
protocol.unthrottle_download()
|
||||
self.dl_throttled = False
|
||||
|
||||
def unthrottle_ul(self):
|
||||
if self.ul_throttled is True:
|
||||
for protocol in self.protocols:
|
||||
protocol.unthrottle_upload()
|
||||
self.ul_throttled = False
|
||||
|
||||
#called by protocols
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_interval += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
self.check_dl()
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_interval += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
self.check_ul()
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
if protocol not in self.protocols:
|
||||
self.protocols.append(protocol)
|
||||
if self.dl_throttled is True:
|
||||
protocol.throttle_download()
|
||||
if self.ul_throttled is True:
|
||||
protocol.throttle_upload()
|
||||
|
||||
def unregister_protocol(self, protocol):
|
||||
if protocol in self.protocols:
|
||||
self.protocols.remove(protocol)
|
||||
@@ -1,282 +0,0 @@
|
||||
import logging
|
||||
import miniupnpc
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.BlobManager import DiskBlobManager
|
||||
from lbrynet.dht import node, hashannouncer
|
||||
from lbrynet.database.storage import SQLiteStorage
|
||||
from lbrynet.core.RateLimiter import RateLimiter
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager, OnlyFreePaymentsManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Session(object):
|
||||
"""This class manages all important services common to any application that uses the network.
|
||||
|
||||
the hash announcer, which informs other peers that this peer is
|
||||
associated with some hash. Usually, this means this peer has a
|
||||
blob identified by the hash in question, but it can be used for
|
||||
other purposes.
|
||||
|
||||
the peer finder, which finds peers that are associated with some
|
||||
hash.
|
||||
|
||||
the blob manager, which keeps track of which blobs have been
|
||||
downloaded and provides access to them,
|
||||
|
||||
the rate limiter, which attempts to ensure download and upload
|
||||
rates stay below a set maximum
|
||||
|
||||
upnp, which opens holes in compatible firewalls so that remote
|
||||
peers can connect to this peer.
|
||||
"""
|
||||
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None, node_id=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None, hash_announcer=None, blob_dir=None, blob_manager=None,
|
||||
peer_port=None, use_upnp=True, rate_limiter=None, wallet=None, dht_node_class=node.Node,
|
||||
blob_tracker_class=None, payment_rate_manager_class=None, is_generous=True, external_ip=None,
|
||||
storage=None):
|
||||
"""@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
@param db_dir: The directory in which levelDB files should be stored
|
||||
|
||||
@param node_id: The unique ID of this node
|
||||
|
||||
@param peer_manager: An object which keeps track of all known
|
||||
peers. If None, a PeerManager will be created
|
||||
|
||||
@param dht_node_port: The port on which the dht node should
|
||||
listen for incoming connections
|
||||
|
||||
@param known_dht_nodes: A list of nodes which the dht node
|
||||
should use to bootstrap into the dht
|
||||
|
||||
@param peer_finder: An object which is used to look up peers
|
||||
that are associated with some hash. If None, a
|
||||
DHTPeerFinder will be used, which looks for peers in the
|
||||
distributed hash table.
|
||||
|
||||
@param hash_announcer: An object which announces to other
|
||||
peers that this peer is associated with some hash. If
|
||||
None, and peer_port is not None, a DHTHashAnnouncer will
|
||||
be used. If None and peer_port is None, a
|
||||
DummyHashAnnouncer will be used, which will not actually
|
||||
announce anything.
|
||||
|
||||
@param blob_dir: The directory in which blobs will be
|
||||
stored. If None and blob_manager is None, blobs will be
|
||||
stored in memory only.
|
||||
|
||||
@param blob_manager: An object which keeps track of downloaded
|
||||
blobs and provides access to them. If None, and blob_dir
|
||||
is not None, a DiskBlobManager will be used, with the
|
||||
given blob_dir. If None and blob_dir is None, a
|
||||
TempBlobManager will be used, which stores blobs in memory
|
||||
only.
|
||||
|
||||
@param peer_port: The port on which other peers should connect
|
||||
to this peer
|
||||
|
||||
@param use_upnp: Whether or not to try to open a hole in the
|
||||
firewall so that outside peers can connect to this peer's
|
||||
peer_port and dht_node_port
|
||||
|
||||
@param rate_limiter: An object which keeps track of the amount
|
||||
of data transferred to and from this peer, and can limit
|
||||
that rate if desired
|
||||
|
||||
@param wallet: An object which will be used to keep track of
|
||||
expected payments and which will pay peers. If None, a
|
||||
wallet which uses the Point Trader system will be used,
|
||||
which is meant for testing only
|
||||
|
||||
"""
|
||||
self.db_dir = db_dir
|
||||
self.node_id = node_id
|
||||
self.peer_manager = peer_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.hash_announcer = hash_announcer
|
||||
self.dht_node_port = dht_node_port
|
||||
self.known_dht_nodes = known_dht_nodes
|
||||
if self.known_dht_nodes is None:
|
||||
self.known_dht_nodes = []
|
||||
self.blob_dir = blob_dir
|
||||
self.blob_manager = blob_manager
|
||||
# self.blob_tracker = None
|
||||
# self.blob_tracker_class = blob_tracker_class or BlobAvailabilityTracker
|
||||
self.peer_port = peer_port
|
||||
self.use_upnp = use_upnp
|
||||
self.rate_limiter = rate_limiter
|
||||
self.external_ip = external_ip
|
||||
self.upnp_redirects = []
|
||||
self.wallet = wallet
|
||||
self.dht_node_class = dht_node_class
|
||||
self.dht_node = None
|
||||
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
|
||||
self.payment_rate_manager = OnlyFreePaymentsManager()
|
||||
# self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager
|
||||
# self.is_generous = is_generous
|
||||
self.storage = storage or SQLiteStorage(self.db_dir)
|
||||
|
||||
def setup(self):
|
||||
"""Create the blob directory and database if necessary, start all desired services"""
|
||||
|
||||
log.debug("Starting session.")
|
||||
|
||||
if self.node_id is None:
|
||||
self.node_id = generate_id()
|
||||
|
||||
if self.use_upnp is True:
|
||||
d = self._try_upnp()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: self.storage.setup())
|
||||
d.addCallback(lambda _: self._setup_dht())
|
||||
d.addCallback(lambda _: self._setup_other_components())
|
||||
return d
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop all services"""
|
||||
log.info('Stopping session.')
|
||||
ds = []
|
||||
if self.hash_announcer:
|
||||
self.hash_announcer.stop()
|
||||
# if self.blob_tracker is not None:
|
||||
# ds.append(defer.maybeDeferred(self.blob_tracker.stop))
|
||||
if self.dht_node is not None:
|
||||
ds.append(defer.maybeDeferred(self.dht_node.stop))
|
||||
if self.rate_limiter is not None:
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
if self.wallet is not None:
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.use_upnp is True:
|
||||
ds.append(defer.maybeDeferred(self._unset_upnp))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _try_upnp(self):
|
||||
|
||||
log.debug("In _try_upnp")
|
||||
|
||||
def get_free_port(upnp, port, protocol):
|
||||
# returns an existing mapping if it exists
|
||||
mapping = upnp.getspecificportmapping(port, protocol)
|
||||
if not mapping:
|
||||
return port
|
||||
if upnp.lanaddr == mapping[0]:
|
||||
return mapping[1]
|
||||
return get_free_port(upnp, port + 1, protocol)
|
||||
|
||||
def get_port_mapping(upnp, port, protocol, description):
|
||||
# try to map to the requested port, if there is already a mapping use the next external
|
||||
# port available
|
||||
if protocol not in ['UDP', 'TCP']:
|
||||
raise Exception("invalid protocol")
|
||||
port = get_free_port(upnp, port, protocol)
|
||||
if isinstance(port, tuple):
|
||||
log.info("Found existing UPnP redirect %s:%i (%s) to %s:%i, using it",
|
||||
self.external_ip, port, protocol, upnp.lanaddr, port)
|
||||
return port
|
||||
upnp.addportmapping(port, protocol, upnp.lanaddr, port,
|
||||
description, '')
|
||||
log.info("Set UPnP redirect %s:%i (%s) to %s:%i", self.external_ip, port,
|
||||
protocol, upnp.lanaddr, port)
|
||||
return port
|
||||
|
||||
def threaded_try_upnp():
|
||||
if self.use_upnp is False:
|
||||
log.debug("Not using upnp")
|
||||
return False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
external_ip = u.externalipaddress()
|
||||
if external_ip != '0.0.0.0' and not self.external_ip:
|
||||
# best not to rely on this external ip, the router can be behind layers of NATs
|
||||
self.external_ip = external_ip
|
||||
if self.peer_port:
|
||||
self.peer_port = get_port_mapping(u, self.peer_port, 'TCP', 'LBRY peer port')
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
if self.dht_node_port:
|
||||
self.dht_node_port = get_port_mapping(u, self.dht_node_port, 'UDP', 'LBRY DHT port')
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
return True
|
||||
return False
|
||||
|
||||
def upnp_failed(err):
|
||||
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
return False
|
||||
|
||||
d = threads.deferToThread(threaded_try_upnp)
|
||||
d.addErrback(upnp_failed)
|
||||
return d
|
||||
|
||||
def _setup_dht(self): # does not block startup, the dht will re-attempt if necessary
|
||||
self.dht_node = self.dht_node_class(
|
||||
node_id=self.node_id,
|
||||
udpPort=self.dht_node_port,
|
||||
externalIP=self.external_ip,
|
||||
peerPort=self.peer_port,
|
||||
peer_manager=self.peer_manager,
|
||||
peer_finder=self.peer_finder,
|
||||
)
|
||||
if not self.hash_announcer:
|
||||
self.hash_announcer = hashannouncer.DHTHashAnnouncer(self.dht_node, self.storage)
|
||||
self.peer_manager = self.dht_node.peer_manager
|
||||
self.peer_finder = self.dht_node.peer_finder
|
||||
d = self.dht_node.start(self.known_dht_nodes)
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
d.addCallback(lambda _: self.hash_announcer.start())
|
||||
|
||||
def _setup_other_components(self):
|
||||
log.debug("Setting up the rest of the components")
|
||||
|
||||
if self.rate_limiter is None:
|
||||
self.rate_limiter = RateLimiter()
|
||||
|
||||
if self.blob_manager is None:
|
||||
if self.blob_dir is None:
|
||||
raise Exception(
|
||||
"TempBlobManager is no longer supported, specify BlobManager or db_dir")
|
||||
else:
|
||||
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage, self.dht_node._dataStore)
|
||||
|
||||
# if self.blob_tracker is None:
|
||||
# self.blob_tracker = self.blob_tracker_class(
|
||||
# self.blob_manager, self.dht_node.peer_finder, self.dht_node
|
||||
# )
|
||||
# if self.payment_rate_manager is None:
|
||||
# self.payment_rate_manager = self.payment_rate_manager_class(
|
||||
# self.base_payment_rate_manager, self.blob_tracker, self.is_generous
|
||||
# )
|
||||
|
||||
self.rate_limiter.start()
|
||||
d = self.blob_manager.setup()
|
||||
d.addCallback(lambda _: self.wallet.start())
|
||||
# d.addCallback(lambda _: self.blob_tracker.start())
|
||||
return d
|
||||
|
||||
def _unset_upnp(self):
|
||||
log.info("Unsetting upnp for session")
|
||||
|
||||
def threaded_unset_upnp():
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
for port, protocol in self.upnp_redirects:
|
||||
if u.getspecificportmapping(port, protocol) is None:
|
||||
log.warning(
|
||||
"UPnP redirect for %s %d was removed by something else.",
|
||||
protocol, port)
|
||||
else:
|
||||
u.deleteportmapping(port, protocol)
|
||||
log.info("Removed UPnP redirect for %s %d.", protocol, port)
|
||||
self.upnp_redirects = []
|
||||
|
||||
d = threads.deferToThread(threaded_unset_upnp)
|
||||
d.addErrback(lambda err: str(err))
|
||||
return d
|
||||
@@ -1,105 +0,0 @@
|
||||
import logging
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from twisted.internet import defer, threads, reactor
|
||||
|
||||
from lbrynet.blob import BlobFile
|
||||
from lbrynet.core.BlobManager import DiskBlobManager
|
||||
from lbrynet.core.RateLimiter import DummyRateLimiter
|
||||
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.dht.peerfinder import DummyPeerFinder
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SinglePeerFinder(DummyPeerFinder):
|
||||
def __init__(self, peer):
|
||||
DummyPeerFinder.__init__(self)
|
||||
self.peer = peer
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=False):
|
||||
return defer.succeed([self.peer])
|
||||
|
||||
|
||||
class BlobCallback(BlobFile):
|
||||
def __init__(self, blob_dir, blob_hash, timeout):
|
||||
BlobFile.__init__(self, blob_dir, blob_hash)
|
||||
self.callback = defer.Deferred()
|
||||
reactor.callLater(timeout, self._cancel)
|
||||
|
||||
def _cancel(self):
|
||||
if not self.callback.called:
|
||||
self.callback.callback(False)
|
||||
|
||||
def save_verified_blob(self, writer):
|
||||
result = BlobFile.save_verified_blob(self, writer)
|
||||
if not self.callback.called:
|
||||
self.callback.callback(True)
|
||||
return result
|
||||
|
||||
|
||||
class SingleBlobDownloadManager(object):
|
||||
def __init__(self, blob):
|
||||
self.blob = blob
|
||||
|
||||
def needed_blobs(self):
|
||||
if self.blob.verified:
|
||||
return []
|
||||
else:
|
||||
return [self.blob]
|
||||
|
||||
def get_head_blob_hash(self):
|
||||
return self.blob.blob_hash
|
||||
|
||||
|
||||
class SinglePeerDownloader(object):
|
||||
def __init__(self):
|
||||
self._payment_rate_manager = OnlyFreePaymentsManager()
|
||||
self._rate_limiter = DummyRateLimiter()
|
||||
self._wallet = None
|
||||
self._blob_manager = None
|
||||
|
||||
def setup(self, wallet, blob_manager=None):
|
||||
if not self._wallet:
|
||||
self._wallet = wallet
|
||||
if not self._blob_manager:
|
||||
self._blob_manager = blob_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_blob_from_peer(self, peer, timeout, blob_hash, blob_manager):
|
||||
log.debug("Try to download %s from %s", blob_hash, peer.host)
|
||||
blob_manager = blob_manager
|
||||
blob = BlobCallback(blob_manager.blob_dir, blob_hash, timeout)
|
||||
download_manager = SingleBlobDownloadManager(blob)
|
||||
peer_finder = SinglePeerFinder(peer)
|
||||
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
|
||||
self._wallet, download_manager)
|
||||
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
|
||||
self._rate_limiter, self._payment_rate_manager,
|
||||
self._wallet, timeout=timeout)
|
||||
info_exchanger = self._wallet.get_info_exchanger()
|
||||
connection_manager = ConnectionManager(downloader, self._rate_limiter, [requester],
|
||||
[info_exchanger])
|
||||
connection_manager.start()
|
||||
|
||||
result = yield blob.callback
|
||||
if not result:
|
||||
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
|
||||
yield connection_manager.stop()
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
|
||||
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_dir)
|
||||
try:
|
||||
result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager)
|
||||
finally:
|
||||
yield tmp_blob_manager.stop()
|
||||
yield threads.deferToThread(shutil.rmtree, tmp_dir)
|
||||
defer.returnValue(result)
|
||||
@@ -1,152 +0,0 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.interfaces import INegotiationStrategy
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
|
||||
|
||||
|
||||
def get_default_strategy(blob_tracker, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
|
||||
|
||||
|
||||
class Strategy(object):
|
||||
"""
|
||||
Base for negotiation strategies
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
|
||||
self.price_model = price_model
|
||||
self.is_generous = (
|
||||
is_generous if is_generous is not None else conf.settings['is_generous_host'])
|
||||
self.accepted_offers = {}
|
||||
self.pending_sent_offers = {}
|
||||
self.offers_sent = {}
|
||||
self.offers_received = {}
|
||||
self.max_rate = max_rate or Decimal(self.price_model.base_price * 50)
|
||||
self.min_rate = Decimal(min_rate)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def make_offer(self, peer, blobs):
|
||||
offer_count = self.offers_sent.get(peer, 0)
|
||||
self._add_offer_sent(peer)
|
||||
if peer in self.accepted_offers:
|
||||
# if there was a previous accepted offer, use that
|
||||
offer = self.accepted_offers[peer]
|
||||
if peer in self.pending_sent_offers:
|
||||
del self.pending_sent_offers[peer]
|
||||
elif offer_count == 0 and self.is_generous:
|
||||
# Try asking for it for free
|
||||
offer = Offer(Decimal(0.0))
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
else:
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._make_rate_offer(rates, offer_count)
|
||||
offer = Offer(price)
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
return offer
|
||||
|
||||
def respond_to_offer(self, offer, peer, blobs):
|
||||
offer_count = self.offers_received.get(peer, 0)
|
||||
self._add_offer_received(peer)
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._get_response_rate(rates, offer_count)
|
||||
if peer in self.accepted_offers:
|
||||
offer = self.accepted_offers[peer]
|
||||
elif offer.rate == 0.0 and offer_count == 0 and self.is_generous:
|
||||
# give blobs away for free by default on the first request
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
elif offer.rate >= price:
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
else:
|
||||
offer.reject()
|
||||
if peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
return offer
|
||||
|
||||
def update_accepted_offers(self, peer, offer):
|
||||
if not offer.is_accepted and peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
if offer.is_accepted:
|
||||
self.accepted_offers.update({peer: offer})
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
|
||||
def _add_offer_sent(self, peer):
|
||||
turn = self.offers_sent.get(peer, 0) + 1
|
||||
self.offers_sent.update({peer: turn})
|
||||
|
||||
def _add_offer_received(self, peer):
|
||||
turn = self.offers_received.get(peer, 0) + 1
|
||||
self.offers_received.update({peer: turn})
|
||||
|
||||
def _bounded_price(self, price):
|
||||
price_for_return = Decimal(min(self.max_rate, max(price, self.min_rate)))
|
||||
return price_for_return
|
||||
|
||||
|
||||
class BasicAvailabilityWeightedStrategy(Strategy):
|
||||
"""Basic strategy to target blob prices based on supply relative to mean supply
|
||||
|
||||
Discount price target with each incoming request, and raise it
|
||||
with each outgoing from the modeled price until the rate is
|
||||
accepted or a threshold is reached
|
||||
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, blob_tracker, acceleration=1.25,
|
||||
deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0,
|
||||
is_generous=None,
|
||||
base_price=0.0001, alpha=1.0):
|
||||
price_model = MeanAvailabilityWeightedPrice(
|
||||
blob_tracker, base_price=base_price, alpha=alpha)
|
||||
Strategy.__init__(self, price_model, max_rate, min_rate, is_generous)
|
||||
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
|
||||
self._deceleration = Decimal(deceleration)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
mean_rate = Decimal(sum(rates)) / Decimal(max(len(rates), 1))
|
||||
return mean_rate
|
||||
|
||||
def _premium(self, rate, turn):
|
||||
return rate * (self._acceleration ** Decimal(turn))
|
||||
|
||||
def _discount(self, rate, turn):
|
||||
return rate * (self._deceleration ** Decimal(turn))
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
discounted = self._discount(rate, offer_count)
|
||||
rounded_price = round(discounted, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
with_premium = self._premium(rate, offer_count)
|
||||
rounded_price = round(with_premium, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
|
||||
class OnlyFreeStrategy(Strategy):
|
||||
implementer(INegotiationStrategy)
|
||||
def __init__(self, *args, **kwargs):
|
||||
price_model = ZeroPrice()
|
||||
Strategy.__init__(self, price_model, 0.0, 0.0, True)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
return 0.0
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return 0.0
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return 0.0
|
||||
@@ -1,459 +0,0 @@
|
||||
import binascii
|
||||
from collections import defaultdict
|
||||
import json
|
||||
import logging
|
||||
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamDescriptorReader(object):
|
||||
"""Classes which derive from this class read a stream descriptor file return
|
||||
a dictionary containing the fields in the file"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def _get_raw_data(self):
|
||||
"""This method must be overridden by subclasses. It should return a deferred
|
||||
which fires with the raw data in the stream descriptor"""
|
||||
pass
|
||||
|
||||
def get_info(self):
|
||||
"""Return the fields contained in the file"""
|
||||
d = self._get_raw_data()
|
||||
d.addCallback(json.loads)
|
||||
return d
|
||||
|
||||
|
||||
class PlainStreamDescriptorReader(StreamDescriptorReader):
|
||||
"""Read a stream descriptor file which is not a blob but a regular file"""
|
||||
def __init__(self, stream_descriptor_filename):
|
||||
StreamDescriptorReader.__init__(self)
|
||||
self.stream_descriptor_filename = stream_descriptor_filename
|
||||
|
||||
def _get_raw_data(self):
|
||||
|
||||
def get_data():
|
||||
with open(self.stream_descriptor_filename) as file_handle:
|
||||
raw_data = file_handle.read()
|
||||
return raw_data
|
||||
|
||||
return threads.deferToThread(get_data)
|
||||
|
||||
|
||||
class BlobStreamDescriptorReader(StreamDescriptorReader):
|
||||
"""Read a stream descriptor file which is a blob"""
|
||||
def __init__(self, blob):
|
||||
StreamDescriptorReader.__init__(self)
|
||||
self.blob = blob
|
||||
|
||||
def _get_raw_data(self):
|
||||
|
||||
def get_data():
|
||||
f = self.blob.open_for_reading()
|
||||
if f is not None:
|
||||
raw_data = f.read()
|
||||
f.close()
|
||||
return raw_data
|
||||
else:
|
||||
raise ValueError("Could not open the blob for reading")
|
||||
|
||||
return threads.deferToThread(get_data)
|
||||
|
||||
|
||||
class StreamDescriptorWriter(object):
|
||||
"""Classes which derive from this class write fields from a dictionary
|
||||
of fields to a stream descriptor"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def create_descriptor(self, sd_info):
|
||||
return self._write_stream_descriptor(json.dumps(sd_info))
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
"""This method must be overridden by subclasses to write raw data to
|
||||
the stream descriptor
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class PlainStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def __init__(self, sd_file_name):
|
||||
StreamDescriptorWriter.__init__(self)
|
||||
self.sd_file_name = sd_file_name
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
|
||||
def write_file():
|
||||
log.info("Writing the sd file to disk")
|
||||
with open(self.sd_file_name, 'w') as sd_file:
|
||||
sd_file.write(raw_data)
|
||||
return self.sd_file_name
|
||||
|
||||
return threads.deferToThread(write_file)
|
||||
|
||||
|
||||
class BlobStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def __init__(self, blob_manager):
|
||||
StreamDescriptorWriter.__init__(self)
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
log.debug("Creating the new blob for the stream descriptor")
|
||||
blob_creator = self.blob_manager.get_blob_creator()
|
||||
blob_creator.write(raw_data)
|
||||
log.debug("Wrote the data to the new blob")
|
||||
sd_hash = yield blob_creator.close()
|
||||
yield self.blob_manager.creator_finished(blob_creator, should_announce=True)
|
||||
defer.returnValue(sd_hash)
|
||||
|
||||
|
||||
class StreamMetadata(object):
|
||||
FROM_BLOB = 1
|
||||
FROM_PLAIN = 2
|
||||
|
||||
def __init__(self, validator, options, factories):
|
||||
self.validator = validator
|
||||
self.options = options
|
||||
self.factories = factories
|
||||
self.metadata_source = None
|
||||
self.source_blob_hash = None
|
||||
self.source_file = None
|
||||
|
||||
|
||||
class StreamDescriptorIdentifier(object):
|
||||
"""Tries to determine the type of stream described by the stream descriptor using the
|
||||
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
|
||||
and returns the appropriate ones based on the type of the stream descriptor given
|
||||
"""
|
||||
def __init__(self):
|
||||
# {stream_type: IStreamDescriptorValidator}
|
||||
self._sd_info_validators = {}
|
||||
# {stream_type: IStreamOptions
|
||||
self._stream_options = {}
|
||||
# {stream_type: [IStreamDownloaderFactory]}
|
||||
self._stream_downloader_factories = defaultdict(list)
|
||||
|
||||
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
|
||||
"""This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
|
||||
There can only be one StreamDescriptorValidator for each type of stream.
|
||||
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor. This must be unique to this stream descriptor.
|
||||
|
||||
@param sd_info_validator: A class implementing the
|
||||
IStreamDescriptorValidator interface. This class's
|
||||
constructor will be passed the raw metadata in the stream
|
||||
descriptor file and its 'validate' method will then be
|
||||
called. If the validation step fails, an exception will be
|
||||
thrown, preventing the stream descriptor from being
|
||||
further processed.
|
||||
|
||||
@param stream_options: A class implementing the IStreamOptions
|
||||
interface. This class's constructor will be passed the
|
||||
sd_info_validator object containing the raw metadata from
|
||||
the stream descriptor file.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._sd_info_validators[stream_type] = sd_info_validator
|
||||
self._stream_options[stream_type] = stream_options
|
||||
|
||||
def add_stream_downloader_factory(self, stream_type, factory):
|
||||
"""Register a stream downloader factory with the StreamDescriptorIdentifier.
|
||||
|
||||
This is how the StreamDescriptorIdentifier determines what
|
||||
factories may be used to process different stream descriptor
|
||||
files. There must be at least one factory for each type of
|
||||
stream added via "add_stream_info_validator".
|
||||
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor which the factory knows how to process.
|
||||
|
||||
@param factory: An object implementing the IStreamDownloaderFactory interface.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._stream_downloader_factories[stream_type].append(factory)
|
||||
|
||||
def _return_metadata(self, options_validator_factories, source_type, source):
|
||||
validator, options, factories = options_validator_factories
|
||||
m = StreamMetadata(validator, options, factories)
|
||||
m.metadata_source = source_type
|
||||
if source_type == StreamMetadata.FROM_BLOB:
|
||||
m.source_blob_hash = source
|
||||
if source_type == StreamMetadata.FROM_PLAIN:
|
||||
m.source_file = source
|
||||
return m
|
||||
|
||||
def get_metadata_for_sd_file(self, sd_path):
|
||||
sd_reader = PlainStreamDescriptorReader(sd_path)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
|
||||
return d
|
||||
|
||||
def get_metadata_for_sd_blob(self, sd_blob):
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
|
||||
return d
|
||||
|
||||
def _get_factories(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_downloader_factories[stream_type]
|
||||
|
||||
def _get_validator(self, stream_type):
|
||||
if not stream_type in self._sd_info_validators:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._sd_info_validators[stream_type]
|
||||
|
||||
def _get_options(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_options[stream_type]
|
||||
|
||||
def _return_options_and_validator_and_factories(self, sd_info):
|
||||
if not 'stream_type' in sd_info:
|
||||
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
|
||||
stream_type = sd_info['stream_type']
|
||||
validator = self._get_validator(stream_type)(sd_info)
|
||||
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
|
||||
|
||||
d = validator.validate()
|
||||
|
||||
def get_options():
|
||||
options = self._get_options(stream_type)
|
||||
return validator, options, factories
|
||||
|
||||
d.addCallback(lambda _: get_options())
|
||||
return d
|
||||
|
||||
|
||||
EncryptedFileStreamType = "lbryfile"
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
if not blob_manager.blobs.get(sd_hash) or not blob_manager.blobs[sd_hash].get_is_verified():
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
calculated_sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
if calculated_sd_hash != sd_hash:
|
||||
raise InvalidStreamDescriptorError("%s does not match calculated %s" %
|
||||
(sd_hash, calculated_sd_hash))
|
||||
stream_hash = yield blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
if not stream_hash:
|
||||
log.debug("Saving info for %s", sd_info['stream_name'].decode('hex'))
|
||||
stream_name = sd_info['stream_name']
|
||||
key = sd_info['key']
|
||||
stream_hash = sd_info['stream_hash']
|
||||
stream_blobs = sd_info['blobs']
|
||||
suggested_file_name = sd_info['suggested_file_name']
|
||||
yield blob_manager.storage.add_known_blobs(stream_blobs)
|
||||
yield blob_manager.storage.store_stream(
|
||||
stream_hash, sd_hash, stream_name, key, suggested_file_name, stream_blobs
|
||||
)
|
||||
defer.returnValue(stream_hash)
|
||||
|
||||
|
||||
def format_blobs(crypt_blob_infos):
|
||||
formatted_blobs = []
|
||||
for blob_info in crypt_blob_infos:
|
||||
blob = {}
|
||||
if blob_info.length != 0:
|
||||
blob['blob_hash'] = str(blob_info.blob_hash)
|
||||
blob['blob_num'] = blob_info.blob_num
|
||||
blob['iv'] = str(blob_info.iv)
|
||||
blob['length'] = blob_info.length
|
||||
formatted_blobs.append(blob)
|
||||
return formatted_blobs
|
||||
|
||||
|
||||
def format_sd_info(stream_type, stream_name, key, suggested_file_name, stream_hash, blobs):
|
||||
return {
|
||||
"stream_type": stream_type,
|
||||
"stream_name": stream_name,
|
||||
"key": key,
|
||||
"suggested_file_name": suggested_file_name,
|
||||
"stream_hash": stream_hash,
|
||||
"blobs": blobs
|
||||
}
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_sd_info(storage, stream_hash, include_blobs):
|
||||
"""
|
||||
Get an sd info dictionary from storage
|
||||
|
||||
:param storage: (SQLiteStorage) storage instance
|
||||
:param stream_hash: (str) stream hash
|
||||
:param include_blobs: (bool) include stream blob infos
|
||||
|
||||
:return: {
|
||||
"stream_type": "lbryfile",
|
||||
"stream_name": <hex encoded stream name>,
|
||||
"key": <stream key>,
|
||||
"suggested_file_name": <hex encoded suggested file name>,
|
||||
"stream_hash": <stream hash>,
|
||||
"blobs": [
|
||||
{
|
||||
"blob_hash": <head blob_hash>,
|
||||
"blob_num": 0,
|
||||
"iv": <iv>,
|
||||
"length": <head blob length>
|
||||
}, ...
|
||||
{
|
||||
"blob_num": <stream length>,
|
||||
"iv": <iv>,
|
||||
"length": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
stream_info = yield storage.get_stream_info(stream_hash)
|
||||
blobs = []
|
||||
if include_blobs:
|
||||
blobs = yield storage.get_blobs_for_stream(stream_hash)
|
||||
defer.returnValue(
|
||||
format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_blob_hashsum(b):
|
||||
length = b['length']
|
||||
if length != 0:
|
||||
blob_hash = b['blob_hash']
|
||||
else:
|
||||
blob_hash = None
|
||||
blob_num = b['blob_num']
|
||||
iv = b['iv']
|
||||
blob_hashsum = get_lbry_hash_obj()
|
||||
if length != 0:
|
||||
blob_hashsum.update(blob_hash)
|
||||
blob_hashsum.update(str(blob_num))
|
||||
blob_hashsum.update(iv)
|
||||
blob_hashsum.update(str(length))
|
||||
return blob_hashsum.digest()
|
||||
|
||||
|
||||
def get_stream_hash(hex_stream_name, key, hex_suggested_file_name, blob_infos):
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(hex_stream_name)
|
||||
h.update(key)
|
||||
h.update(hex_suggested_file_name)
|
||||
blobs_hashsum = get_lbry_hash_obj()
|
||||
for blob in blob_infos:
|
||||
blobs_hashsum.update(get_blob_hashsum(blob))
|
||||
h.update(blobs_hashsum.digest())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def verify_hex(text, field_name):
|
||||
for c in text:
|
||||
if c not in '0123456789abcdef':
|
||||
raise InvalidStreamDescriptorError("%s is not a hex-encoded string" % field_name)
|
||||
|
||||
|
||||
def validate_descriptor(stream_info):
|
||||
try:
|
||||
hex_stream_name = stream_info['stream_name']
|
||||
key = stream_info['key']
|
||||
hex_suggested_file_name = stream_info['suggested_file_name']
|
||||
stream_hash = stream_info['stream_hash']
|
||||
blobs = stream_info['blobs']
|
||||
except KeyError as e:
|
||||
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
|
||||
if stream_info['blobs'][-1]['length'] != 0:
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
if any([False if blob_info['length'] > 0 else True for blob_info in stream_info['blobs'][:-1]]):
|
||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||
if 'blob_hash' in stream_info['blobs'][-1]:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
|
||||
verify_hex(key, "key")
|
||||
verify_hex(hex_suggested_file_name, "suggested file name")
|
||||
verify_hex(stream_hash, "stream_hash")
|
||||
|
||||
calculated_stream_hash = get_stream_hash(
|
||||
hex_stream_name, key, hex_suggested_file_name, blobs
|
||||
)
|
||||
if calculated_stream_hash != stream_hash:
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
return True
|
||||
|
||||
|
||||
class EncryptedFileStreamDescriptorValidator(object):
|
||||
def __init__(self, raw_info):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
return defer.succeed(validate_descriptor(self.raw_info))
|
||||
|
||||
def info_to_show(self):
|
||||
info = []
|
||||
info.append(("stream_name", binascii.unhexlify(self.raw_info.get("stream_name"))))
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
info.append(("stream_size", str(self.get_length_of_stream())))
|
||||
suggested_file_name = self.raw_info.get("suggested_file_name", None)
|
||||
if suggested_file_name is not None:
|
||||
suggested_file_name = binascii.unhexlify(suggested_file_name)
|
||||
info.append(("suggested_file_name", suggested_file_name))
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
return size_so_far
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_sd_blob(session, blob_hash, payment_rate_manager, timeout=None):
|
||||
"""
|
||||
Downloads a single blob from the network
|
||||
|
||||
@param session:
|
||||
|
||||
@param blob_hash:
|
||||
|
||||
@param payment_rate_manager:
|
||||
|
||||
@return: An object of type HashBlob
|
||||
"""
|
||||
|
||||
downloader = StandaloneBlobDownloader(blob_hash,
|
||||
session.blob_manager,
|
||||
session.peer_finder,
|
||||
session.rate_limiter,
|
||||
payment_rate_manager,
|
||||
session.wallet,
|
||||
timeout)
|
||||
sd_blob = yield downloader.download()
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
sd_info = yield sd_reader.get_info()
|
||||
try:
|
||||
validate_descriptor(sd_info)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
yield session.blob_manager.delete_blobs([blob_hash])
|
||||
raise err
|
||||
raw_sd = yield sd_reader._get_raw_data()
|
||||
yield session.blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
|
||||
yield save_sd_info(session.blob_manager, sd_blob.blob_hash, sd_info)
|
||||
defer.returnValue(sd_blob)
|
||||
@@ -1,7 +0,0 @@
|
||||
"""
|
||||
Classes and functions which can be used by any application wishing to make use of the LBRY network.
|
||||
|
||||
This includes classes for connecting to other peers and downloading blobs from them, listening for
|
||||
connections from peers and responding to their requests, managing locally stored blobs, sending
|
||||
and receiving payments, and locating peers in the DHT.
|
||||
"""
|
||||
@@ -1,81 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
MIN_DELAY = 0.0
|
||||
MAX_DELAY = 0.01
|
||||
DELAY_INCREMENT = 0.0001
|
||||
QUEUE_SIZE_THRESHOLD = 100
|
||||
|
||||
|
||||
class CallLaterManager(object):
|
||||
def __init__(self, callLater):
|
||||
"""
|
||||
:param callLater: (IReactorTime.callLater)
|
||||
"""
|
||||
|
||||
self._callLater = callLater
|
||||
self._pendingCallLaters = []
|
||||
self._delay = MIN_DELAY
|
||||
|
||||
def get_min_delay(self):
|
||||
self._pendingCallLaters = [cl for cl in self._pendingCallLaters if cl.active()]
|
||||
queue_size = len(self._pendingCallLaters)
|
||||
if queue_size > QUEUE_SIZE_THRESHOLD:
|
||||
self._delay = min((self._delay + DELAY_INCREMENT), MAX_DELAY)
|
||||
else:
|
||||
self._delay = max((self._delay - 2.0 * DELAY_INCREMENT), MIN_DELAY)
|
||||
return self._delay
|
||||
|
||||
def _cancel(self, call_later):
|
||||
"""
|
||||
:param call_later: DelayedCall
|
||||
:return: (callable) canceller function
|
||||
"""
|
||||
|
||||
def cancel(reason=None):
|
||||
"""
|
||||
:param reason: reason for cancellation, this is returned after cancelling the DelayedCall
|
||||
:return: reason
|
||||
"""
|
||||
|
||||
if call_later.active():
|
||||
call_later.cancel()
|
||||
if call_later in self._pendingCallLaters:
|
||||
self._pendingCallLaters.remove(call_later)
|
||||
return reason
|
||||
return cancel
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Cancel any callLaters that are still running
|
||||
"""
|
||||
|
||||
from twisted.internet import defer
|
||||
while self._pendingCallLaters:
|
||||
canceller = self._cancel(self._pendingCallLaters[0])
|
||||
try:
|
||||
canceller()
|
||||
except (defer.CancelledError, defer.AlreadyCalledError, ValueError):
|
||||
pass
|
||||
|
||||
def call_later(self, when, what, *args, **kwargs):
|
||||
"""
|
||||
Schedule a call later and get a canceller callback function
|
||||
|
||||
:param when: (float) delay in seconds
|
||||
:param what: (callable)
|
||||
:param args: (*tuple) args to be passed to the callable
|
||||
:param kwargs: (**dict) kwargs to be passed to the callable
|
||||
|
||||
:return: (tuple) twisted.internet.base.DelayedCall object, canceller function
|
||||
"""
|
||||
|
||||
call_later = self._callLater(when, what, *args, **kwargs)
|
||||
canceller = self._cancel(call_later)
|
||||
self._pendingCallLaters.append(call_later)
|
||||
return call_later, canceller
|
||||
|
||||
def call_soon(self, what, *args, **kwargs):
|
||||
delay = self.get_min_delay()
|
||||
return self.call_later(delay, what, *args, **kwargs)
|
||||
@@ -1,588 +0,0 @@
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from decimal import Decimal
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.error import ConnectionAborted
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
|
||||
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
|
||||
from lbrynet.interfaces import IRequestCreator
|
||||
from lbrynet.core.Offer import Offer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_points(num_bytes, rate):
|
||||
if isinstance(rate, float):
|
||||
return 1.0 * num_bytes * rate / 2**20
|
||||
elif isinstance(rate, Decimal):
|
||||
return 1.0 * num_bytes * float(rate) / 2**20
|
||||
else:
|
||||
raise Exception("Unknown rate type")
|
||||
|
||||
|
||||
def cache(fn):
|
||||
"""Caches the function call for each instance"""
|
||||
attr = '__{}_value'.format(fn.__name__)
|
||||
|
||||
def helper(self):
|
||||
if not hasattr(self, attr):
|
||||
value = fn(self)
|
||||
setattr(self, attr, value)
|
||||
return getattr(self, attr)
|
||||
return helper
|
||||
|
||||
|
||||
class BlobRequester(object):
|
||||
implements(IRequestCreator)
|
||||
|
||||
def __init__(self, blob_manager, peer_finder, payment_rate_manager, wallet, download_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self._download_manager = download_manager
|
||||
self._peers = defaultdict(int) # {Peer: score}
|
||||
self._available_blobs = defaultdict(list) # {Peer: [blob_hash]}
|
||||
self._unavailable_blobs = defaultdict(list) # {Peer: [blob_hash]}}
|
||||
self._protocol_prices = {} # {ClientProtocol: price}
|
||||
self._protocol_offers = {}
|
||||
self._price_disagreements = [] # [Peer]
|
||||
self._protocol_tries = {}
|
||||
self._maxed_out_peers = []
|
||||
self._incompatible_peers = []
|
||||
|
||||
######## IRequestCreator #########
|
||||
def send_next_request(self, peer, protocol):
|
||||
"""Makes an availability request, download request and price request"""
|
||||
if not self.should_send_next_request(peer):
|
||||
return defer.succeed(False)
|
||||
return self._send_next_request(peer, protocol)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_new_peers_for_head_blob(self):
|
||||
""" look for peers for the head blob """
|
||||
head_blob_hash = self._download_manager.get_head_blob_hash()
|
||||
peers = yield self._find_peers_for_hash(head_blob_hash)
|
||||
defer.returnValue(peers)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_new_peers_for_next_unavailable(self):
|
||||
"""
|
||||
Look for peers for the next unavailable blob, if we have
|
||||
all blobs, return an empty list
|
||||
"""
|
||||
blob_hash = yield self._get_hash_for_peer_search()
|
||||
if blob_hash is None:
|
||||
defer.returnValue([])
|
||||
peers = yield self._find_peers_for_hash(blob_hash)
|
||||
defer.returnValue(peers)
|
||||
|
||||
######### internal calls #########
|
||||
def should_send_next_request(self, peer):
|
||||
return (
|
||||
self._blobs_to_download() and
|
||||
self._should_send_request_to(peer)
|
||||
)
|
||||
|
||||
def _send_next_request(self, peer, protocol):
|
||||
log.debug('Sending a blob request for %s and %s', peer, protocol)
|
||||
availability = AvailabilityRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
head_blob_hash = self._download_manager.get_head_blob_hash()
|
||||
download = DownloadRequest(self, peer, protocol, self.payment_rate_manager,
|
||||
self.wallet, head_blob_hash)
|
||||
price = PriceRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
|
||||
sent_request = False
|
||||
if availability.can_make_request():
|
||||
availability.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if price.can_make_request():
|
||||
# TODO: document why a PriceRequest is only made if an
|
||||
# Availability or Download request was made
|
||||
price.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if download.can_make_request():
|
||||
try:
|
||||
download.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
except InsufficientFundsError as err:
|
||||
return defer.fail(err)
|
||||
|
||||
return defer.succeed(sent_request)
|
||||
|
||||
def _get_hash_for_peer_search(self):
|
||||
"""
|
||||
Get next unavailable hash for blob,
|
||||
returns None if there is nothing left to download
|
||||
"""
|
||||
r = None
|
||||
blobs_to_download = self._blobs_to_download()
|
||||
if blobs_to_download:
|
||||
blobs_without_sources = self._blobs_without_sources()
|
||||
if not blobs_without_sources:
|
||||
blob_hash = blobs_to_download[0].blob_hash
|
||||
else:
|
||||
blob_hash = blobs_without_sources[0].blob_hash
|
||||
r = blob_hash
|
||||
log.debug("Blob requester peer search response: %s", str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
d = self.peer_finder.find_peers_for_blob(h, filter_self=True)
|
||||
|
||||
def choose_best_peers(peers):
|
||||
bad_peers = self._get_bad_peers()
|
||||
without_bad_peers = [p for p in peers if not p in bad_peers]
|
||||
without_maxed_out_peers = [
|
||||
p for p in without_bad_peers if p not in self._maxed_out_peers]
|
||||
return without_maxed_out_peers
|
||||
|
||||
d.addCallback(choose_best_peers)
|
||||
|
||||
def lookup_failed(err):
|
||||
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
|
||||
return []
|
||||
|
||||
d.addErrback(lookup_failed)
|
||||
return d
|
||||
|
||||
def _should_send_request_to(self, peer):
|
||||
if self._peers[peer] < -5.0:
|
||||
return False
|
||||
if peer in self._price_disagreements:
|
||||
return False
|
||||
if peer in self._incompatible_peers:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _get_bad_peers(self):
|
||||
return [p for p in self._peers.iterkeys() if not self._should_send_request_to(p)]
|
||||
|
||||
def _hash_available(self, blob_hash):
|
||||
for peer in self._available_blobs:
|
||||
if blob_hash in self._available_blobs[peer]:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _hash_available_on(self, blob_hash, peer):
|
||||
if blob_hash in self._available_blobs[peer]:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _blobs_to_download(self):
|
||||
needed_blobs = self._download_manager.needed_blobs()
|
||||
return sorted(needed_blobs, key=lambda b: b.is_downloading())
|
||||
|
||||
def _blobs_without_sources(self):
|
||||
return [
|
||||
b for b in self._download_manager.needed_blobs()
|
||||
if not self._hash_available(b.blob_hash)
|
||||
]
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
if protocol in self._protocol_prices:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
|
||||
|
||||
class RequestHelper(object):
|
||||
def __init__(self, requestor, peer, protocol, payment_rate_manager):
|
||||
self.requestor = requestor
|
||||
self.peer = peer
|
||||
self.protocol = protocol
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
|
||||
@property
|
||||
def protocol_prices(self):
|
||||
return self.requestor._protocol_prices
|
||||
|
||||
@property
|
||||
def protocol_offers(self):
|
||||
return self.requestor._protocol_offers
|
||||
|
||||
@property
|
||||
def available_blobs(self):
|
||||
return self.requestor._available_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def unavailable_blobs(self):
|
||||
return self.requestor._unavailable_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def maxed_out_peers(self):
|
||||
return self.requestor._maxed_out_peers
|
||||
|
||||
def update_local_score(self, score):
|
||||
self.requestor._update_local_score(self.peer, score)
|
||||
|
||||
def _request_failed(self, reason, request_type):
|
||||
if reason.check(DownloadCanceledError, RequestCanceledError, ConnectionAborted,
|
||||
ConnectionClosedBeforeResponseError, ValueError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self.requestor._incompatible_peers.append(self.peer)
|
||||
log.warning("A request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
request_type, reason.getErrorMessage(), reason.type)
|
||||
self.update_local_score(-10.0)
|
||||
if isinstance(reason, (InvalidResponseError, NoResponseError)):
|
||||
self.peer.update_score(-10.0)
|
||||
else:
|
||||
self.peer.update_score(-2.0)
|
||||
return reason
|
||||
|
||||
def get_rate(self):
|
||||
if self.payment_rate_manager.price_limit_reached(self.peer):
|
||||
if self.peer not in self.maxed_out_peers:
|
||||
self.maxed_out_peers.append(self.peer)
|
||||
return None
|
||||
rate = self.protocol_prices.get(self.protocol)
|
||||
if rate is None:
|
||||
if self.peer in self.payment_rate_manager.strategy.pending_sent_offers:
|
||||
pending = self.payment_rate_manager.strategy.pending_sent_offers[self.peer]
|
||||
if not pending.is_too_low and not pending.is_accepted:
|
||||
return pending.rate
|
||||
rate = self.payment_rate_manager.get_rate_blob_data(self.peer, self.available_blobs)
|
||||
return rate
|
||||
|
||||
|
||||
def _handle_incoming_blob(response_dict, peer, request):
|
||||
if request.response_identifier not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not isinstance(response_dict[request.response_identifier], dict):
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
type(response_dict[request.response_identifier]))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if 'blob_hash' not in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError(
|
||||
"Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob'])
|
||||
)
|
||||
if 'length' not in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
|
||||
|
||||
def _handle_download_error(err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError,
|
||||
ConnectionClosedBeforeResponseError):
|
||||
log.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
|
||||
|
||||
class AvailabilityRequest(RequestHelper):
|
||||
"""Ask a peer what blobs it has available.
|
||||
|
||||
Results are saved in `_available_blobs` and `_unavailable_blobs`
|
||||
on the parent BlobRequester.
|
||||
"""
|
||||
def can_make_request(self):
|
||||
return self.get_top_needed_blobs()
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
to_request = self.get_top_needed_blobs()
|
||||
if not to_request:
|
||||
raise Exception('Unable to make a request without available blobs')
|
||||
return self._make_request(to_request)
|
||||
|
||||
@cache
|
||||
def get_top_needed_blobs(self, limit=20):
|
||||
all_needed = [
|
||||
b.blob_hash for b in self.requestor._blobs_to_download()
|
||||
if not self.is_available(b)
|
||||
]
|
||||
# sort them so that the peer will be asked first for blobs it
|
||||
# hasn't said it doesn't have
|
||||
sorted_needed = sorted(
|
||||
all_needed,
|
||||
key=lambda b: b in self.unavailable_blobs
|
||||
)
|
||||
return sorted_needed[:limit]
|
||||
|
||||
def is_available(self, blob):
|
||||
return blob.blob_hash in self.available_blobs
|
||||
|
||||
def _make_request(self, to_request):
|
||||
log.debug('Requesting blobs: %s', to_request)
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
|
||||
def _handle_request(self, a_r):
|
||||
log.debug('making an availability request')
|
||||
d1 = self.protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request")
|
||||
|
||||
def _handle_availability(self, response_dict, request):
|
||||
assert request.response_identifier == 'available_blobs'
|
||||
if 'available_blobs' not in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
log.debug("Received a response to the availability request")
|
||||
# save available blobs
|
||||
blob_hashes = response_dict['available_blobs']
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
self.process_available_blob_hash(blob_hash, request)
|
||||
# everything left in the request is missing
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self.unavailable_blobs.append(blob_hash)
|
||||
return True
|
||||
|
||||
def process_available_blob_hash(self, blob_hash, request):
|
||||
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self.available_blobs.append(blob_hash)
|
||||
self.remove_from_unavailable_blobs(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
|
||||
def remove_from_unavailable_blobs(self, blob_hash):
|
||||
if blob_hash in self.unavailable_blobs:
|
||||
self.unavailable_blobs.remove(blob_hash)
|
||||
|
||||
|
||||
class PriceRequest(RequestHelper):
|
||||
"""Ask a peer if a certain price is acceptable"""
|
||||
def can_make_request(self):
|
||||
if len(self.available_blobs) and self.protocol not in self.protocol_prices:
|
||||
return self.get_rate() is not None
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_price_request()
|
||||
self._handle_price_request(request)
|
||||
|
||||
def _get_price_request(self):
|
||||
rate = self.get_rate()
|
||||
if rate is None:
|
||||
log.debug("No blobs to request from %s", self.peer)
|
||||
raise Exception('Cannot make a price request without a payment rate')
|
||||
log.debug("Offer rate %s to %s for %i blobs", rate, self.peer, len(self.available_blobs))
|
||||
|
||||
request_dict = {'blob_data_payment_rate': rate}
|
||||
assert self.protocol not in self.protocol_offers
|
||||
self.protocol_offers[self.protocol] = rate
|
||||
return ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
|
||||
def _handle_price_request(self, price_request):
|
||||
d = self.protocol.add_request(price_request)
|
||||
d.addCallback(self._handle_price_response, price_request)
|
||||
d.addErrback(self._request_failed, "price request")
|
||||
|
||||
def _handle_price_response(self, response_dict, request):
|
||||
assert request.response_identifier == 'blob_data_payment_rate'
|
||||
if 'blob_data_payment_rate' not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
offer_value = self.protocol_offers.pop(self.protocol)
|
||||
offer = Offer(offer_value)
|
||||
offer.handle(response_dict['blob_data_payment_rate'])
|
||||
self.payment_rate_manager.record_offer_reply(self.peer, offer)
|
||||
if offer.is_accepted:
|
||||
log.info("Offered rate %f/mb accepted by %s", offer.rate, self.peer.host)
|
||||
self.protocol_prices[self.protocol] = offer.rate
|
||||
return True
|
||||
elif offer.is_too_low:
|
||||
log.debug("Offered rate %f/mb rejected by %s", offer.rate, self.peer.host)
|
||||
return not self.payment_rate_manager.price_limit_reached(self.peer)
|
||||
else:
|
||||
log.warning("Price disagreement")
|
||||
self.requestor._price_disagreements.append(self.peer)
|
||||
return False
|
||||
|
||||
|
||||
class DownloadRequest(RequestHelper):
|
||||
"""Choose a blob and download it from a peer and also pay the peer for the data."""
|
||||
def __init__(self, requester, peer, protocol, payment_rate_manager, wallet, head_blob_hash):
|
||||
RequestHelper.__init__(self, requester, peer, protocol, payment_rate_manager)
|
||||
self.wallet = wallet
|
||||
self.head_blob_hash = head_blob_hash
|
||||
|
||||
def can_make_request(self):
|
||||
if self.protocol in self.protocol_prices:
|
||||
return self.get_blob_details()
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_download_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
blob_details = self.get_blob_details()
|
||||
if not blob_details:
|
||||
raise Exception('No blobs available to download')
|
||||
return self._make_request(blob_details)
|
||||
|
||||
@cache
|
||||
def get_blob_details(self):
|
||||
"""Open a blob for writing and return the details.
|
||||
|
||||
If no blob can be opened, returns None.
|
||||
"""
|
||||
to_download = self.get_available_blobs()
|
||||
return self.find_blob(to_download)
|
||||
|
||||
def get_available_blobs(self):
|
||||
available_blobs = [
|
||||
b for b in self.requestor._blobs_to_download()
|
||||
if self.requestor._hash_available_on(b.blob_hash, self.peer)
|
||||
]
|
||||
log.debug('available blobs: %s', available_blobs)
|
||||
return available_blobs
|
||||
|
||||
def find_blob(self, to_download):
|
||||
"""Return the first blob in `to_download` that is successfully opened for write."""
|
||||
for blob in to_download:
|
||||
if blob.get_is_verified():
|
||||
log.debug('Skipping blob %s as its already validated', blob)
|
||||
continue
|
||||
writer, d = blob.open_for_writing(self.peer)
|
||||
if d is not None:
|
||||
return BlobDownloadDetails(blob, d, writer.write, writer.close, self.peer)
|
||||
log.warning('Skipping blob %s as there was an issue opening it for writing', blob)
|
||||
return None
|
||||
|
||||
def _make_request(self, blob_details):
|
||||
blob = blob_details.blob
|
||||
request = ClientBlobRequest(
|
||||
{'requested_blob': blob.blob_hash},
|
||||
'incoming_blob',
|
||||
blob_details.counting_write_func,
|
||||
blob_details.deferred,
|
||||
blob_details.cancel_func,
|
||||
blob
|
||||
)
|
||||
log.info("Requesting blob %s from %s", blob.blob_hash, self.peer)
|
||||
return request
|
||||
|
||||
def _handle_download_request(self, client_blob_request):
|
||||
reserved_points = self.reserve_funds_or_cancel(client_blob_request)
|
||||
self.add_callbacks_to_download_request(client_blob_request, reserved_points)
|
||||
self.create_add_blob_request(client_blob_request)
|
||||
|
||||
def reserve_funds_or_cancel(self, client_blob_request):
|
||||
reserved_points = self._reserve_points(client_blob_request.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
return reserved_points
|
||||
client_blob_request.cancel(InsufficientFundsError())
|
||||
client_blob_request.finished_deferred.addErrback(lambda _: True)
|
||||
raise InsufficientFundsError()
|
||||
|
||||
def add_callbacks_to_download_request(self, client_blob_request, reserved_points):
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
client_blob_request.finished_deferred.addCallbacks(
|
||||
self._download_succeeded,
|
||||
self._download_failed,
|
||||
callbackArgs=(client_blob_request.blob,),
|
||||
)
|
||||
client_blob_request.finished_deferred.addBoth(self._pay_or_cancel_payment,
|
||||
reserved_points, client_blob_request.blob)
|
||||
client_blob_request.finished_deferred.addErrback(_handle_download_error, self.peer,
|
||||
client_blob_request.blob)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, reserved_points, blob):
|
||||
if self._can_pay_peer(blob, arg):
|
||||
self._pay_peer(blob.length, reserved_points)
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _can_pay_peer(self, blob, arg):
|
||||
return (
|
||||
blob.length != 0 and
|
||||
(not isinstance(arg, Failure) or arg.check(DownloadCanceledError))
|
||||
)
|
||||
|
||||
def _pay_peer(self, num_bytes, reserved_points):
|
||||
assert num_bytes != 0
|
||||
rate = self.get_rate()
|
||||
point_amount = get_points(num_bytes, rate)
|
||||
self.wallet.send_points(reserved_points, point_amount)
|
||||
self.payment_rate_manager.record_points_paid(point_amount)
|
||||
|
||||
def _cancel_points(self, reserved_points):
|
||||
self.wallet.cancel_point_reservation(reserved_points)
|
||||
|
||||
def create_add_blob_request(self, client_blob_request):
|
||||
d = self.protocol.add_blob_request(client_blob_request)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d.addCallback(_handle_incoming_blob, self.peer, client_blob_request)
|
||||
d.addErrback(self._request_failed, "download request")
|
||||
|
||||
def _reserve_points(self, num_bytes):
|
||||
# jobevers: there was an assertion here, but I don't think it
|
||||
# was a valid assertion to make. It is possible for a rate to
|
||||
# not yet been set for this protocol or for it to have been
|
||||
# removed so instead I switched it to check if a rate has been set
|
||||
# and calculate it if it has not
|
||||
rate = self.get_rate()
|
||||
points_to_reserve = get_points(num_bytes, rate)
|
||||
return self.wallet.reserve_points(self.peer, points_to_reserve)
|
||||
|
||||
def _download_succeeded(self, arg, blob):
|
||||
log.info("Blob %s has been successfully downloaded from %s", blob, self.peer)
|
||||
self.update_local_score(5.0)
|
||||
self.peer.update_stats('blobs_downloaded', 1)
|
||||
self.peer.update_score(5.0)
|
||||
should_announce = blob.blob_hash == self.head_blob_hash
|
||||
d = self.requestor.blob_manager.blob_completed(blob, should_announce=should_announce)
|
||||
d.addCallback(lambda _: arg)
|
||||
return d
|
||||
|
||||
def _download_failed(self, reason):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self.update_local_score(-10.0)
|
||||
return reason
|
||||
|
||||
|
||||
class BlobDownloadDetails(object):
|
||||
"""Contains the information needed to make a ClientBlobRequest from an open blob"""
|
||||
def __init__(self, blob, deferred, write_func, cancel_func, peer):
|
||||
self.blob = blob
|
||||
self.deferred = deferred
|
||||
self.write_func = write_func
|
||||
self.cancel_func = cancel_func
|
||||
self.peer = peer
|
||||
|
||||
def counting_write_func(self, data):
|
||||
self.peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return self.write_func(data)
|
||||
@@ -1,286 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.protocols.policies import TimeoutMixin
|
||||
from twisted.python import failure
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
|
||||
from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
from lbrynet.interfaces import IRequestSender, IRateLimited
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_decimal(obj):
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
raise TypeError(repr(obj) + " is not JSON serializable")
|
||||
|
||||
|
||||
class ClientProtocol(Protocol, TimeoutMixin):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
######### Protocol #########
|
||||
PROTOCOL_TIMEOUT = 30
|
||||
|
||||
def connectionMade(self):
|
||||
log.debug("Connection made to %s", self.factory.peer)
|
||||
self._connection_manager = self.factory.connection_manager
|
||||
self._rate_limiter = self.factory.rate_limiter
|
||||
self.peer = self.factory.peer
|
||||
self._response_deferreds = {}
|
||||
self._response_buff = ''
|
||||
self._downloading_blob = False
|
||||
self._blob_download_request = None
|
||||
self._next_request = {}
|
||||
self.connection_closed = False
|
||||
self.connection_closing = False
|
||||
# This needs to be set for TimeoutMixin
|
||||
self.callLater = utils.call_later
|
||||
self.peer.report_up()
|
||||
|
||||
self._ask_for_request()
|
||||
|
||||
def dataReceived(self, data):
|
||||
log.debug("Received %d bytes from %s", len(data), self.peer)
|
||||
self.setTimeout(None)
|
||||
self._rate_limiter.report_dl_bytes(len(data))
|
||||
|
||||
if self._downloading_blob is True:
|
||||
self._blob_download_request.write(data)
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
|
||||
log.warning("Response is too large from %s. Size %s",
|
||||
self.peer, len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
response, extra_data = self._get_valid_response(self._response_buff)
|
||||
if response is not None:
|
||||
self._response_buff = ''
|
||||
self._handle_response(response)
|
||||
if self._downloading_blob is True and len(extra_data) != 0:
|
||||
self._blob_download_request.write(extra_data)
|
||||
|
||||
def timeoutConnection(self):
|
||||
log.info("Connection timed out to %s", self.peer)
|
||||
self.peer.report_down()
|
||||
self.transport.abortConnection()
|
||||
|
||||
def connectionLost(self, reason):
|
||||
log.debug("Connection lost to %s: %s", self.peer, reason)
|
||||
self.setTimeout(None)
|
||||
self.connection_closed = True
|
||||
if reason.check(error.ConnectionDone):
|
||||
err = failure.Failure(ConnectionClosedBeforeResponseError())
|
||||
else:
|
||||
err = reason
|
||||
for key, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
if self._blob_download_request is not None:
|
||||
self._blob_download_request.cancel(err)
|
||||
self.factory.connection_was_made_deferred.callback(True)
|
||||
|
||||
######### IRequestSender #########
|
||||
|
||||
def add_request(self, request):
|
||||
if request.response_identifier in self._response_deferreds:
|
||||
raise ValueError("There is already a request for that response active")
|
||||
self._next_request.update(request.request_dict)
|
||||
d = defer.Deferred()
|
||||
log.debug("Adding a request for %s. Request: %s", self.peer, request.request_dict)
|
||||
self._response_deferreds[request.response_identifier] = d
|
||||
return d
|
||||
|
||||
def add_blob_request(self, blob_request):
|
||||
if self._blob_download_request is None:
|
||||
d = self.add_request(blob_request)
|
||||
self._blob_download_request = blob_request
|
||||
blob_request.finished_deferred.addCallbacks(self._downloading_finished,
|
||||
self._handle_response_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(ValueError("There is already a blob download request active"))
|
||||
|
||||
def cancel_requests(self):
|
||||
self.connection_closing = True
|
||||
ds = []
|
||||
err = RequestCanceledError()
|
||||
for key, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
ds.append(d)
|
||||
if self._blob_download_request is not None:
|
||||
ds.append(self._blob_download_request.finished_deferred)
|
||||
self._blob_download_request.cancel(err)
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
######### Internal request handling #########
|
||||
|
||||
def _handle_request_error(self, err):
|
||||
log.error("An unexpected error occurred creating or sending a request to %s. %s: %s",
|
||||
self.peer, err.type, err.message)
|
||||
self.transport.loseConnection()
|
||||
|
||||
def _ask_for_request(self):
|
||||
if self.connection_closed is True or self.connection_closing is True:
|
||||
return
|
||||
|
||||
def send_request_or_close(do_request):
|
||||
if do_request is True:
|
||||
request_msg, self._next_request = self._next_request, {}
|
||||
self._send_request_message(request_msg)
|
||||
else:
|
||||
# The connection manager has indicated that this connection should be terminated
|
||||
log.debug("Closing the connection to %s due to having no further requests to send",
|
||||
self.peer)
|
||||
self.peer.report_success()
|
||||
self.transport.loseConnection()
|
||||
d = self._connection_manager.get_next_request(self.peer, self)
|
||||
d.addCallback(send_request_or_close)
|
||||
d.addErrback(self._handle_request_error)
|
||||
|
||||
def _send_request_message(self, request_msg):
|
||||
self.setTimeout(self.PROTOCOL_TIMEOUT)
|
||||
# TODO: compare this message to the last one. If they're the same,
|
||||
# TODO: incrementally delay this message.
|
||||
m = json.dumps(request_msg, default=encode_decimal)
|
||||
self.transport.write(m)
|
||||
|
||||
def _get_valid_response(self, response_msg):
|
||||
extra_data = None
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while 1:
|
||||
next_close_paren = response_msg.find('}', curr_pos)
|
||||
if next_close_paren != -1:
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
else:
|
||||
break
|
||||
return response, extra_data
|
||||
|
||||
def _handle_response_error(self, err):
|
||||
# If an error gets to this point, log it and kill the connection.
|
||||
if err.check(DownloadCanceledError, RequestCanceledError, error.ConnectionAborted,
|
||||
ConnectionClosedBeforeResponseError):
|
||||
# TODO: (wish-list) it seems silly to close the connection over this, and it shouldn't
|
||||
# TODO: always be this way. it's done this way now because the client has no other way
|
||||
# TODO: of telling the server it wants the download to stop. It would be great if the
|
||||
# TODO: protocol had such a mechanism.
|
||||
log.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
self.peer, self._blob_download_request.blob)
|
||||
result = None
|
||||
elif err.check(MisbehavingPeerError):
|
||||
log.warning("The connection to %s is closing due to: %s", self.peer, err)
|
||||
result = err
|
||||
else:
|
||||
log.error("The connection to %s is closing due to an unexpected error: %s",
|
||||
self.peer, err)
|
||||
result = err
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
self.transport.loseConnection()
|
||||
return result
|
||||
|
||||
def _handle_response(self, response):
|
||||
ds = []
|
||||
log.debug(
|
||||
"Handling a response from %s. Expected responses: %s. Actual responses: %s",
|
||||
self.peer, self._response_deferreds.keys(), response.keys())
|
||||
for key, val in response.items():
|
||||
if key in self._response_deferreds:
|
||||
d = self._response_deferreds.pop(key)
|
||||
d.callback({key: val})
|
||||
ds.append(d)
|
||||
for k, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[k]
|
||||
d.errback(failure.Failure(NoResponseError()))
|
||||
ds.append(d)
|
||||
|
||||
if self._blob_download_request is not None:
|
||||
self._downloading_blob = True
|
||||
d = self._blob_download_request.finished_deferred
|
||||
d.addErrback(self._handle_response_error)
|
||||
ds.append(d)
|
||||
|
||||
# TODO: are we sure we want to consume errors here
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
|
||||
def get_next_request(results):
|
||||
failed = False
|
||||
for success, result in results:
|
||||
if success is False:
|
||||
failed = True
|
||||
if not isinstance(result.value, DownloadCanceledError):
|
||||
log.info(result.value)
|
||||
log.info("The connection to %s is closing due to an error: %s",
|
||||
self.peer, result.getTraceback())
|
||||
|
||||
self.peer.report_down()
|
||||
if failed is False:
|
||||
log.debug("Asking for another request from %s", self.peer)
|
||||
self._ask_for_request()
|
||||
else:
|
||||
log.warning("Not asking for another request from %s", self.peer)
|
||||
self.transport.loseConnection()
|
||||
|
||||
dl.addCallback(get_next_request)
|
||||
|
||||
def _downloading_finished(self, arg):
|
||||
log.debug("The blob has finished downloading from %s", self.peer)
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return arg
|
||||
|
||||
######### IRateLimited #########
|
||||
|
||||
def throttle_upload(self):
|
||||
pass
|
||||
|
||||
def unthrottle_upload(self):
|
||||
pass
|
||||
|
||||
def throttle_download(self):
|
||||
self.transport.pauseProducing()
|
||||
|
||||
def unthrottle_download(self):
|
||||
self.transport.resumeProducing()
|
||||
|
||||
|
||||
class ClientProtocolFactory(ClientFactory):
|
||||
protocol = ClientProtocol
|
||||
|
||||
def __init__(self, peer, rate_limiter, connection_manager):
|
||||
self.peer = peer
|
||||
self.rate_limiter = rate_limiter
|
||||
self.connection_manager = connection_manager
|
||||
self.p = None
|
||||
# This defer fires and returns True when connection was
|
||||
# made and completed, or fires and returns False if
|
||||
# connection failed
|
||||
self.connection_was_made_deferred = defer.Deferred()
|
||||
|
||||
def clientConnectionFailed(self, connector, reason):
|
||||
log.debug("Connection failed to %s: %s", self.peer, reason)
|
||||
self.peer.report_down()
|
||||
self.connection_was_made_deferred.callback(False)
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = self.protocol()
|
||||
p.factory = self
|
||||
self.p = p
|
||||
return p
|
||||
@@ -1,27 +0,0 @@
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
|
||||
class ClientRequest(object):
|
||||
def __init__(self, request_dict, response_identifier=None):
|
||||
self.request_dict = request_dict
|
||||
self.response_identifier = response_identifier
|
||||
|
||||
|
||||
class ClientPaidRequest(ClientRequest):
|
||||
def __init__(self, request_dict, response_identifier, max_pay_units):
|
||||
ClientRequest.__init__(self, request_dict, response_identifier)
|
||||
self.max_pay_units = max_pay_units
|
||||
|
||||
|
||||
class ClientBlobRequest(ClientPaidRequest):
|
||||
def __init__(self, request_dict, response_identifier, write_func, finished_deferred,
|
||||
cancel_func, blob):
|
||||
if blob.length is None:
|
||||
max_pay_units = MAX_BLOB_SIZE
|
||||
else:
|
||||
max_pay_units = blob.length
|
||||
ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units)
|
||||
self.write = write_func
|
||||
self.finished_deferred = finished_deferred
|
||||
self.cancel = cancel_func
|
||||
self.blob = blob
|
||||
@@ -1,230 +0,0 @@
|
||||
import random
|
||||
import logging
|
||||
from twisted.internet import defer, reactor
|
||||
from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnectionHandler(object):
|
||||
def __init__(self, request_creators, factory):
|
||||
self.request_creators = request_creators
|
||||
self.factory = factory
|
||||
self.connection = None
|
||||
|
||||
|
||||
class ConnectionManager(object):
|
||||
implements(interfaces.IConnectionManager)
|
||||
MANAGE_CALL_INTERVAL_SEC = 5
|
||||
TCP_CONNECT_TIMEOUT = 15
|
||||
|
||||
def __init__(self, downloader, rate_limiter,
|
||||
primary_request_creators, secondary_request_creators):
|
||||
|
||||
self.seek_head_blob_first = conf.settings['seek_head_blob_first']
|
||||
self.max_connections_per_stream = conf.settings['max_connections_per_stream']
|
||||
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
self._primary_request_creators = primary_request_creators
|
||||
self._secondary_request_creators = secondary_request_creators
|
||||
self._peer_connections = {} # {Peer: PeerConnectionHandler}
|
||||
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self._next_manage_call = None
|
||||
# a deferred that gets fired when a _manage call is set
|
||||
self._manage_deferred = None
|
||||
self.stopped = True
|
||||
log.debug("%s initialized", self._get_log_name())
|
||||
|
||||
# this identifies what the connection manager is for,
|
||||
# used for logging purposes only
|
||||
def _get_log_name(self):
|
||||
out = 'Connection Manager Unknown'
|
||||
if hasattr(self.downloader, 'stream_name'):
|
||||
out = 'Connection Manager '+self.downloader.stream_name
|
||||
elif hasattr(self.downloader, 'blob_hash'):
|
||||
out = 'Connection Manager '+self.downloader.blob_hash
|
||||
return out
|
||||
|
||||
def _start(self):
|
||||
self.stopped = False
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
|
||||
def start(self):
|
||||
log.debug("%s starting", self._get_log_name())
|
||||
self._start()
|
||||
self._next_manage_call = utils.call_later(0, self.manage)
|
||||
return defer.succeed(True)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
log.debug("%s stopping", self._get_log_name())
|
||||
self.stopped = True
|
||||
# wait for the current manage call to finish
|
||||
if self._manage_deferred:
|
||||
yield self._manage_deferred
|
||||
# in case we stopped between manage calls, cancel the next one
|
||||
if self._next_manage_call and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
yield self._close_peers()
|
||||
|
||||
def num_peer_connections(self):
|
||||
return len(self._peer_connections)
|
||||
|
||||
def _close_peers(self):
|
||||
def disconnect_peer(p):
|
||||
d = defer.Deferred()
|
||||
self._connections_closing[p] = d
|
||||
self._peer_connections[p].connection.disconnect()
|
||||
if p in self._peer_connections:
|
||||
del self._peer_connections[p]
|
||||
return d
|
||||
|
||||
def close_connection(p):
|
||||
log.debug("%s Abruptly closing a connection to %s due to downloading being paused",
|
||||
self._get_log_name(), p)
|
||||
if self._peer_connections[p].factory.p is not None:
|
||||
d = self._peer_connections[p].factory.p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addBoth(lambda _: disconnect_peer(p))
|
||||
return d
|
||||
|
||||
closing_deferreds = [close_connection(peer) for peer in self._peer_connections.keys()]
|
||||
return defer.DeferredList(closing_deferreds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_next_request(self, peer, protocol):
|
||||
log.debug("%s Trying to get the next request for peer %s", self._get_log_name(), peer)
|
||||
if not peer in self._peer_connections or self.stopped is True:
|
||||
log.debug("%s The peer %s has already been told to shut down.",
|
||||
self._get_log_name(), peer)
|
||||
defer.returnValue(False)
|
||||
requests = yield self._send_primary_requests(peer, protocol)
|
||||
have_request = any(r[1] for r in requests if r[0] is True)
|
||||
if have_request:
|
||||
yield self._send_secondary_requests(peer, protocol)
|
||||
defer.returnValue(have_request)
|
||||
|
||||
def _send_primary_requests(self, peer, protocol):
|
||||
def handle_error(err):
|
||||
err.trap(InsufficientFundsError)
|
||||
self.downloader.insufficient_funds(err)
|
||||
return False
|
||||
|
||||
def check_if_request_sent(request_sent, request_creator):
|
||||
if peer not in self._peer_connections:
|
||||
# This can happen if the connection is told to close
|
||||
return False
|
||||
if request_sent is False:
|
||||
if request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.remove(request_creator)
|
||||
else:
|
||||
if not request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.append(request_creator)
|
||||
return request_sent
|
||||
|
||||
ds = []
|
||||
for p_r_c in self._primary_request_creators:
|
||||
d = p_r_c.send_next_request(peer, protocol)
|
||||
d.addErrback(handle_error)
|
||||
d.addCallback(check_if_request_sent, p_r_c)
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds, fireOnOneErrback=True)
|
||||
|
||||
def _send_secondary_requests(self, peer, protocol):
|
||||
ds = [
|
||||
s_r_c.send_next_request(peer, protocol)
|
||||
for s_r_c in self._secondary_request_creators
|
||||
]
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def manage(self, schedule_next_call=True):
|
||||
self._manage_deferred = defer.Deferred()
|
||||
if len(self._peer_connections) < self.max_connections_per_stream:
|
||||
log.debug("%s have %d connections, looking for %d",
|
||||
self._get_log_name(), len(self._peer_connections),
|
||||
self.max_connections_per_stream)
|
||||
peers = yield self._get_new_peers()
|
||||
for peer in peers:
|
||||
self._connect_to_peer(peer)
|
||||
self._manage_deferred.callback(None)
|
||||
self._manage_deferred = None
|
||||
if not self.stopped and schedule_next_call:
|
||||
self._next_manage_call = utils.call_later(self.MANAGE_CALL_INTERVAL_SEC, self.manage)
|
||||
|
||||
def return_shuffled_peers_not_connected_to(self, peers, new_conns_needed):
|
||||
out = [peer for peer in peers if peer not in self._peer_connections]
|
||||
random.shuffle(out)
|
||||
return out[0:new_conns_needed]
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _get_new_peers(self):
|
||||
new_conns_needed = self.max_connections_per_stream - len(self._peer_connections)
|
||||
if new_conns_needed < 1:
|
||||
defer.returnValue([])
|
||||
# we always get the peer from the first request creator
|
||||
# must be a type BlobRequester...
|
||||
request_creator = self._primary_request_creators[0]
|
||||
log.debug("%s Trying to get a new peer to connect to", self._get_log_name())
|
||||
|
||||
# find peers for the head blob if configured to do so
|
||||
if self.seek_head_blob_first:
|
||||
try:
|
||||
peers = yield request_creator.get_new_peers_for_head_blob()
|
||||
peers = self.return_shuffled_peers_not_connected_to(peers, new_conns_needed)
|
||||
except KeyError:
|
||||
log.warning("%s does not have a head blob", self._get_log_name())
|
||||
peers = []
|
||||
else:
|
||||
peers = []
|
||||
|
||||
# we didn't find any new peers on the head blob,
|
||||
# we have to look for the first unavailable blob
|
||||
if not peers:
|
||||
peers = yield request_creator.get_new_peers_for_next_unavailable()
|
||||
peers = self.return_shuffled_peers_not_connected_to(peers, new_conns_needed)
|
||||
|
||||
log.debug("%s Got a list of peers to choose from: %s",
|
||||
self._get_log_name(), peers)
|
||||
log.debug("%s Current connections: %s",
|
||||
self._get_log_name(), self._peer_connections.keys())
|
||||
log.debug("%s List of connection states: %s", self._get_log_name(),
|
||||
[p_c_h.connection.state for p_c_h in self._peer_connections.values()])
|
||||
defer.returnValue(peers)
|
||||
|
||||
def _connect_to_peer(self, peer):
|
||||
if self.stopped:
|
||||
return
|
||||
|
||||
log.debug("%s Trying to connect to %s", self._get_log_name(), peer)
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
factory.connection_was_made_deferred.addCallback(
|
||||
lambda c_was_made: self._peer_disconnected(c_was_made, peer))
|
||||
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
|
||||
factory)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory,
|
||||
timeout=self.TCP_CONNECT_TIMEOUT)
|
||||
self._peer_connections[peer].connection = connection
|
||||
|
||||
def _peer_disconnected(self, connection_was_made, peer):
|
||||
log.debug("%s protocol disconnected for %s",
|
||||
self._get_log_name(), peer)
|
||||
if peer in self._peer_connections:
|
||||
del self._peer_connections[peer]
|
||||
if peer in self._connections_closing:
|
||||
d = self._connections_closing[peer]
|
||||
del self._connections_closing[peer]
|
||||
d.callback(True)
|
||||
return connection_was_made
|
||||
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadManager(object):
|
||||
implements(interfaces.IDownloadManager)
|
||||
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.blob_info_finder = None
|
||||
self.progress_manager = None
|
||||
self.blob_handler = None
|
||||
self.connection_manager = None
|
||||
self.blobs = {}
|
||||
self.blob_infos = {}
|
||||
|
||||
######### IDownloadManager #########
|
||||
|
||||
def start_downloading(self):
|
||||
d = self.blob_info_finder.get_initial_blobs()
|
||||
log.debug("Requested the initial blobs from the info finder")
|
||||
d.addCallback(self.add_blobs_to_download)
|
||||
d.addCallback(lambda _: self.resume_downloading())
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def resume_downloading(self):
|
||||
yield self.connection_manager.start()
|
||||
yield self.progress_manager.start()
|
||||
defer.returnValue(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop_downloading(self):
|
||||
yield self.progress_manager.stop()
|
||||
yield self.connection_manager.stop()
|
||||
defer.returnValue(True)
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
log.debug(
|
||||
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
|
||||
|
||||
def error_during_add(err):
|
||||
log.warning(
|
||||
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
log.debug(
|
||||
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
|
||||
def stream_position(self):
|
||||
return self.progress_manager.stream_position()
|
||||
|
||||
def needed_blobs(self):
|
||||
return self.progress_manager.needed_blobs()
|
||||
|
||||
def final_blob_num(self):
|
||||
return self.blob_info_finder.final_blob_num()
|
||||
|
||||
def handle_blob(self, blob_num):
|
||||
return self.blob_handler.handle_blob(self.blobs[blob_num], self.blob_infos[blob_num])
|
||||
|
||||
def calculate_total_bytes(self):
|
||||
return sum([bi.length for bi in self.blob_infos.itervalues()])
|
||||
|
||||
def calculate_bytes_left_to_output(self):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
to_be_outputted = [
|
||||
b for n, b in self.blobs.iteritems()
|
||||
if n >= self.progress_manager.last_blob_outputted
|
||||
]
|
||||
return sum([b.length for b in to_be_outputted if b.length is not None])
|
||||
|
||||
def calculate_bytes_left_to_download(self):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
|
||||
def get_head_blob_hash(self):
|
||||
return self.blobs[0].blob_hash
|
||||
@@ -1,144 +0,0 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.Error import InvalidBlobHashError, DownloadSDTimeout
|
||||
from lbrynet.core.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleBlobMetadataHandler(object):
|
||||
implements(interfaces.IMetadataHandler)
|
||||
|
||||
def __init__(self, blob_hash, download_manager):
|
||||
self.blob_hash = blob_hash
|
||||
self.download_manager = download_manager
|
||||
|
||||
######## IMetadataHandler #########
|
||||
|
||||
def get_initial_blobs(self):
|
||||
log.debug("Returning the blob info")
|
||||
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
|
||||
|
||||
def final_blob_num(self):
|
||||
return 0
|
||||
|
||||
|
||||
class SingleProgressManager(object):
|
||||
def __init__(self, download_manager, finished_callback, timeout_callback, timeout):
|
||||
self.finished_callback = finished_callback
|
||||
self.timeout_callback = timeout_callback
|
||||
self.download_manager = download_manager
|
||||
|
||||
self.timeout = timeout
|
||||
self.timeout_counter = 0
|
||||
self.checker = LoopingCall(self._check_if_finished)
|
||||
|
||||
def start(self):
|
||||
safe_start_looping_call(self.checker, 1)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
safe_stop_looping_call(self.checker)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _check_if_finished(self):
|
||||
if self.stream_position() == 1:
|
||||
blob_downloaded = self.download_manager.blobs[0]
|
||||
log.debug("The blob %s has been downloaded. Calling the finished callback",
|
||||
str(blob_downloaded))
|
||||
safe_stop_looping_call(self.checker)
|
||||
self.finished_callback(blob_downloaded)
|
||||
elif self.timeout is not None:
|
||||
self.timeout_counter += 1
|
||||
if self.timeout_counter >= self.timeout:
|
||||
safe_stop_looping_call(self.checker)
|
||||
self.timeout_callback()
|
||||
|
||||
def stream_position(self):
|
||||
blobs = self.download_manager.blobs
|
||||
if blobs and blobs[0].get_is_verified():
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def needed_blobs(self):
|
||||
blobs = self.download_manager.blobs
|
||||
assert len(blobs) == 1
|
||||
return [b for b in blobs.itervalues() if not b.get_is_verified()]
|
||||
|
||||
|
||||
class DummyBlobHandler(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def handle_blob(self, blob, blob_info):
|
||||
pass
|
||||
|
||||
|
||||
class StandaloneBlobDownloader(object):
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, wallet,
|
||||
timeout=None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.timeout = timeout
|
||||
self.download_manager = None
|
||||
self.finished_deferred = None
|
||||
|
||||
def download(self):
|
||||
if not is_valid_blobhash(self.blob_hash):
|
||||
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
|
||||
|
||||
def cancel_download(d):
|
||||
self.stop()
|
||||
|
||||
self.finished_deferred = defer.Deferred(canceller=cancel_download)
|
||||
self.download_manager = DownloadManager(self.blob_manager)
|
||||
self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
self.download_manager)
|
||||
self.download_manager.blob_info_finder = SingleBlobMetadataHandler(self.blob_hash,
|
||||
self.download_manager)
|
||||
self.download_manager.progress_manager = SingleProgressManager(self.download_manager,
|
||||
self._blob_downloaded,
|
||||
self._download_timedout,
|
||||
self.timeout)
|
||||
self.download_manager.blob_handler = DummyBlobHandler()
|
||||
self.download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
|
||||
self.download_manager.connection_manager = ConnectionManager(
|
||||
self, self.rate_limiter,
|
||||
[self.download_manager.blob_requester],
|
||||
[self.download_manager.wallet_info_exchanger]
|
||||
)
|
||||
d = self.download_manager.start_downloading()
|
||||
d.addCallback(lambda _: self.finished_deferred)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
return self.download_manager.stop_downloading()
|
||||
|
||||
def _blob_downloaded(self, blob):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.callback(blob)
|
||||
|
||||
def _download_timedout(self):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(DownloadSDTimeout(self.blob_hash))
|
||||
|
||||
def insufficient_funds(self, err):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(err)
|
||||
@@ -1,167 +0,0 @@
|
||||
import logging
|
||||
from lbrynet.interfaces import IProgressManager
|
||||
from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamProgressManager(object):
|
||||
implements(IProgressManager)
|
||||
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
self.finished_callback = finished_callback
|
||||
self.blob_manager = blob_manager
|
||||
self.delete_blob_after_finished = delete_blob_after_finished
|
||||
self.download_manager = download_manager
|
||||
self.provided_blob_nums = []
|
||||
self.last_blob_outputted = -1
|
||||
self.stopped = True
|
||||
self._next_try_to_output_call = None
|
||||
self.outputting_d = None
|
||||
|
||||
######### IProgressManager #########
|
||||
|
||||
def start(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.stopped = False
|
||||
self._next_try_to_output_call = reactor.callLater(0, self._try_to_output)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
self.stopped = True
|
||||
if self._next_try_to_output_call is not None and self._next_try_to_output_call.active():
|
||||
self._next_try_to_output_call.cancel()
|
||||
self._next_try_to_output_call = None
|
||||
return self._stop_outputting()
|
||||
|
||||
def blob_downloaded(self, blob, blob_num):
|
||||
if self.outputting_d is None:
|
||||
self._output_loop()
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _finished_outputting(self):
|
||||
self.finished_callback(True)
|
||||
|
||||
def _try_to_output(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self._next_try_to_output_call = reactor.callLater(1, self._try_to_output)
|
||||
if self.outputting_d is None:
|
||||
self._output_loop()
|
||||
|
||||
def _output_loop(self):
|
||||
pass
|
||||
|
||||
def _stop_outputting(self):
|
||||
if self.outputting_d is not None:
|
||||
return self.outputting_d
|
||||
return defer.succeed(None)
|
||||
|
||||
def _finished_with_blob(self, blob_num):
|
||||
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
if self.delete_blob_after_finished is True:
|
||||
log.debug("delete_blob_after_finished is True")
|
||||
blobs = self.download_manager.blobs
|
||||
if blob_num in blobs:
|
||||
log.debug("Telling the blob manager, %s, to delete blob %s",
|
||||
self.blob_manager, blobs[blob_num].blob_hash)
|
||||
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
|
||||
else:
|
||||
log.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
else:
|
||||
log.debug("delete_blob_after_finished is False")
|
||||
|
||||
|
||||
class FullStreamProgressManager(StreamProgressManager):
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
StreamProgressManager.__init__(self, finished_callback, blob_manager, download_manager,
|
||||
delete_blob_after_finished)
|
||||
self.outputting_d = None
|
||||
|
||||
######### IProgressManager #########
|
||||
|
||||
def _done(self, i, blobs):
|
||||
"""Return true if `i` is a blob number we don't have"""
|
||||
return (
|
||||
i not in blobs or
|
||||
(
|
||||
not blobs[i].get_is_verified() and
|
||||
i not in self.provided_blob_nums
|
||||
)
|
||||
)
|
||||
|
||||
def stream_position(self):
|
||||
blobs = self.download_manager.blobs
|
||||
if not blobs:
|
||||
return 0
|
||||
else:
|
||||
for i in xrange(max(blobs.iterkeys())):
|
||||
if self._done(i, blobs):
|
||||
return i
|
||||
return max(blobs.iterkeys()) + 1
|
||||
|
||||
def needed_blobs(self):
|
||||
blobs = self.download_manager.blobs
|
||||
return [
|
||||
b for n, b in blobs.iteritems()
|
||||
if not b.get_is_verified() and not n in self.provided_blob_nums
|
||||
]
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _output_loop(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.stopped:
|
||||
if self.outputting_d is not None:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
return
|
||||
|
||||
if self.outputting_d is None:
|
||||
self.outputting_d = defer.Deferred()
|
||||
blobs = self.download_manager.blobs
|
||||
|
||||
def finished_outputting_blob():
|
||||
self.last_blob_outputted += 1
|
||||
|
||||
def check_if_finished():
|
||||
final_blob_num = self.download_manager.final_blob_num()
|
||||
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
|
||||
self._finished_outputting()
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
else:
|
||||
reactor.callLater(0, self._output_loop)
|
||||
|
||||
current_blob_num = self.last_blob_outputted + 1
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].get_is_verified():
|
||||
log.debug("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
self.provided_blob_nums.append(self.last_blob_outputted + 1)
|
||||
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
d.addCallback(lambda _: check_if_finished())
|
||||
|
||||
def log_error(err):
|
||||
log.warning("Error outputting blob %s: %s", blobs[current_blob_num].blob_hash,
|
||||
err.getErrorMessage())
|
||||
if self.outputting_d is not None and not self.outputting_d.called:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
self.stop()
|
||||
|
||||
d.addErrback(log_error)
|
||||
else:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
@@ -1,5 +0,0 @@
|
||||
import hashlib
|
||||
|
||||
|
||||
def get_lbry_hash_obj():
|
||||
return hashlib.sha384()
|
||||
@@ -1,17 +0,0 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_read_handle(path):
|
||||
"""
|
||||
Get os independent read handle for a file
|
||||
"""
|
||||
|
||||
if os.name == "nt":
|
||||
file_mode = 'rb'
|
||||
else:
|
||||
file_mode = 'r'
|
||||
read_handle = open(path, file_mode)
|
||||
yield read_handle
|
||||
read_handle.close()
|
||||
@@ -1,380 +0,0 @@
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer
|
||||
import twisted.python.log
|
||||
|
||||
from lbrynet import __version__ as lbrynet_version, build_type, conf
|
||||
from lbrynet.core import utils
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
#####
|
||||
|
||||
|
||||
TRACE = 5
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
logging.Handler.__init__(self)
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
response = yield treq.post(self.url, data=payload, cookies=self.cookies)
|
||||
self.cookies.update(response.cookies())
|
||||
|
||||
def emit(self, record):
|
||||
return self._emit(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
for handler in log.handlers:
|
||||
if handler.name == handler_name:
|
||||
log.removeHandler(handler)
|
||||
|
||||
|
||||
def _log_decorator(fn):
|
||||
"""Configure a logging handler.
|
||||
|
||||
`fn` is a function that returns a logging handler. The returned
|
||||
handler has its log-level set and is attached to the specified
|
||||
logger or the root logger.
|
||||
"""
|
||||
|
||||
def helper(*args, **kwargs):
|
||||
log = kwargs.pop('log', logging.getLogger())
|
||||
level = kwargs.pop('level', logging.INFO)
|
||||
if not isinstance(level, int):
|
||||
# despite the name, getLevelName returns
|
||||
# the numeric level when passed a text level
|
||||
level = logging.getLevelName(level)
|
||||
handler = fn(*args, **kwargs)
|
||||
configure_handler(handler, log, level)
|
||||
return handler
|
||||
|
||||
return helper
|
||||
|
||||
|
||||
def configure_handler(handler, log, level):
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
return handler
|
||||
|
||||
|
||||
def disable_third_party_loggers():
|
||||
logging.getLogger('requests').setLevel(logging.CRITICAL)
|
||||
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
|
||||
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
|
||||
logging.getLogger('lbryum').setLevel(logging.WARNING)
|
||||
logging.getLogger('twisted').setLevel(logging.CRITICAL)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_console(**kwargs):
|
||||
"""Convenience function to configure a log-handler that outputs to stdout"""
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'console'
|
||||
return handler
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_file_handler(file_name, **kwargs):
|
||||
"""Convenience function to configure a log-handler that writes to `file_name`"""
|
||||
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'file'
|
||||
return handler
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
|
||||
version = version or lbrynet_version
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
def configure_loggly_handler():
|
||||
if build_type.BUILD == 'dev':
|
||||
return
|
||||
level = logging.ERROR
|
||||
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
|
||||
session_id=conf.settings.get_session_id())
|
||||
log = logging.getLogger("lbrynet")
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
|
||||
|
||||
def get_loggly_handler(level, installation_id, session_id):
|
||||
formatter = JsonFormatter(level=level, installation_id=installation_id, session_id=session_id)
|
||||
handler = HTTPSHandler(get_loggly_url())
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
|
||||
def failure(failure, log, msg, *args):
|
||||
"""Log a failure message from a deferred.
|
||||
|
||||
Args:
|
||||
failure: twisted.python.failure.Failure
|
||||
log: a python logger instance
|
||||
msg: the message to log. Can use normal logging string interpolation.
|
||||
the last argument will be set to the error message from the failure.
|
||||
args: values to substitute into `msg`
|
||||
"""
|
||||
args += (failure.getErrorMessage(),)
|
||||
exc_info = (failure.type, failure.value, failure.getTracebackObject())
|
||||
log.error(msg, *args, exc_info=exc_info)
|
||||
|
||||
|
||||
def convert_verbose(verbose):
|
||||
"""Convert the results of the --verbose flag into a list of logger names
|
||||
|
||||
if --verbose is not provided, args.verbose will be None and logging
|
||||
should be at the info level.
|
||||
if --verbose is provided, but not followed by any arguments, then
|
||||
args.verbose = [] and debug logging should be enabled for all of lbrynet
|
||||
along with info logging on lbryum.
|
||||
if --verbose is provided and followed by arguments, those arguments
|
||||
will be in a list
|
||||
"""
|
||||
if verbose is None:
|
||||
return []
|
||||
if verbose == []:
|
||||
return ['lbrynet', 'lbryum']
|
||||
return verbose
|
||||
|
||||
|
||||
def configure_logging(file_name, console, verbose=None):
|
||||
"""Apply the default logging configuration.
|
||||
|
||||
Enables two log-handlers at the INFO level: a file logger and a loggly logger.
|
||||
Optionally turns on a console logger that defaults to the INFO level, with
|
||||
specified loggers being set to the DEBUG level.
|
||||
|
||||
Args:
|
||||
file_name: the file to which logs should be saved
|
||||
console: If true, enable a console logger
|
||||
verbose: a list of loggers to set to debug level.
|
||||
See `convert_verbose` for more details.
|
||||
"""
|
||||
verbose = convert_verbose(verbose)
|
||||
configure_twisted()
|
||||
configure_file_handler(file_name)
|
||||
disable_third_party_loggers()
|
||||
if console:
|
||||
# if there are some loggers at the debug level, we need
|
||||
# to enable the console to allow debug. Otherwise, only
|
||||
# allow info.
|
||||
level = 'DEBUG' if verbose else 'INFO'
|
||||
handler = configure_console(level=level)
|
||||
if 'lbryum' in verbose:
|
||||
# TODO: this enables lbryum logging on the other handlers
|
||||
# too which isn't consistent with how verbose logging
|
||||
# happens with other loggers. Should change the configuration
|
||||
# so that its only logging at the INFO level for the console.
|
||||
logging.getLogger('lbryum').setLevel(logging.INFO)
|
||||
verbose.remove('lbryum')
|
||||
if verbose:
|
||||
handler.addFilter(LoggerNameFilter(verbose))
|
||||
|
||||
|
||||
def configure_twisted():
|
||||
"""Setup twisted logging to output events to the python stdlib logger"""
|
||||
# I tried using the new logging api
|
||||
# https://twistedmatrix.com/documents/current/core/howto/logger.html#compatibility-with-standard-library-logging
|
||||
# and it simply didn't work
|
||||
observer = twisted.python.log.PythonLoggingObserver()
|
||||
observer.start()
|
||||
|
||||
|
||||
class LoggerNameFilter(object):
|
||||
"""Filter a log record based on its name.
|
||||
|
||||
Allows all info level and higher records to pass thru.
|
||||
Debug records pass if the log record name (or a parent) match
|
||||
the input list of logger names.
|
||||
"""
|
||||
|
||||
def __init__(self, logger_names):
|
||||
self.logger_names = logger_names
|
||||
|
||||
def filter(self, record):
|
||||
if record.levelno >= logging.INFO:
|
||||
return True
|
||||
name = record.name
|
||||
while name:
|
||||
if name in self.logger_names:
|
||||
return True
|
||||
name = get_parent(name)
|
||||
return False
|
||||
|
||||
|
||||
def get_parent(logger_name):
|
||||
names = logger_name.split('.')
|
||||
if len(names) == 1:
|
||||
return ''
|
||||
names = names[:-1]
|
||||
return '.'.join(names)
|
||||
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to subtitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
@@ -1,20 +0,0 @@
|
||||
class LoopingCallManager(object):
|
||||
def __init__(self, calls=None):
|
||||
self.calls = calls or {}
|
||||
|
||||
def register_looping_call(self, name, call):
|
||||
assert name not in self.calls, '{} is already registered'.format(name)
|
||||
self.calls[name] = call
|
||||
|
||||
def start(self, name, *args):
|
||||
lcall = self.calls[name]
|
||||
if not lcall.running:
|
||||
lcall.start(*args)
|
||||
|
||||
def stop(self, name):
|
||||
self.calls[name].stop()
|
||||
|
||||
def shutdown(self):
|
||||
for lcall in self.calls.itervalues():
|
||||
if lcall.running:
|
||||
lcall.stop()
|
||||
@@ -1,59 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = BlobAvailabilityHandler(self.blob_manager)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'requested_blobs'
|
||||
|
||||
def get_description(self):
|
||||
return "Blob Availability - blobs that are available to be uploaded"
|
||||
|
||||
|
||||
class BlobAvailabilityHandler(object):
|
||||
implements(IQueryHandler)
|
||||
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.query_identifiers = ['requested_blobs']
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
return d
|
||||
return defer.succeed({})
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
return d
|
||||
@@ -1,229 +0,0 @@
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet import analytics
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequestHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = BlobRequestHandler(
|
||||
self.blob_manager, self.wallet, self.payment_rate_manager, self.analytics_manager)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'requested_blob'
|
||||
|
||||
def get_description(self):
|
||||
return "Blob Uploader - uploads blobs"
|
||||
|
||||
|
||||
class BlobRequestHandler(object):
|
||||
implements(IQueryHandler, IBlobSender)
|
||||
PAYMENT_RATE_QUERY = 'blob_data_payment_rate'
|
||||
BLOB_QUERY = 'requested_blob'
|
||||
AVAILABILITY_QUERY = 'requested_blobs'
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = [self.PAYMENT_RATE_QUERY, self.BLOB_QUERY, self.AVAILABILITY_QUERY]
|
||||
self.analytics_manager = analytics_manager
|
||||
self.peer = None
|
||||
self.blob_data_payment_rate = None
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
self.blob_bytes_uploaded = 0
|
||||
self._blobs_requested = []
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
self.peer = peer
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = defer.succeed({})
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
|
||||
def send_blob_if_requested(self, consumer):
|
||||
if self.currently_uploading is not None:
|
||||
return self.send_file(consumer)
|
||||
return defer.succeed(True)
|
||||
|
||||
def cancel_send(self, err):
|
||||
if self.currently_uploading is not None:
|
||||
self.read_handle.close()
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
return err
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
log.debug("Offered rate %f LBC/mb for %i blobs", offer.rate, len(blobs))
|
||||
reply = self.payment_rate_manager.reply_to_offer(self.peer, blobs, offer)
|
||||
if reply.is_accepted:
|
||||
self.blob_data_payment_rate = offer.rate
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_ACCEPTED"
|
||||
log.debug("Accepted rate: %f", offer.rate)
|
||||
elif reply.is_too_low:
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_TOO_LOW"
|
||||
log.debug("Reject rate: %f", offer.rate)
|
||||
elif reply.is_unset:
|
||||
log.warning("Rate unset")
|
||||
request['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
log.debug("Returning rate query result: %s", str(request))
|
||||
|
||||
return request
|
||||
|
||||
def _handle_blob_query(self, response, query):
|
||||
log.debug("Received the client's request to send a blob")
|
||||
response['incoming_blob'] = {}
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response['incoming_blob'] = {'error': "RATE_UNSET"}
|
||||
return response
|
||||
else:
|
||||
return self._send_blob(response, query)
|
||||
|
||||
def _send_blob(self, response, query):
|
||||
d = self.blob_manager.get_blob(query)
|
||||
d.addCallback(self.open_blob_for_reading, response)
|
||||
return d
|
||||
|
||||
def open_blob_for_reading(self, blob, response):
|
||||
response_fields = {}
|
||||
d = defer.succeed(None)
|
||||
if blob.get_is_verified():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
log.info("Sending %s to %s", str(blob), self.peer)
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
response['incoming_blob'] = response_fields
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
log.debug("We can not send %s", str(blob))
|
||||
response['incoming_blob'] = {'error': 'BLOB_UNAVAILABLE'}
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
|
||||
def _reply_to_send_request(self, response, incoming):
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
log.debug("Rate not set yet")
|
||||
response['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
log.debug("Requested blob: %s", str(incoming))
|
||||
d = self.blob_manager.get_blob(incoming)
|
||||
d.addCallback(lambda blob: self.open_blob_for_reading(blob, response))
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
def _send_file():
|
||||
inner_d = start_transfer()
|
||||
# TODO: if the transfer fails, check if it's because the connection was cut off.
|
||||
# TODO: if so, perhaps bill the client
|
||||
inner_d.addCallback(lambda _: set_expected_payment())
|
||||
inner_d.addBoth(set_not_uploading)
|
||||
return inner_d
|
||||
|
||||
def count_bytes(data):
|
||||
uploaded = len(data)
|
||||
self.blob_bytes_uploaded += uploaded
|
||||
self.peer.update_stats('blob_bytes_uploaded', uploaded)
|
||||
if self.analytics_manager is not None:
|
||||
self.analytics_manager.add_observation(analytics.BLOB_BYTES_UPLOADED, uploaded)
|
||||
return data
|
||||
|
||||
def start_transfer():
|
||||
self.file_sender = FileSender()
|
||||
log.debug("Starting the file upload")
|
||||
assert self.read_handle is not None, \
|
||||
"self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
|
||||
return d
|
||||
|
||||
def set_expected_payment():
|
||||
log.debug("Setting expected payment")
|
||||
if (
|
||||
self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None
|
||||
and self.blob_data_payment_rate > 0
|
||||
):
|
||||
# TODO: explain why 2**20
|
||||
self.wallet.add_expected_payment(self.peer,
|
||||
self.currently_uploading.length * 1.0 *
|
||||
self.blob_data_payment_rate / 2 ** 20)
|
||||
self.blob_bytes_uploaded = 0
|
||||
self.peer.update_stats('blobs_uploaded', 1)
|
||||
return None
|
||||
|
||||
def set_not_uploading(reason=None):
|
||||
if self.currently_uploading is not None:
|
||||
self.read_handle.close()
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
if reason is not None and isinstance(reason, Failure):
|
||||
log.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
return _send_file()
|
||||
@@ -1,93 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import interfaces, error
|
||||
from twisted.internet.protocol import Protocol, ServerFactory
|
||||
from twisted.python import failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerProtocol(Protocol):
|
||||
"""ServerProtocol needs to:
|
||||
|
||||
1) Receive requests from its transport
|
||||
2) Pass those requests on to its request handler
|
||||
3) Tell the request handler to pause/resume producing
|
||||
4) Tell its transport to pause/resume producing
|
||||
5) Hang up when the request handler is done producing
|
||||
6) Tell the request handler to stop producing if the connection is lost
|
||||
7) Upon creation, register with the rate limiter
|
||||
8) Upon connection loss, unregister with the rate limiter
|
||||
9) Report all uploaded and downloaded bytes to the rate limiter
|
||||
10) Pause/resume production when told by the rate limiter
|
||||
"""
|
||||
|
||||
implements(interfaces.IConsumer)
|
||||
|
||||
#Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
log.debug("Got a connection")
|
||||
peer_info = self.transport.getPeer()
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.request_handler = ServerRequestHandler(self)
|
||||
for query_handler_factory in self.factory.query_handler_factories.values():
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
log.debug("Setting the request handler")
|
||||
self.factory.rate_limiter.register_protocol(self)
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.stopProducing()
|
||||
self.factory.rate_limiter.unregister_protocol(self)
|
||||
if not reason.check(error.ConnectionDone):
|
||||
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
def dataReceived(self, data):
|
||||
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
self.factory.rate_limiter.report_dl_bytes(len(data))
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.data_received(data)
|
||||
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
log.debug("Registering the producer")
|
||||
assert streaming is True
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.request_handler = None
|
||||
self.transport.loseConnection()
|
||||
|
||||
def write(self, data):
|
||||
log.trace("Writing %s bytes of data to the transport", len(data))
|
||||
self.transport.write(data)
|
||||
self.factory.rate_limiter.report_ul_bytes(len(data))
|
||||
|
||||
#Rate limiter stuff
|
||||
|
||||
def throttle_upload(self):
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.pauseProducing()
|
||||
|
||||
def unthrottle_upload(self):
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.resumeProducing()
|
||||
|
||||
def throttle_download(self):
|
||||
self.transport.pauseProducing()
|
||||
|
||||
def unthrottle_download(self):
|
||||
self.transport.resumeProducing()
|
||||
|
||||
|
||||
class ServerProtocolFactory(ServerFactory):
|
||||
protocol = ServerProtocol
|
||||
|
||||
def __init__(self, rate_limiter, query_handler_factories, peer_manager):
|
||||
self.rate_limiter = rate_limiter
|
||||
self.query_handler_factories = query_handler_factories
|
||||
self.peer_manager = peer_manager
|
||||
@@ -1,186 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and
|
||||
return request for information about more blobs that are
|
||||
associated with streams.
|
||||
"""
|
||||
implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
|
||||
|
||||
def __init__(self, consumer):
|
||||
self.consumer = consumer
|
||||
self.production_paused = False
|
||||
self.request_buff = ''
|
||||
self.response_buff = ''
|
||||
self.producer = None
|
||||
self.request_received = False
|
||||
self.CHUNK_SIZE = 2**14
|
||||
self.query_handlers = {} # {IQueryHandler: [query_identifiers]}
|
||||
self.blob_sender = None
|
||||
self.consumer.registerProducer(self, True)
|
||||
|
||||
#IPushProducer stuff
|
||||
|
||||
def pauseProducing(self):
|
||||
self.production_paused = True
|
||||
|
||||
def stopProducing(self):
|
||||
if self.producer is not None:
|
||||
self.producer.stopProducing()
|
||||
self.producer = None
|
||||
self.production_paused = True
|
||||
self.consumer.unregisterProducer()
|
||||
|
||||
def resumeProducing(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.production_paused = False
|
||||
self._produce_more()
|
||||
if self.producer is not None:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def _produce_more(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.production_paused:
|
||||
return
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk == '':
|
||||
return
|
||||
log.trace("writing %s bytes to the client", len(chunk))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
self.producer = producer
|
||||
assert streaming is False
|
||||
producer.resumeProducing()
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.producer = None
|
||||
|
||||
def write(self, data):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.response_buff = self.response_buff + data
|
||||
self._produce_more()
|
||||
|
||||
def get_more_data():
|
||||
if self.producer is not None:
|
||||
log.trace("Requesting more data from the producer")
|
||||
self.producer.resumeProducing()
|
||||
|
||||
reactor.callLater(0, get_more_data)
|
||||
|
||||
#From Protocol
|
||||
|
||||
def data_received(self, data):
|
||||
log.debug("Received data")
|
||||
log.debug("%s", str(data))
|
||||
if self.request_received is False:
|
||||
return self._parse_data_and_maybe_send_blob(data)
|
||||
else:
|
||||
log.warning(
|
||||
"The client sent data when we were uploading a file. This should not happen")
|
||||
|
||||
def _parse_data_and_maybe_send_blob(self, data):
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg:
|
||||
self.request_buff = ''
|
||||
self._process_msg(msg)
|
||||
else:
|
||||
log.debug("Request buff not a valid json message")
|
||||
log.debug("Request buff: %s", self.request_buff)
|
||||
|
||||
def _process_msg(self, msg):
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
|
||||
|
||||
######### IRequestHandler #########
|
||||
|
||||
def register_query_handler(self, query_handler, query_identifiers):
|
||||
self.query_handlers[query_handler] = query_identifiers
|
||||
|
||||
def register_blob_sender(self, blob_sender):
|
||||
self.blob_sender = blob_sender
|
||||
|
||||
#response handling
|
||||
|
||||
def request_failure_handler(self, err):
|
||||
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
self.stopProducing()
|
||||
return err
|
||||
|
||||
def finished_response(self):
|
||||
self.request_received = False
|
||||
self._produce_more()
|
||||
|
||||
def send_response(self, msg):
|
||||
m = json.dumps(msg)
|
||||
log.debug("Sending a response of length %s", str(len(m)))
|
||||
log.debug("Response: %s", str(m))
|
||||
self.response_buff = self.response_buff + m
|
||||
self._produce_more()
|
||||
return True
|
||||
|
||||
def handle_request(self, msg):
|
||||
log.debug("Handling a request")
|
||||
log.debug(str(msg))
|
||||
|
||||
def create_response_message(results):
|
||||
response = {}
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
response.update(result)
|
||||
else:
|
||||
# result is a Failure
|
||||
return result
|
||||
log.debug("Finished making the response message. Response: %s", str(response))
|
||||
return response
|
||||
|
||||
def log_errors(err):
|
||||
log.warning(
|
||||
"An error occurred handling a client request. Error message: %s",
|
||||
err.getErrorMessage())
|
||||
return err
|
||||
|
||||
def send_response(response):
|
||||
self.send_response(response)
|
||||
return True
|
||||
|
||||
ds = []
|
||||
for query_handler, query_identifiers in self.query_handlers.iteritems():
|
||||
queries = {q_i: msg[q_i] for q_i in query_identifiers if q_i in msg}
|
||||
d = query_handler.handle_queries(queries)
|
||||
d.addErrback(log_errors)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(create_response_message)
|
||||
dl.addCallback(send_response)
|
||||
return dl
|
||||
|
||||
def try_to_parse_request(self, request_buff):
|
||||
try:
|
||||
msg = json.loads(request_buff)
|
||||
return msg
|
||||
except ValueError:
|
||||
return None
|
||||
@@ -1,54 +0,0 @@
|
||||
import platform
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
from urllib2 import urlopen, URLError
|
||||
from lbryschema import __version__ as lbryschema_version
|
||||
from lbryum import __version__ as LBRYUM_VERSION
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from lbrynet.conf import ROOT_DIR
|
||||
|
||||
|
||||
def get_lbrynet_version():
|
||||
if build_type.BUILD == "dev":
|
||||
try:
|
||||
with open(os.devnull, 'w') as devnull:
|
||||
git_dir = ROOT_DIR + '/.git'
|
||||
return subprocess.check_output(
|
||||
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
|
||||
stderr=devnull
|
||||
).strip().lstrip('v')
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
print "failed to get version from git"
|
||||
return lbrynet_version
|
||||
|
||||
|
||||
def get_platform(get_ip=True):
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": get_lbrynet_version(),
|
||||
"lbryum_version": LBRYUM_VERSION,
|
||||
"lbryschema_version": lbryschema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
|
||||
# TODO: remove this from get_platform and add a get_external_ip function using treq
|
||||
if get_ip:
|
||||
try:
|
||||
response = json.loads(urlopen("https://api.lbry.io/ip").read())
|
||||
if not response['success']:
|
||||
raise URLError("failed to get external ip")
|
||||
p['ip'] = response['data']['ip']
|
||||
except (URLError, AssertionError):
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
@@ -1,243 +0,0 @@
|
||||
import base64
|
||||
import datetime
|
||||
import random
|
||||
import socket
|
||||
import string
|
||||
import json
|
||||
import traceback
|
||||
import functools
|
||||
import logging
|
||||
import pkg_resources
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
from lbryschema.claim import ClaimDict
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
|
||||
# defining these time functions here allows for easier overriding in testing
|
||||
def now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def utcnow():
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def isonow():
|
||||
"""Return utc now in isoformat with timezone"""
|
||||
return utcnow().isoformat() + 'Z'
|
||||
|
||||
|
||||
def today():
|
||||
return datetime.datetime.today()
|
||||
|
||||
|
||||
def timedelta(**kwargs):
|
||||
return datetime.timedelta(**kwargs)
|
||||
|
||||
|
||||
def datetime_obj(*args, **kwargs):
|
||||
return datetime.datetime(*args, **kwargs)
|
||||
|
||||
|
||||
def call_later(delay, func, *args, **kwargs):
|
||||
# Import here to ensure that it gets called after installing a reactor
|
||||
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
|
||||
from twisted.internet import reactor
|
||||
return reactor.callLater(delay, func, *args, **kwargs)
|
||||
|
||||
def safe_start_looping_call(looping_call, interval_sec):
|
||||
if not looping_call.running:
|
||||
looping_call.start(interval_sec)
|
||||
|
||||
def safe_stop_looping_call(looping_call):
|
||||
if looping_call.running:
|
||||
looping_call.stop()
|
||||
|
||||
def generate_id(num=None):
|
||||
h = get_lbry_hash_obj()
|
||||
if num is not None:
|
||||
h.update(str(num))
|
||||
else:
|
||||
h.update(str(random.getrandbits(512)))
|
||||
return h.digest()
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def version_is_greater_than(a, b):
|
||||
"""Returns True if version a is more recent than version b"""
|
||||
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
|
||||
|
||||
|
||||
def deobfuscate(obfustacated):
|
||||
return base64.b64decode(obfustacated.decode('rot13'))
|
||||
|
||||
|
||||
def obfuscate(plain):
|
||||
return base64.b64encode(plain).encode('rot13')
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80, timeout=2):
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
try:
|
||||
server = socket.gethostbyname(server)
|
||||
socket.create_connection((server, port), timeout)
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except (socket.gaierror, socket.herror) as ex:
|
||||
log.warning("Failed to connect to %s:%s. Unable to resolve domain. Trying to bypass DNS",
|
||||
server, port)
|
||||
try:
|
||||
server = "8.8.8.8"
|
||||
port = 53
|
||||
socket.create_connection((server, port), timeout)
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except Exception as ex:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
return False
|
||||
except Exception as ex:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
return False
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
|
||||
def short_hash(hash_str):
|
||||
return hash_str[:6]
|
||||
|
||||
|
||||
def get_sd_hash(stream_info):
|
||||
if not stream_info:
|
||||
return None
|
||||
if isinstance(stream_info, ClaimDict):
|
||||
return stream_info.source_hash
|
||||
result = stream_info.get('claim', {}).\
|
||||
get('value', {}).\
|
||||
get('stream', {}).\
|
||||
get('source', {}).\
|
||||
get('source')
|
||||
if not result:
|
||||
log.warn("Unable to get sd_hash")
|
||||
return result
|
||||
|
||||
|
||||
def json_dumps_pretty(obj, **kwargs):
|
||||
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
|
||||
|
||||
|
||||
class DeferredLockContextManager(object):
|
||||
def __init__(self, lock):
|
||||
self._lock = lock
|
||||
|
||||
def __enter__(self):
|
||||
yield self._lock.aquire()
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
yield self._lock.release()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def DeferredDict(d, consumeErrors=False):
|
||||
keys = []
|
||||
dl = []
|
||||
response = {}
|
||||
for k, v in d.iteritems():
|
||||
keys.append(k)
|
||||
dl.append(v)
|
||||
results = yield defer.DeferredList(dl, consumeErrors=consumeErrors)
|
||||
for k, (success, result) in zip(keys, results):
|
||||
if success:
|
||||
response[k] = result
|
||||
defer.returnValue(response)
|
||||
|
||||
|
||||
class DeferredProfiler(object):
|
||||
def __init__(self):
|
||||
self.profile_results = {}
|
||||
|
||||
def add_result(self, fn, start_time, finished_time, stack, success):
|
||||
self.profile_results[fn].append((start_time, finished_time, stack, success))
|
||||
|
||||
def show_profile_results(self, fn):
|
||||
profile_results = list(self.profile_results[fn])
|
||||
call_counts = {
|
||||
caller: [(start, finished, finished - start, success)
|
||||
for (start, finished, _caller, success) in profile_results
|
||||
if _caller == caller]
|
||||
for caller in set(result[2] for result in profile_results)
|
||||
}
|
||||
|
||||
log.info("called %s %i times from %i sources\n", fn.__name__, len(profile_results), len(call_counts))
|
||||
for caller in sorted(list(call_counts.keys()), key=lambda c: len(call_counts[c]), reverse=True):
|
||||
call_info = call_counts[caller]
|
||||
times = [r[2] for r in call_info]
|
||||
own_time = sum(times)
|
||||
times.sort()
|
||||
longest = 0 if not times else times[-1]
|
||||
shortest = 0 if not times else times[0]
|
||||
log.info(
|
||||
"%i successes and %i failures\nlongest %f, shortest %f, avg %f\ncaller:\n%s",
|
||||
len([r for r in call_info if r[3]]),
|
||||
len([r for r in call_info if not r[3]]),
|
||||
longest, shortest, own_time / float(len(call_info)), caller
|
||||
)
|
||||
|
||||
def profiled_deferred(self, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
|
||||
def _cb(result, fn, start, caller_info):
|
||||
if isinstance(result, (Failure, Exception)):
|
||||
error = result
|
||||
result = None
|
||||
else:
|
||||
error = None
|
||||
self.add_result(fn, start, reactor.seconds(), caller_info, error is None)
|
||||
if error is None:
|
||||
return result
|
||||
raise error
|
||||
|
||||
def _profiled_deferred(fn):
|
||||
reactor.addSystemEventTrigger("after", "shutdown", self.show_profile_results, fn)
|
||||
self.profile_results[fn] = []
|
||||
|
||||
@functools.wraps(fn)
|
||||
def _wrapper(*args, **kwargs):
|
||||
caller_info = "".join(traceback.format_list(traceback.extract_stack()[-3:-1]))
|
||||
start = reactor.seconds()
|
||||
d = defer.maybeDeferred(fn, *args, **kwargs)
|
||||
d.addBoth(_cb, fn, start, caller_info)
|
||||
return d
|
||||
|
||||
return _wrapper
|
||||
|
||||
return _profiled_deferred
|
||||
|
||||
|
||||
_profiler = DeferredProfiler()
|
||||
profile_deferred = _profiler.profiled_deferred
|
||||
@@ -0,0 +1,9 @@
|
||||
import hashlib
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
|
||||
backend = default_backend()
|
||||
|
||||
|
||||
def get_lbry_hash_obj():
|
||||
return hashlib.sha384()
|
||||
@@ -1,166 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
backend = default_backend()
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
self.iv = iv
|
||||
|
||||
def get_dict(self):
|
||||
info = {
|
||||
"blob_num": self.blob_num,
|
||||
"length": self.length,
|
||||
"iv": self.iv
|
||||
}
|
||||
if self.blob_hash:
|
||||
info['blob_hash'] = self.blob_hash
|
||||
return info
|
||||
|
||||
|
||||
class StreamBlobDecryptor(object):
|
||||
def __init__(self, blob, key, iv, length):
|
||||
"""
|
||||
This class decrypts blob
|
||||
|
||||
blob - object which implements read() function.
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
length = length in bytes of blob
|
||||
"""
|
||||
self.blob = blob
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
self.length = length
|
||||
self.buff = b''
|
||||
self.len_read = 0
|
||||
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
|
||||
self.unpadder = PKCS7(AES.block_size).unpadder()
|
||||
self.cipher = cipher.decryptor()
|
||||
|
||||
def decrypt(self, write_func):
|
||||
"""
|
||||
Decrypt blob and write its content useing write_func
|
||||
|
||||
write_func - function that takes decrypted string as
|
||||
arugment and writes it somewhere
|
||||
|
||||
Returns:
|
||||
|
||||
deferred that returns after decrypting blob and writing content
|
||||
"""
|
||||
|
||||
def remove_padding(data):
|
||||
return self.unpadder.update(data) + self.unpadder.finalize()
|
||||
|
||||
def write_bytes():
|
||||
if self.len_read < self.length:
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size / 8))
|
||||
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
|
||||
write_func(self.cipher.update(data_to_decrypt))
|
||||
|
||||
def finish_decrypt():
|
||||
bytes_left = len(self.buff) % (AES.block_size / 8)
|
||||
if bytes_left != 0:
|
||||
log.warning(self.buff[-1 * (AES.block_size / 8):].encode('hex'))
|
||||
raise Exception("blob %s has incorrect padding: %i bytes left" %
|
||||
(self.blob.blob_hash, bytes_left))
|
||||
data_to_decrypt, self.buff = self.buff, b''
|
||||
last_chunk = self.cipher.update(data_to_decrypt) + self.cipher.finalize()
|
||||
write_func(remove_padding(last_chunk))
|
||||
|
||||
|
||||
read_handle = self.blob.open_for_reading()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def decrypt_bytes():
|
||||
producer = FileBodyProducer(read_handle)
|
||||
buff = BytesIO()
|
||||
yield producer.startProducing(buff)
|
||||
self.buff = buff.getvalue()
|
||||
self.len_read += len(self.buff)
|
||||
write_bytes()
|
||||
finish_decrypt()
|
||||
|
||||
d = decrypt_bytes()
|
||||
return d
|
||||
|
||||
|
||||
class CryptStreamBlobMaker(object):
|
||||
def __init__(self, key, iv, blob_num, blob):
|
||||
"""
|
||||
This class encrypts data and writes it to a new blob
|
||||
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
blob = object which implements write(), close() function , close() function must
|
||||
be a deferred. (Will generally be of HashBlobCreator type)
|
||||
"""
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
self.blob_num = blob_num
|
||||
self.blob = blob
|
||||
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
|
||||
self.padder = PKCS7(AES.block_size).padder()
|
||||
self.cipher = cipher.encryptor()
|
||||
self.length = 0
|
||||
|
||||
def write(self, data):
|
||||
"""
|
||||
encrypt and write string data
|
||||
|
||||
Returns:
|
||||
tuple (done, num_bytes_to_write) where done is True if
|
||||
max bytes are written. num_bytes_to_write is the number
|
||||
of bytes that will be written from data in this call
|
||||
"""
|
||||
max_bytes_to_write = MAX_BLOB_SIZE - self.length - 1
|
||||
done = False
|
||||
if max_bytes_to_write <= len(data):
|
||||
num_bytes_to_write = max_bytes_to_write
|
||||
done = True
|
||||
else:
|
||||
num_bytes_to_write = len(data)
|
||||
data_to_write = data[:num_bytes_to_write]
|
||||
self.length += len(data_to_write)
|
||||
padded_data = self.padder.update(data_to_write)
|
||||
encrypted_data = self.cipher.update(padded_data)
|
||||
self.blob.write(encrypted_data)
|
||||
return done, num_bytes_to_write
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def close(self):
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self.length += (AES.block_size / 8) - (self.length % (AES.block_size / 8))
|
||||
padded_data = self.padder.finalize()
|
||||
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
blob_hash = yield self.blob.close()
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
blob = CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
defer.returnValue(blob)
|
||||
|
||||
|
||||
def greatest_multiple(a, b):
|
||||
"""return the largest value `c`, that is a multiple of `b` and is <= `a`"""
|
||||
return (a // b) * b
|
||||
|
||||
|
||||
def split(buff, cutoff):
|
||||
return buff[:cutoff], buff[cutoff:]
|
||||
@@ -1,150 +0,0 @@
|
||||
"""
|
||||
Utility for creating Crypt Streams, which are encrypted blobs and associated metadata.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(object):
|
||||
"""
|
||||
Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
Each blob is encrypted with the same key, but each blob has its
|
||||
own initialization vector which is associated with the blob when
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
|
||||
implements(interfaces.IConsumer)
|
||||
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@type blob_manager: BlobManager
|
||||
|
||||
@param name: the name of the stream, which will be presented to the user
|
||||
@type name: string
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the
|
||||
blobs. If None, a random key will be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self.blob_manager = blob_manager
|
||||
self.name = name
|
||||
self.key = key
|
||||
if iv_generator is None:
|
||||
self.iv_generator = self.random_iv_generator()
|
||||
else:
|
||||
self.iv_generator = iv_generator
|
||||
|
||||
self.stopped = True
|
||||
self.producer = None
|
||||
self.streaming = None
|
||||
self.blob_count = -1
|
||||
self.current_blob = None
|
||||
self.finished_deferreds = []
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.producer = producer
|
||||
self.streaming = streaming
|
||||
self.stopped = False
|
||||
if streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.stopped = True
|
||||
self.producer = None
|
||||
|
||||
def _close_current_blob(self):
|
||||
# close the blob that was being written to
|
||||
# and save it to blob manager
|
||||
should_announce = self.blob_count == 0
|
||||
d = self.current_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
d.addCallback(lambda blob_info: self.blob_manager.creator_finished(blob_info,
|
||||
should_announce))
|
||||
self.finished_deferreds.append(d)
|
||||
self.current_blob = None
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
self._close_current_blob()
|
||||
d = self._finalize()
|
||||
d.addCallback(lambda _: self._finished())
|
||||
return d
|
||||
|
||||
# TODO: move the stream creation process to its own thread and
|
||||
# remove the reactor from this process.
|
||||
def write(self, data):
|
||||
from twisted.internet import reactor
|
||||
self._write(data)
|
||||
if self.stopped is False and self.streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
@staticmethod
|
||||
def random_iv_generator():
|
||||
while 1:
|
||||
yield os.urandom(AES.block_size / 8)
|
||||
|
||||
def setup(self):
|
||||
"""Create the symmetric key if it wasn't provided"""
|
||||
|
||||
if self.key is None:
|
||||
self.key = os.urandom(AES.block_size / 8)
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _finalize(self):
|
||||
"""
|
||||
Finalize a stream by adding an empty
|
||||
blob at the end, this is to indicate that
|
||||
the stream has ended. This empty blob is not
|
||||
saved to the blob manager
|
||||
"""
|
||||
|
||||
yield defer.DeferredList(self.finished_deferreds)
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
|
||||
stream_terminator = yield final_blob.close()
|
||||
terminator_info = yield self._blob_finished(stream_terminator)
|
||||
defer.returnValue(terminator_info)
|
||||
|
||||
def _write(self, data):
|
||||
while len(data) > 0:
|
||||
if self.current_blob is None:
|
||||
self.next_blob_creator = self.blob_manager.get_blob_creator()
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
|
||||
done, num_bytes_written = self.current_blob.write(data)
|
||||
data = data[num_bytes_written:]
|
||||
if done is True:
|
||||
self._close_current_blob()
|
||||
|
||||
def _get_blob_maker(self, iv, blob_creator):
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
|
||||
def _finished(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
raise NotImplementedError()
|
||||
@@ -1,8 +0,0 @@
|
||||
"""
|
||||
Classes and functions for dealing with Crypt Streams.
|
||||
|
||||
Crypt Streams are encrypted blobs and metadata tying those blobs together. At least some of the
|
||||
metadata is generally stored in a Stream Descriptor File, for example containing a public key
|
||||
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
|
||||
may or may not be present.
|
||||
"""
|
||||