Compare commits
856
Commits
release_time
...
v0.37.2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe55187696 | ||
|
|
db8fb15cc5 | ||
|
|
a165ad910f | ||
|
|
b711069456 | ||
|
|
c977a672b0 | ||
|
|
ac5794d96d | ||
|
|
bb1f42057e | ||
|
|
695f6658a0 | ||
|
|
eca677a720 | ||
|
|
51f301785f | ||
|
|
41a6e288aa | ||
|
|
97d8d0ef4c | ||
|
|
5baeb5f538 | ||
|
|
cc49c8407c | ||
|
|
3f37963f7f | ||
|
|
12eb83f773 | ||
|
|
ca1d135ba4 | ||
|
|
107e57e3ff | ||
|
|
928689a10d | ||
|
|
6ecc22f2c7 | ||
|
|
079c81f298 | ||
|
|
8afe178d62 | ||
|
|
8777dcee7c | ||
|
|
1cca4a91dd | ||
|
|
41abfbdd9b | ||
|
|
46ee65c8f2 | ||
|
|
fd19daad9e | ||
|
|
26d183cbab | ||
|
|
71a4be02dd | ||
|
|
124fdb7d31 | ||
|
|
2439743804 | ||
|
|
6f79875135 | ||
|
|
b91d2190f4 | ||
|
|
d024433d1b | ||
|
|
7c4eb416d6 | ||
|
|
fb457c820a | ||
|
|
18af2dcd4e | ||
|
|
0075dcc2c0 | ||
|
|
6e327d729b | ||
|
|
20022d49c1 | ||
|
|
32d1563885 | ||
|
|
f5cef14d8d | ||
|
|
bac7d99b8a | ||
|
|
6f06026511 | ||
|
|
f02df86709 | ||
|
|
b7d76fd09f | ||
|
|
e675f1387c | ||
|
|
0da2827c78 | ||
|
|
309cfc840c | ||
|
|
68ee31b62a | ||
|
|
ad2da24d64 | ||
|
|
648dbcc36b | ||
|
|
e59e96190e | ||
|
|
7d2f592bb5 | ||
|
|
1a182bc9bd | ||
|
|
324cbdcdb2 | ||
|
|
6c0c0ee322 | ||
|
|
1e54271d8a | ||
|
|
54b9ed6cb2 | ||
|
|
558a02d187 | ||
|
|
3567b331c0 | ||
|
|
9e4cf3c995 | ||
|
|
db1d547b19 | ||
|
|
c31efd780b | ||
|
|
7cdb7987e3 | ||
|
|
ec07037e88 | ||
|
|
9bb7b0571e | ||
|
|
d98162a118 | ||
|
|
c5ae59bff7 | ||
|
|
efbf2f49a9 | ||
|
|
77c69f661d | ||
|
|
2e09d2adfa | ||
|
|
8620a0e424 | ||
|
|
e2c123456b | ||
|
|
c9be79f062 | ||
|
|
a2a1ef34ab | ||
|
|
761a2b235c | ||
|
|
3ca2fee592 | ||
|
|
a480b2d25f | ||
|
|
d7032b12d7 | ||
|
|
84381ff76c | ||
|
|
007dd43861 | ||
|
|
1bc4e4b702 | ||
|
|
b91f27219d | ||
|
|
dc4560cc9a | ||
|
|
84b471d486 | ||
|
|
c6a3c05a0c | ||
|
|
dd665a758d | ||
|
|
32dc1297ec | ||
|
|
bef244ba77 | ||
|
|
3c231d6215 | ||
|
|
a212cf6ba4 | ||
|
|
bc060f1378 | ||
|
|
22c701fd50 | ||
|
|
4f0e8fce6e | ||
|
|
f642cfe9dd | ||
|
|
24e073680b | ||
|
|
b2f63a1545 | ||
|
|
f506b3e6d4 | ||
|
|
f7412514b3 | ||
|
|
bd6a609b30 | ||
|
|
4e32b69d1d | ||
|
|
1116c7f29e | ||
|
|
d80c671cf2 | ||
|
|
14b12cbea2 | ||
|
|
1f7feafb67 | ||
|
|
cbe689ea7f | ||
|
|
9099ee2e8e | ||
|
|
b134e0c9c9 | ||
|
|
f8c0e80cfc | ||
|
|
1f266ebdad | ||
|
|
b09e1f8825 | ||
|
|
c496278a5b | ||
|
|
cacc65d45e | ||
|
|
2247b354b3 | ||
|
|
43111f0497 | ||
|
|
6c142b4477 | ||
|
|
e8683a24bd | ||
|
|
a4d17d1705 | ||
|
|
a030e13548 | ||
|
|
32b8601c38 | ||
|
|
f5bd28bfdb | ||
|
|
6cbbe23c7c | ||
|
|
d7c2e4e9b3 | ||
|
|
d61ddbb950 | ||
|
|
cf7bb6a391 | ||
|
|
68d7328918 | ||
|
|
ff2985cc80 | ||
|
|
ac69fcbc5f | ||
|
|
609cf42868 | ||
|
|
e43b29fcd1 | ||
|
|
6594b7c50c | ||
|
|
971f2a1cef | ||
|
|
fc88261cc6 | ||
|
|
895b3992c9 | ||
|
|
41951d8ec2 | ||
|
|
ae2121c5c4 | ||
|
|
771fe935f4 | ||
|
|
60f7766cf7 | ||
|
|
63bd983012 | ||
|
|
7ffce1040c | ||
|
|
ac6835fa60 | ||
|
|
3cf5c536c0 | ||
|
|
85cc1a4763 | ||
|
|
180cfcfb72 | ||
|
|
864c1dd901 | ||
|
|
aeaa7ba986 | ||
|
|
b8325c1f89 | ||
|
|
a36936b5ff | ||
|
|
7cf245a0e9 | ||
|
|
9f83cce574 | ||
|
|
30b113d64a | ||
|
|
ccd2023d37 | ||
|
|
3127cd11cf | ||
|
|
6e483cba8d | ||
|
|
46c03c0cb2 | ||
|
|
c2ca0a9467 | ||
|
|
b350e74aca | ||
|
|
de9473f75a | ||
|
|
bfc107fdb3 | ||
|
|
6a2efa54ab | ||
|
|
72f0e1ca14 | ||
|
|
9a1262d557 | ||
|
|
706a8e52dc | ||
|
|
f3c8776878 | ||
|
|
539ea04039 | ||
|
|
8f96b07559 | ||
|
|
0a1e97fe23 | ||
|
|
da637b9c0e | ||
|
|
3618de1227 | ||
|
|
3cb8a63bf9 | ||
|
|
bad99de5b3 | ||
|
|
3d1efbc9bb | ||
|
|
7bdef3146b | ||
|
|
2145bfcabf | ||
|
|
b88832d0a9 | ||
|
|
db08882f20 | ||
|
|
6f52d36b22 | ||
|
|
056a629e87 | ||
|
|
1ffb99f9f9 | ||
|
|
a8e612773d | ||
|
|
67088b2ca2 | ||
|
|
dab49e812f | ||
|
|
0f3f38d8a3 | ||
|
|
7ba433c2e5 | ||
|
|
bb7649e8ad | ||
|
|
832537a5cf | ||
|
|
3234d70270 | ||
|
|
432fe444f8 | ||
|
|
861c25716e | ||
|
|
9ba6653475 | ||
|
|
8107e97304 | ||
|
|
e2663f6d57 | ||
|
|
35479dcbd2 | ||
|
|
3a997277aa | ||
|
|
a7d94f4514 | ||
|
|
a0e3338177 | ||
|
|
d5a353066c | ||
|
|
bc29fc4465 | ||
|
|
987f175fcc | ||
|
|
5b1398b588 | ||
|
|
873b58efc9 | ||
|
|
01a81fceed | ||
|
|
a14b0aa573 | ||
|
|
a1abb7c8fb | ||
|
|
46b166952b | ||
|
|
1375153fda | ||
|
|
1a14157d37 | ||
|
|
40b9fa38b0 | ||
|
|
4d0e9e0428 | ||
|
|
3a53ef5690 | ||
|
|
33a68b5cef | ||
|
|
63bf084a6a | ||
|
|
a882f2e8c5 | ||
|
|
119e0ec408 | ||
|
|
7d782d516a | ||
|
|
d3eac48035 | ||
|
|
60c9ae64b4 | ||
|
|
2d314dce60 | ||
|
|
287b89db66 | ||
|
|
7a160f7335 | ||
|
|
04ddee7d89 | ||
|
|
3a916a8e8e | ||
|
|
f125468ebf | ||
|
|
f0e17cff9a | ||
|
|
93267efe0b | ||
|
|
c663e5a3cf | ||
|
|
676f0015aa | ||
|
|
d44a79ada2 | ||
|
|
d7ed8ffaa6 | ||
|
|
b230d8fbcb | ||
|
|
7d2428fd17 | ||
|
|
4157045a41 | ||
|
|
58987857cf | ||
|
|
fa8858325c | ||
|
|
3357a419ab | ||
|
|
a782adea75 | ||
|
|
e8991e6d52 | ||
|
|
13bd81c2b6 | ||
|
|
21241b98a4 | ||
|
|
bf467e06c5 | ||
|
|
886ed6415a | ||
|
|
97db241109 | ||
|
|
216379918b | ||
|
|
fcab4eb9bd | ||
|
|
19f300cc45 | ||
|
|
26fec098e4 | ||
|
|
7824de655e | ||
|
|
6ef1b1dd59 | ||
|
|
9fae8f5153 | ||
|
|
f660691e78 | ||
|
|
bf60683ba6 | ||
|
|
68215699f7 | ||
|
|
1644ffbf22 | ||
|
|
71f5061848 | ||
|
|
61cadb5cbe | ||
|
|
2034104cfc | ||
|
|
633f725634 | ||
|
|
8bf2e41c95 | ||
|
|
fa9a4a75c6 | ||
|
|
90bef98bc3 | ||
|
|
35c43647aa | ||
|
|
449a583286 | ||
|
|
b1f813782a | ||
|
|
60080cbd19 | ||
|
|
665dba62c7 | ||
|
|
76dea9e89b | ||
|
|
ebc9ed01c5 | ||
|
|
55116763e9 | ||
|
|
e1a6b4e3d6 | ||
|
|
911ca8c37e | ||
|
|
7f5deaf6c8 | ||
|
|
0200756218 | ||
|
|
a63442f843 | ||
|
|
74b3da79aa | ||
|
|
b6a4589da5 | ||
|
|
fbbb0a9931 | ||
|
|
a94d65d522 | ||
|
|
fa2ead002e | ||
|
|
8dfee4867a | ||
|
|
6614ab8e79 | ||
|
|
d32c1dfbb3 | ||
|
|
9f42b958fa | ||
|
|
574a912f7d | ||
|
|
2cfcc3cd45 | ||
|
|
9af625e627 | ||
|
|
1871c94003 | ||
|
|
93220c3874 | ||
|
|
362ed3f1c8 | ||
|
|
62a182b8bb | ||
|
|
6f76d9e24a | ||
|
|
55e83fe05c | ||
|
|
2f0c7d6a3f | ||
|
|
59e1df9efa | ||
|
|
fd56dc68cd | ||
|
|
a0a810edc3 | ||
|
|
c1d3b5984d | ||
|
|
1fb719dd61 | ||
|
|
b87316fb02 | ||
|
|
65f3d78084 | ||
|
|
47bc1526f7 | ||
|
|
3acabec4bb | ||
|
|
36a62d3756 | ||
|
|
2e22473ce3 | ||
|
|
0b0fd64642 | ||
|
|
ca17d8597d | ||
|
|
d16e2716c8 | ||
|
|
89c80cedfd | ||
|
|
021e30ffde | ||
|
|
eb2cd8fec1 | ||
|
|
6356818557 | ||
|
|
8bf724a78b | ||
|
|
65d0a924c1 | ||
|
|
ef7bd00f22 | ||
|
|
3c1fdb5114 | ||
|
|
83411acfbd | ||
|
|
4230812f82 | ||
|
|
7b57ba96dd | ||
|
|
668840921f | ||
|
|
a3bd335f7f | ||
|
|
4e6bca1389 | ||
|
|
c4b15920ff | ||
|
|
2a55e583e7 | ||
|
|
2c4fde06c5 | ||
|
|
eca83d7f47 | ||
|
|
87e818ac39 | ||
|
|
adab7bd7b1 | ||
|
|
750c0e4f4f | ||
|
|
5f8482bdb7 | ||
|
|
cd266477fb | ||
|
|
7860b956ff | ||
|
|
148481a5ca | ||
|
|
93faa276a5 | ||
|
|
53e67a6e81 | ||
|
|
3ad6808d87 | ||
|
|
071ab32ab7 | ||
|
|
ab87588250 | ||
|
|
22db29ee1d | ||
|
|
66bf6c61f0 | ||
|
|
9abf40d21f | ||
|
|
ea4444edbe | ||
|
|
51bb8fd283 | ||
|
|
183e3ac1c7 | ||
|
|
8087b457c6 | ||
|
|
fedb7c23d0 | ||
|
|
e2964f9728 | ||
|
|
e0cea60c87 | ||
|
|
c708c1db07 | ||
|
|
1cb696fb5c | ||
|
|
7cd0213eb9 | ||
|
|
2007c48af3 | ||
|
|
50721786d2 | ||
|
|
6d7eb259cb | ||
|
|
cc5852a9b9 | ||
|
|
0592cee567 | ||
|
|
814eaca524 | ||
|
|
e06916f5ae | ||
|
|
d61e1e9d8c | ||
|
|
25436893bc | ||
|
|
5b9d41fd5c | ||
|
|
b1720d1659 | ||
|
|
4b3a6ab875 | ||
|
|
67dcede2d8 | ||
|
|
0872a6fea3 | ||
|
|
a3aed3219f | ||
|
|
af73e1a2f5 | ||
|
|
55bbf5aa74 | ||
|
|
c1fbb02bfe | ||
|
|
d47575e8e0 | ||
|
|
8fc4f4aa73 | ||
|
|
fe9aa666e9 | ||
|
|
a9e0eeba7e | ||
|
|
0830917afb | ||
|
|
1ed086e90f | ||
|
|
56db631626 | ||
|
|
aa029955ee | ||
|
|
f2236eb7a4 | ||
|
|
df649341cd | ||
|
|
a3c3143868 | ||
|
|
d0201fe7c7 | ||
|
|
664a2a324d | ||
|
|
ea09aca175 | ||
|
|
ca92fddecf | ||
|
|
8259d7c680 | ||
|
|
6653da2973 | ||
|
|
e74998209d | ||
|
|
2bea0f697a | ||
|
|
1d68bef6f2 | ||
|
|
22bacc907b | ||
|
|
1e8ad900c7 | ||
|
|
cd2b535afb | ||
|
|
9dfe7bbcf6 | ||
|
|
120618ffd5 | ||
|
|
e8abcbb06e | ||
|
|
1ec8f0b0b4 | ||
|
|
323694fb12 | ||
|
|
d5e37acd8f | ||
|
|
129a886f2b | ||
|
|
cd15230a92 | ||
|
|
c1b7fdc68e | ||
|
|
9fd67aaad6 | ||
|
|
ce9dc1836e | ||
|
|
2e557d3a36 | ||
|
|
5df05cb37e | ||
|
|
326b62dc57 | ||
|
|
25cc7ba284 | ||
|
|
550f064302 | ||
|
|
11086b93f1 | ||
|
|
df5662136d | ||
|
|
291d65070d | ||
|
|
9b64ee33dc | ||
|
|
5404d26f2f | ||
|
|
26ade1f228 | ||
|
|
b49f1f1b54 | ||
|
|
3bafa2ad22 | ||
|
|
378ec44f3c | ||
|
|
9d0d17b422 | ||
|
|
f7c8145283 | ||
|
|
0de17730ca | ||
|
|
58743ba19b | ||
|
|
f2fefbe287 | ||
|
|
60a24f0e6e | ||
|
|
664f91bfab | ||
|
|
44f92a271f | ||
|
|
6565ca8558 | ||
|
|
3b0ba3e534 | ||
|
|
8106eb1fa5 | ||
|
|
c9b2829227 | ||
|
|
4f2f8b16f3 | ||
|
|
2ab2b846b1 | ||
|
|
2df7284c29 | ||
|
|
ab5d71ff61 | ||
|
|
99b289f135 | ||
|
|
6db02e4362 | ||
|
|
98d8b7d463 | ||
|
|
96f1883fc4 | ||
|
|
a914de155a | ||
|
|
2dee0ff0bf | ||
|
|
993cb43e5b | ||
|
|
5d212a0f82 | ||
|
|
0cc602ca53 | ||
|
|
f390ff144d | ||
|
|
b266cde511 | ||
|
|
d0a1541d0a | ||
|
|
4cbde1b08f | ||
|
|
c6740d8d70 | ||
|
|
46358142ed | ||
|
|
f0f651fc77 | ||
|
|
5d428138ed | ||
|
|
8d9da61c7d | ||
|
|
c326a9382c | ||
|
|
5bdd87c904 | ||
|
|
ad134fe8bb | ||
|
|
2f3c616acd | ||
|
|
3caa2a5039 | ||
|
|
e38a0f7a95 | ||
|
|
9de3f3ff09 | ||
|
|
df8934d472 | ||
|
|
be4a96c1b3 | ||
|
|
d97d978961 | ||
|
|
f52e44b56a | ||
|
|
10d73fe480 | ||
|
|
4fd2d994f2 | ||
|
|
d95903d219 | ||
|
|
591e66bfbb | ||
|
|
e047748961 | ||
|
|
9f071bbe36 | ||
|
|
2d8ebe25ed | ||
|
|
7a96e742f2 | ||
|
|
13c7026b20 | ||
|
|
876a3c68dd | ||
|
|
573617d6b6 | ||
|
|
e8c9abb944 | ||
|
|
8cee627dfa | ||
|
|
ab4a5cee01 | ||
|
|
5bb0a07fae | ||
|
|
5ca594b9fd | ||
|
|
3910964df1 | ||
|
|
faa1935203 | ||
|
|
ba518e2a0e | ||
|
|
2a82ff7cc5 | ||
|
|
673b958020 | ||
|
|
3f4ed0a896 | ||
|
|
f49578cdbc | ||
|
|
b834f29634 | ||
|
|
61132312dd | ||
|
|
fcdd9295b4 | ||
|
|
edb5083a5e | ||
|
|
4e7d88311f | ||
|
|
aa40337ded | ||
|
|
690e2e7e0c | ||
|
|
0c67469321 | ||
|
|
fc64d8d1a8 | ||
|
|
7fe9513857 | ||
|
|
e76107c318 | ||
|
|
3868dea819 | ||
|
|
c52541197f | ||
|
|
03cd6b7200 | ||
|
|
6915a7a553 | ||
|
|
7ad29b4bfb | ||
|
|
4294bf742d | ||
|
|
57ad9f1c52 | ||
|
|
e260de59f1 | ||
|
|
154702e6dd | ||
|
|
d06d998c81 | ||
|
|
8aff860bfe | ||
|
|
d7ba90bbea | ||
|
|
59325ad8c1 | ||
|
|
1f72e27272 | ||
|
|
aa41b8b558 | ||
|
|
2496e7ed44 | ||
|
|
ee12985135 | ||
|
|
4c7f357441 | ||
|
|
8944a36291 | ||
|
|
cb5d428d12 | ||
|
|
f175e8826c | ||
|
|
38ed212c01 | ||
|
|
d035c13883 | ||
|
|
82ce2ccee6 | ||
|
|
2cf9b2dd9e | ||
|
|
fcbdaa336f | ||
|
|
3ae6137edb | ||
|
|
0fdd2ae178 | ||
|
|
8c5007fe07 | ||
|
|
d2150cf142 | ||
|
|
ce2bb22929 | ||
|
|
1ce3adb6a0 | ||
|
|
c2dc1f3c1d | ||
|
|
b36c22e2f4 | ||
|
|
71b66ab337 | ||
|
|
461c3c8408 | ||
|
|
03396c1993 | ||
|
|
fa7e9923f4 | ||
|
|
3f7e466c97 | ||
|
|
4483380352 | ||
|
|
4a0a6b0243 | ||
|
|
a003e794ba | ||
|
|
051fabc180 | ||
|
|
ca686041e7 | ||
|
|
fc450eb164 | ||
|
|
652d7cbd24 | ||
|
|
43b372c0fe | ||
|
|
eaacf7e034 | ||
|
|
576da1bc30 | ||
|
|
5c217f1544 | ||
|
|
cfbf0dbe0f | ||
|
|
fd081496c0 | ||
|
|
32b4405a4c | ||
|
|
0d558bd552 | ||
|
|
ca835f3f80 | ||
|
|
dbb6ba6241 | ||
|
|
a228d20137 | ||
|
|
f1b60e3ef2 | ||
|
|
39737c790f | ||
|
|
dba4a09d38 | ||
|
|
918122e7cf | ||
|
|
a49f479a5f | ||
|
|
320aabbd70 | ||
|
|
19fdcad5af | ||
|
|
1734e08297 | ||
|
|
9d89f9b834 | ||
|
|
56c41b2fea | ||
|
|
9c8593d88e | ||
|
|
85f41887fd | ||
|
|
8eede6dfc7 | ||
|
|
0d8c37a63b | ||
|
|
d2ba3fdcfb | ||
|
|
be5ab0f307 | ||
|
|
04f70d1f31 | ||
|
|
60ba07cc17 | ||
|
|
dbc8cfe5ae | ||
|
|
d91e8f5996 | ||
|
|
b6df2b9d28 | ||
|
|
5c1ab42277 | ||
|
|
bbdc96e762 | ||
|
|
fc0c949187 | ||
|
|
3fce8e4023 | ||
|
|
37bb765a2e | ||
|
|
21ca4f60f7 | ||
|
|
119d498bce | ||
|
|
8108aa39c1 | ||
|
|
0a068011d8 | ||
|
|
1193d2cea4 | ||
|
|
76ecb5d50d | ||
|
|
dbc1164562 | ||
|
|
e2170ec83d | ||
|
|
266e777655 | ||
|
|
68905d82b9 | ||
|
|
396c49f7c6 | ||
|
|
5e509c9fd6 | ||
|
|
e6b6a3f55e | ||
|
|
019af119d9 | ||
|
|
58da2f16c2 | ||
|
|
8aa50aa14f | ||
|
|
65c6c83b51 | ||
|
|
f497989871 | ||
|
|
5366779b1b | ||
|
|
a3f59f041a | ||
|
|
d560b0c1cd | ||
|
|
e94229121a | ||
|
|
097291a498 | ||
|
|
afdc44ca13 | ||
|
|
6732b302bb | ||
|
|
c9d54bb049 | ||
|
|
f65e70df2f | ||
|
|
4541e754dd | ||
|
|
9f89c62e8a | ||
|
|
6386815a29 | ||
|
|
0c93090d2c | ||
|
|
45404b6ad4 | ||
|
|
aeb2891baa | ||
|
|
8abfdc82b2 | ||
|
|
da9b4e317a | ||
|
|
e414cc5c48 | ||
|
|
939edb2f18 | ||
|
|
868110a6f2 | ||
|
|
869a8b712b | ||
|
|
9ecec52412 | ||
|
|
8b25acff6e | ||
|
|
e2b06677b5 | ||
|
|
af1619ebfb | ||
|
|
3352e0e4f4 | ||
|
|
141d68a2cd | ||
|
|
c06ec6cd69 | ||
|
|
1be5dce30e | ||
|
|
73da223ae1 | ||
|
|
2d7eb83518 | ||
|
|
ad03f91d24 | ||
|
|
5586a226c2 | ||
|
|
a616582733 | ||
|
|
43ac928f0b | ||
|
|
a121110743 | ||
|
|
8bc5bb4e14 | ||
|
|
3a15ff4bcd | ||
|
|
3508da4993 | ||
|
|
dfdc8eda00 | ||
|
|
58f6cb71c6 | ||
|
|
ae11c5bb4b | ||
|
|
f22b5da170 | ||
|
|
c1c6d5bc99 | ||
|
|
0e972ec2ae | ||
|
|
f8dc0f298e | ||
|
|
8b5803c945 | ||
|
|
1314f75219 | ||
|
|
be90ec51e4 | ||
|
|
e06c903731 | ||
|
|
76c8447a56 | ||
|
|
82b53e108c | ||
|
|
7c8ad0fe30 | ||
|
|
073ffdd049 | ||
|
|
7fe211d134 | ||
|
|
43d424978d | ||
|
|
a4ced4dbd2 | ||
|
|
c6b910a5f3 | ||
|
|
63d1ec3c28 | ||
|
|
c31e61586d | ||
|
|
7b8f42e6d0 | ||
|
|
2c275efa48 | ||
|
|
30846f932b | ||
|
|
6ead932ccb | ||
|
|
e9b58577dd | ||
|
|
f766dbefe0 | ||
|
|
f8a7d2ebe2 | ||
|
|
af43b21ef6 | ||
|
|
0685bc5689 | ||
|
|
8cda3143f2 | ||
|
|
5af351a234 | ||
|
|
b9934a7eb7 | ||
|
|
b82d2c3120 | ||
|
|
8e607fcc2a | ||
|
|
2bce7cecfa | ||
|
|
e141cc9e22 | ||
|
|
8883587cc7 | ||
|
|
c85991704d | ||
|
|
377442d9f0 | ||
|
|
ad30b7df6d | ||
|
|
f0b49fc153 | ||
|
|
6033399f05 | ||
|
|
ae1162f5b0 | ||
|
|
7d7d4c35c3 | ||
|
|
ad71a9a89d | ||
|
|
13b3e4ba8e | ||
|
|
43799168ba | ||
|
|
736a949cb5 | ||
|
|
11bb1cc7ee | ||
|
|
dbce49e567 | ||
|
|
dcd78bd86e | ||
|
|
e5af041627 | ||
|
|
73f4e954e5 | ||
|
|
0a027cf2de | ||
|
|
3f98b16800 | ||
|
|
bbadece75d | ||
|
|
8f5a8ab8ba | ||
|
|
91e581422c | ||
|
|
b54d7185a1 | ||
|
|
f2968aab22 | ||
|
|
4e27eaf8e6 | ||
|
|
ccbe8aa0eb | ||
|
|
478f79353a | ||
|
|
c3b0151943 | ||
|
|
ba818354af | ||
|
|
e96b75a0d0 | ||
|
|
efe4afd09e | ||
|
|
c7cb6822cc | ||
|
|
c75665d3f0 | ||
|
|
744375b2c0 | ||
|
|
c1b4a012ec | ||
|
|
40da5b6cb6 | ||
|
|
b36cf3e36c | ||
|
|
4e3b529aec | ||
|
|
0a08f445a4 | ||
|
|
3eac99c509 | ||
|
|
f5bf1463af | ||
|
|
ca4a55ef28 | ||
|
|
832be0758b | ||
|
|
25050fdeeb | ||
|
|
df0635103e | ||
|
|
dabd9cbe1e | ||
|
|
42d7fbdbb3 | ||
|
|
2336015f9a | ||
|
|
5a28128b69 | ||
|
|
31445c7797 | ||
|
|
16d0ff8376 | ||
|
|
87f5d1bd4a | ||
|
|
be6e8744b0 | ||
|
|
2e9acd1baa | ||
|
|
607ccefe90 | ||
|
|
df86937f4e | ||
|
|
957d0290df | ||
|
|
40b90ce79b | ||
|
|
aa7f0845cc | ||
|
|
6e69d306de | ||
|
|
d85d3dfaa4 | ||
|
|
2b035009ef | ||
|
|
f9fd62c214 | ||
|
|
e8a1953815 | ||
|
|
34776e7eb9 | ||
|
|
a4419f562f | ||
|
|
c0160ab4e9 | ||
|
|
e60e5b3919 | ||
|
|
3589cc9977 | ||
|
|
2e978c00b2 | ||
|
|
2335243de9 | ||
|
|
6ad68eabd9 | ||
|
|
ca5c638124 | ||
|
|
262b9a624b | ||
|
|
05915c540a | ||
|
|
6aef6a80b7 | ||
|
|
16efe9ba95 | ||
|
|
fcda9500fe | ||
|
|
64dec821b7 | ||
|
|
d5f6e58aad | ||
|
|
087b65522e | ||
|
|
5ee4b2173a | ||
|
|
3e1c3d1e4b | ||
|
|
21f2b2dcd7 | ||
|
|
ffc7fc0793 | ||
|
|
15dbe5e54e | ||
|
|
2e633c0ee6 | ||
|
|
933d58d49b | ||
|
|
8f7cf0b38f | ||
|
|
adf378b6e9 | ||
|
|
1a4f083e76 | ||
|
|
7bde09dcf8 | ||
|
|
4630fd38aa | ||
|
|
418def56ac | ||
|
|
cefe3eb520 | ||
|
|
64dffa306f | ||
|
|
7d33b4f1f3 | ||
|
|
f507d95198 | ||
|
|
2a7d80f8b4 | ||
|
|
305034ffb2 | ||
|
|
bfaebf0085 | ||
|
|
a94cf04e87 | ||
|
|
71f9f8ae9c | ||
|
|
330862e487 | ||
|
|
778d3826ab | ||
|
|
4a11cf007f | ||
|
|
f6e7700c4c | ||
|
|
573285a5bb | ||
|
|
60ed4e503f | ||
|
|
aa1405887e | ||
|
|
cced217e43 | ||
|
|
8a3b8d2df1 | ||
|
|
be0bd3bdea | ||
|
|
e01c73a7f8 | ||
|
|
134338db4b | ||
|
|
f47a685734 | ||
|
|
6e29f7e99b | ||
|
|
026d93e92a | ||
|
|
a55802c1de | ||
|
|
894397a327 | ||
|
|
3e3f6c0e22 | ||
|
|
a7610e3d34 | ||
|
|
d0e6e74e1b | ||
|
|
126d71978b | ||
|
|
69bcb052de | ||
|
|
071af12d1a | ||
|
|
b7d62ca084 | ||
|
|
fa97f7bd38 | ||
|
|
4a35247016 | ||
|
|
d0526ad7d5 | ||
|
|
19b529cf38 | ||
|
|
143353be44 | ||
|
|
0373196707 | ||
|
|
7a601db23f | ||
|
|
ed61c7feec | ||
|
|
77ece96356 | ||
|
|
80440fd6bd | ||
|
|
ed6537db79 | ||
|
|
79e4767f69 | ||
|
|
0ee279bfd7 | ||
|
|
5249d2a3ca | ||
|
|
4d5823f473 | ||
|
|
c2b9909397 | ||
|
|
afff362544 | ||
|
|
64eb7dbc49 | ||
|
|
2dcd187d69 | ||
|
|
3d9ea92118 | ||
|
|
13a18d422e | ||
|
|
1bbbbd289c | ||
|
|
e33b74f60c | ||
|
|
cd76e0c296 | ||
|
|
3b61222e5a | ||
|
|
2ca438c36e | ||
|
|
8948d5be86 | ||
|
|
a3e64f9cef | ||
|
|
09afec1961 | ||
|
|
f9c465a636 | ||
|
|
89efe3d1b9 | ||
|
|
94cec1b86d | ||
|
|
d62ac0b608 | ||
|
|
5596eac930 | ||
|
|
59f5381b0a | ||
|
|
65ae1b3f08 | ||
|
|
977a7b34f4 | ||
|
|
c01728a6a7 | ||
|
|
5bb94d744e | ||
|
|
494917158c | ||
|
|
ea15674e62 | ||
|
|
34a409ade5 | ||
|
|
273e614715 | ||
|
|
34a725f7ad | ||
|
|
bb6112276f | ||
|
|
db75f0baef | ||
|
|
30689bbf8d | ||
|
|
4bf7742c19 | ||
|
|
69b259c285 | ||
|
|
2fa5233796 | ||
|
|
a5524d490c | ||
|
|
c713fac2d9 | ||
|
|
7a038bbb98 | ||
|
|
eba88c1df7 | ||
|
|
881e5aba68 | ||
|
|
368f0683ec | ||
|
|
94dc8c8520 |
No files matched your search
@@ -1,9 +1,13 @@
|
||||
.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
/lbry-venv
|
||||
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
@@ -33,9 +33,7 @@ unsafe-load-any-extension=no
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
miniupnpc,
|
||||
unqlite
|
||||
# extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
@@ -354,13 +352,13 @@ ignore-mixin-members=yes
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb,distutils
|
||||
ignored-modules=leveldb,distutils
|
||||
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet.reactor,RequestMessage
|
||||
# ignored-classes=
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
|
||||
+41
-28
@@ -3,47 +3,53 @@ dist: xenial
|
||||
language: python
|
||||
python: "3.7"
|
||||
|
||||
env:
|
||||
- TORBA=v0.5.3
|
||||
|
||||
jobs:
|
||||
include:
|
||||
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install astroid pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
- &tests
|
||||
stage: test
|
||||
- stage: test
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .[test]
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
|
||||
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
|
||||
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
@@ -53,7 +59,7 @@ jobs:
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh ${TORBA}
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
@@ -77,8 +83,8 @@ jobs:
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- python scripts/set_build.py
|
||||
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
@@ -113,15 +119,22 @@ jobs:
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
directories:
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
## Contributing to LBRY
|
||||
|
||||
https://lbry.io/faq/contributing
|
||||
https://lbry.tech/contribute
|
||||
+11
-12
@@ -13,15 +13,6 @@ Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
@@ -29,15 +20,17 @@ macOS users will need to install [xcode command line tools](https://developer.xa
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python3 protobuf
|
||||
brew install python protobuf
|
||||
```
|
||||
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
@@ -52,8 +45,14 @@ To install:
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
Creating venv:
|
||||
python -m venv lbry-venv
|
||||
|
||||
Activating lbry-venv on Linux/Mac:
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
Activating lbry-venv on Windows:
|
||||
lbry-venv\Scripts\activate
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2018 LBRY Inc
|
||||
Copyright (c) 2015-2019 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
|
||||
## License
|
||||
|
||||
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
|
||||
+2612
-1526
File diff suppressed because it is too large.
Load diff
+2
-4
@@ -1409,7 +1409,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1954,7 +1954,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1979,7 +1978,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2586,4 +2584,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
+2
-4
@@ -1339,7 +1339,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1786,7 +1786,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1811,7 +1810,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2312,4 +2310,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -4,20 +4,22 @@
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.io:50001
|
||||
- lbryumx2.lbry.io:50001
|
||||
- lbryumx1.lbry.com:50001
|
||||
- lbryumx2.lbry.com:50001
|
||||
- lbryumx4.lbry.com:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
use_upnp: true
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - blob_server
|
||||
# - dht
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.5"
|
||||
__version__ = "0.37.2"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from lbrynet.p2p.BlobInfo import BlobInfo
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
backend = default_backend()
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
super().__init__(blob_hash, blob_num, length)
|
||||
self.iv = iv
|
||||
|
||||
def get_dict(self):
|
||||
info = {
|
||||
"blob_num": self.blob_num,
|
||||
"length": self.length,
|
||||
"iv": self.iv.decode()
|
||||
}
|
||||
if self.blob_hash:
|
||||
info['blob_hash'] = self.blob_hash
|
||||
return info
|
||||
|
||||
|
||||
class StreamBlobDecryptor:
|
||||
def __init__(self, blob, key, iv, length):
|
||||
"""
|
||||
This class decrypts blob
|
||||
|
||||
blob - object which implements read() function.
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
length = length in bytes of blob
|
||||
"""
|
||||
self.blob = blob
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
self.length = length
|
||||
self.buff = b''
|
||||
self.len_read = 0
|
||||
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
|
||||
self.unpadder = PKCS7(AES.block_size).unpadder()
|
||||
self.cipher = cipher.decryptor()
|
||||
|
||||
def decrypt(self, write_func):
|
||||
"""
|
||||
Decrypt blob and write its content using write_func
|
||||
|
||||
write_func - function that takes decrypted string as
|
||||
argument and writes it somewhere
|
||||
|
||||
Returns:
|
||||
|
||||
deferred that returns after decrypting blob and writing content
|
||||
"""
|
||||
|
||||
def remove_padding(data):
|
||||
return self.unpadder.update(data) + self.unpadder.finalize()
|
||||
|
||||
def write_bytes():
|
||||
if self.len_read < self.length:
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size // 8))
|
||||
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
|
||||
write_func(self.cipher.update(data_to_decrypt))
|
||||
|
||||
def finish_decrypt():
|
||||
bytes_left = len(self.buff) % (AES.block_size // 8)
|
||||
if bytes_left != 0:
|
||||
log.warning(binascii.hexlify(self.buff[-1 * (AES.block_size // 8):]).decode())
|
||||
raise Exception("blob %s has incorrect padding: %i bytes left" %
|
||||
(self.blob.blob_hash, bytes_left))
|
||||
data_to_decrypt, self.buff = self.buff, b''
|
||||
last_chunk = self.cipher.update(data_to_decrypt) + self.cipher.finalize()
|
||||
write_func(remove_padding(last_chunk))
|
||||
|
||||
|
||||
read_handle = self.blob.open_for_reading()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def decrypt_bytes():
|
||||
producer = FileBodyProducer(read_handle)
|
||||
buff = BytesIO()
|
||||
yield producer.startProducing(buff)
|
||||
self.buff = buff.getvalue()
|
||||
self.len_read += len(self.buff)
|
||||
write_bytes()
|
||||
finish_decrypt()
|
||||
|
||||
d = decrypt_bytes()
|
||||
return d
|
||||
|
||||
|
||||
class CryptStreamBlobMaker:
|
||||
def __init__(self, key, iv, blob_num, blob):
|
||||
"""
|
||||
This class encrypts data and writes it to a new blob
|
||||
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
blob = object which implements write(), close() function , close() function must
|
||||
be a deferred. (Will generally be of HashBlobCreator type)
|
||||
"""
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
self.blob_num = blob_num
|
||||
self.blob = blob
|
||||
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
|
||||
self.padder = PKCS7(AES.block_size).padder()
|
||||
self.cipher = cipher.encryptor()
|
||||
self.length = 0
|
||||
|
||||
def write(self, data):
|
||||
"""
|
||||
encrypt and write string data
|
||||
|
||||
Returns:
|
||||
tuple (done, num_bytes_to_write) where done is True if
|
||||
max bytes are written. num_bytes_to_write is the number
|
||||
of bytes that will be written from data in this call
|
||||
"""
|
||||
max_bytes_to_write = MAX_BLOB_SIZE - self.length - 1
|
||||
done = False
|
||||
if max_bytes_to_write <= len(data):
|
||||
num_bytes_to_write = max_bytes_to_write
|
||||
done = True
|
||||
else:
|
||||
num_bytes_to_write = len(data)
|
||||
data_to_write = data[:num_bytes_to_write]
|
||||
self.length += len(data_to_write)
|
||||
padded_data = self.padder.update(data_to_write)
|
||||
encrypted_data = self.cipher.update(padded_data)
|
||||
self.blob.write(encrypted_data)
|
||||
return done, num_bytes_to_write
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def close(self):
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self.length += (AES.block_size // 8) - (self.length % (AES.block_size // 8))
|
||||
padded_data = self.padder.finalize()
|
||||
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
blob_hash = yield self.blob.close()
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
blob = CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
defer.returnValue(blob)
|
||||
|
||||
|
||||
def greatest_multiple(a, b):
|
||||
"""return the largest value `c`, that is a multiple of `b` and is <= `a`"""
|
||||
return (a // b) * b
|
||||
|
||||
|
||||
def split(buff, cutoff):
|
||||
return buff[:cutoff], buff[cutoff:]
|
||||
@@ -1,149 +0,0 @@
|
||||
"""
|
||||
Utility for creating Crypt Streams, which are encrypted blobs and associated metadata.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from twisted.internet import defer
|
||||
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator:
|
||||
"""
|
||||
Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
Each blob is encrypted with the same key, but each blob has its
|
||||
own initialization vector which is associated with the blob when
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
|
||||
#implements(interfaces.IConsumer)
|
||||
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@type blob_manager: BlobManager
|
||||
|
||||
@param name: the name of the stream, which will be presented to the user
|
||||
@type name: string
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the
|
||||
blobs. If None, a random key will be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self.blob_manager = blob_manager
|
||||
self.name = name
|
||||
self.key = key
|
||||
if iv_generator is None:
|
||||
self.iv_generator = self.random_iv_generator()
|
||||
else:
|
||||
self.iv_generator = iv_generator
|
||||
|
||||
self.stopped = True
|
||||
self.producer = None
|
||||
self.streaming = None
|
||||
self.blob_count = -1
|
||||
self.current_blob = None
|
||||
self.finished_deferreds = []
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.producer = producer
|
||||
self.streaming = streaming
|
||||
self.stopped = False
|
||||
if streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.stopped = True
|
||||
self.producer = None
|
||||
|
||||
def _close_current_blob(self):
|
||||
# close the blob that was being written to
|
||||
# and save it to blob manager
|
||||
should_announce = self.blob_count == 0
|
||||
d = self.current_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
d.addCallback(lambda blob_info: self.blob_manager.creator_finished(blob_info,
|
||||
should_announce))
|
||||
self.finished_deferreds.append(d)
|
||||
self.current_blob = None
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
self._close_current_blob()
|
||||
d = self._finalize()
|
||||
d.addCallback(lambda _: self._finished())
|
||||
return d
|
||||
|
||||
# TODO: move the stream creation process to its own thread and
|
||||
# remove the reactor from this process.
|
||||
def write(self, data):
|
||||
from twisted.internet import reactor
|
||||
self._write(data)
|
||||
if self.stopped is False and self.streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
@staticmethod
|
||||
def random_iv_generator():
|
||||
while 1:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
def setup(self):
|
||||
"""Create the symmetric key if it wasn't provided"""
|
||||
|
||||
if self.key is None:
|
||||
self.key = os.urandom(AES.block_size // 8)
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _finalize(self):
|
||||
"""
|
||||
Finalize a stream by adding an empty
|
||||
blob at the end, this is to indicate that
|
||||
the stream has ended. This empty blob is not
|
||||
saved to the blob manager
|
||||
"""
|
||||
|
||||
yield defer.DeferredList(self.finished_deferreds)
|
||||
self.blob_count += 1
|
||||
iv = next(self.iv_generator)
|
||||
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
|
||||
stream_terminator = yield final_blob.close()
|
||||
terminator_info = yield self._blob_finished(stream_terminator)
|
||||
defer.returnValue(terminator_info)
|
||||
|
||||
def _write(self, data):
|
||||
while len(data) > 0:
|
||||
if self.current_blob is None:
|
||||
self.next_blob_creator = self.blob_manager.get_blob_creator()
|
||||
self.blob_count += 1
|
||||
iv = next(self.iv_generator)
|
||||
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
|
||||
done, num_bytes_written = self.current_blob.write(data)
|
||||
data = data[num_bytes_written:]
|
||||
if done is True:
|
||||
self._close_current_blob()
|
||||
|
||||
def _get_blob_maker(self, iv, blob_creator):
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
|
||||
def _finished(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
raise NotImplementedError()
|
||||
@@ -1,132 +0,0 @@
|
||||
"""
|
||||
Utilities for turning plain files into LBRY Files.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
"""
|
||||
A CryptStreamCreator which adds itself and its additional metadata to an EncryptedFileManager
|
||||
"""
|
||||
|
||||
def __init__(self, blob_manager, lbry_file_manager, stream_name=None,
|
||||
key=None, iv_generator=None):
|
||||
super().__init__(blob_manager, stream_name, key, iv_generator)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.stream_hash = None
|
||||
self.blob_infos = []
|
||||
self.sd_info = None
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
log.debug("length: %s", blob_info.length)
|
||||
self.blob_infos.append(blob_info.get_dict())
|
||||
return blob_info
|
||||
|
||||
def _finished(self):
|
||||
# calculate the stream hash
|
||||
self.stream_hash = get_stream_hash(
|
||||
hexlify(self.name.encode()).decode(), hexlify(self.key).decode(), hexlify(self.name.encode()).decode(),
|
||||
self.blob_infos
|
||||
)
|
||||
|
||||
# generate the sd info
|
||||
self.sd_info = format_sd_info(
|
||||
EncryptedFileStreamType, hexlify(self.name.encode()).decode(), hexlify(self.key).decode(),
|
||||
hexlify(self.name.encode()).decode(), self.stream_hash, self.blob_infos
|
||||
)
|
||||
|
||||
# sanity check
|
||||
validate_descriptor(self.sd_info)
|
||||
return defer.succeed(self.stream_hash)
|
||||
|
||||
|
||||
# TODO: this should be run its own thread. Encrypting a large file can
|
||||
# be very cpu intensive and there is no need to run that on the
|
||||
# main reactor thread. The FileSender mechanism that is used is
|
||||
# great when sending over the network, but this is all local so
|
||||
# we can simply read the file from the disk without needing to
|
||||
# involve reactor.
|
||||
@defer.inlineCallbacks
|
||||
def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_manager, file_name, file_handle,
|
||||
key=None, iv_generator=None):
|
||||
"""Turn a plain file into an LBRY File.
|
||||
|
||||
An LBRY File is a collection of encrypted blobs of data and the metadata that binds them
|
||||
together which, when decrypted and put back together according to the metadata, results
|
||||
in the original file.
|
||||
|
||||
The stream parameters that aren't specified are generated, the file is read and broken
|
||||
into chunks and encrypted, and then a stream descriptor file with the stream parameters
|
||||
and other metadata is written to disk.
|
||||
|
||||
@param session: An Session object.
|
||||
@type session: Session
|
||||
|
||||
@param lbry_file_manager: The EncryptedFileManager object this LBRY File will be added to.
|
||||
@type lbry_file_manager: EncryptedFileManager
|
||||
|
||||
@param file_name: The path to the plain file.
|
||||
@type file_name: string
|
||||
|
||||
@param file_handle: The file-like object to read
|
||||
@type file_handle: any file-like object which can be read by twisted.protocols.basic.FileSender
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
|
||||
be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: a Deferred which fires with the stream_hash of the LBRY File
|
||||
@rtype: Deferred which fires with hex-encoded string
|
||||
"""
|
||||
|
||||
base_file_name = os.path.basename(file_name)
|
||||
file_directory = os.path.dirname(file_handle.name)
|
||||
|
||||
lbry_file_creator = EncryptedFileStreamCreator(
|
||||
blob_manager, lbry_file_manager, base_file_name, key, iv_generator
|
||||
)
|
||||
|
||||
yield lbry_file_creator.setup()
|
||||
# TODO: Using FileSender isn't necessary, we can just read
|
||||
# straight from the disk. The stream creation process
|
||||
# should be in its own thread anyway so we don't need to
|
||||
# worry about interacting with the twisted reactor
|
||||
file_sender = FileSender()
|
||||
yield file_sender.beginFileTransfer(file_handle, lbry_file_creator)
|
||||
|
||||
log.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
yield lbry_file_creator.stop()
|
||||
|
||||
log.debug("making the sd blob")
|
||||
sd_info = lbry_file_creator.sd_info
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield f2d(storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
))
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield f2d(lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
@@ -1,189 +0,0 @@
|
||||
"""
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def log_status(sd_hash, status):
|
||||
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
|
||||
status_string = "running"
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
|
||||
status_string = "stopped"
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
status_string = "finished"
|
||||
else:
|
||||
status_string = "unknown"
|
||||
log.debug("stream %s is %s", short_hash(sd_hash), status_string)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
|
||||
sd_hash, key, suggested_file_name, download_mirrors=None):
|
||||
super().__init__(
|
||||
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
|
||||
wallet, download_directory, key, stream_name, file_name
|
||||
)
|
||||
self.sd_hash = sd_hash
|
||||
self.rowid = rowid
|
||||
self.suggested_file_name = unhexlify(suggested_file_name).decode()
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self._saving_status = False
|
||||
self.claim_id = None
|
||||
self.outpoint = None
|
||||
self.claim_name = None
|
||||
self.txid = None
|
||||
self.nout = None
|
||||
self.channel_claim_id = None
|
||||
self.channel_name = None
|
||||
self.metadata = None
|
||||
self.mirror = None
|
||||
if download_mirrors or conf.download_mirrors:
|
||||
self.mirror = HTTPBlobDownloader(
|
||||
self.blob_manager, servers=download_mirrors or conf.download_mirrors
|
||||
)
|
||||
|
||||
def set_claim_info(self, claim_info):
|
||||
self.claim_id = claim_info['claim_id']
|
||||
self.txid = claim_info['txid']
|
||||
self.nout = claim_info['nout']
|
||||
self.channel_claim_id = claim_info['channel_claim_id']
|
||||
self.outpoint = "%s:%i" % (self.txid, self.nout)
|
||||
self.claim_name = claim_info['name']
|
||||
self.channel_name = claim_info['channel_name']
|
||||
self.metadata = claim_info['value']['stream']['metadata']
|
||||
|
||||
async def get_claim_info(self, include_supports=True):
|
||||
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
if claim_info:
|
||||
self.set_claim_info(claim_info)
|
||||
return claim_info
|
||||
|
||||
@property
|
||||
def saving_status(self):
|
||||
return self._saving_status
|
||||
|
||||
def restore(self, status):
|
||||
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
|
||||
# start returns self.finished_deferred
|
||||
# which fires when we've finished downloading the file
|
||||
# and we don't want to wait for the entire download
|
||||
self.start()
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
|
||||
pass
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
self.completed = True
|
||||
else:
|
||||
raise Exception(f"Unknown status for stream {self.stream_hash}: {status}")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None, change_status=True):
|
||||
log.debug('Stopping download for stream %s', short_hash(self.stream_hash))
|
||||
if self.mirror:
|
||||
self.mirror.stop()
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
yield super().stop(err)
|
||||
if change_status is True:
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
|
||||
async def status(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
|
||||
num_blobs_completed = len(completed_blobs)
|
||||
num_blobs_known = len(blob_hashes)
|
||||
|
||||
if self.completed:
|
||||
status = "completed"
|
||||
elif self.stopped:
|
||||
status = "stopped"
|
||||
else:
|
||||
status = "running"
|
||||
|
||||
return EncryptedFileStatusReport(
|
||||
self.file_name, num_blobs_completed, num_blobs_known, status
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
yield EncryptedFileSaver._start(self)
|
||||
status = yield self._save_status()
|
||||
log_status(self.sd_hash, status)
|
||||
if self.mirror:
|
||||
self.mirror.download_stream(self.stream_hash, self.sd_hash)
|
||||
defer.returnValue(status)
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
if self.completed is True:
|
||||
return "Download successful"
|
||||
else:
|
||||
return "Download stopped"
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _save_status(self):
|
||||
self._saving_status = True
|
||||
if self.completed is True:
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
elif self.stopped is True:
|
||||
status = ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
else:
|
||||
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
|
||||
self._saving_status = False
|
||||
return status
|
||||
|
||||
def save_status(self):
|
||||
return self._save_status()
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, lbry_file_manager, blob_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def can_download(self, sd_validator):
|
||||
# TODO: add a sd_validator for non live streams, use it
|
||||
return True
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def make_downloader(self, metadata, data_rate, payment_rate_manager, download_directory, file_name=None,
|
||||
download_mirrors=None):
|
||||
stream_hash = yield save_sd_info(self.blob_manager,
|
||||
metadata.source_blob_hash,
|
||||
metadata.validator.raw_info)
|
||||
if file_name:
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save the file to disk"
|
||||
@@ -1,254 +0,0 @@
|
||||
"""
|
||||
Keep track of which LBRY Files are downloading and store their LBRY File specific metadata
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileManager:
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and
|
||||
their LBRY File specific metadata.
|
||||
"""
|
||||
# when reflecting files, reflect up to this many files at a time
|
||||
CONCURRENT_REFLECTS = 5
|
||||
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier):
|
||||
self.conf = conf
|
||||
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
|
||||
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.storage = storage
|
||||
# TODO: why is sd_identifier part of the file manager?
|
||||
self.sd_identifier = sd_identifier
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
def setup(self):
|
||||
self._add_to_sd_identifier()
|
||||
return self._start_lbry_files()
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
|
||||
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
|
||||
return self.storage(lbry_file.rowid, new_rate)
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
|
||||
for lbry_file in self.lbry_files:
|
||||
ds.append(lbry_file.status())
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def filter_failures(status_reports):
|
||||
return [status_report for success, status_report in status_reports if success is True]
|
||||
|
||||
dl.addCallback(filter_failures)
|
||||
return dl
|
||||
|
||||
def _add_to_sd_identifier(self):
|
||||
downloader_factory = ManagedEncryptedFileDownloaderFactory(self, self.blob_manager)
|
||||
self.sd_identifier.add_stream_downloader_factory(
|
||||
EncryptedFileStreamType, downloader_factory)
|
||||
|
||||
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
|
||||
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
|
||||
return ManagedEncryptedFileDownloader(
|
||||
self.conf,
|
||||
rowid,
|
||||
stream_hash,
|
||||
self.peer_finder,
|
||||
self.rate_limiter,
|
||||
self.blob_manager,
|
||||
self.storage,
|
||||
self,
|
||||
payment_rate_manager,
|
||||
self.wallet,
|
||||
download_directory,
|
||||
file_name,
|
||||
stream_name=stream_name,
|
||||
sd_hash=sd_hash,
|
||||
key=key,
|
||||
suggested_file_name=suggested_file_name,
|
||||
download_mirrors=download_mirrors
|
||||
)
|
||||
|
||||
def _start_lbry_file(self, file_info, payment_rate_manager, claim_info, download_mirrors=None):
|
||||
lbry_file = self._get_lbry_file(
|
||||
file_info['row_id'], file_info['stream_hash'], payment_rate_manager, file_info['sd_hash'],
|
||||
file_info['key'], file_info['stream_name'], file_info['file_name'], file_info['download_directory'],
|
||||
file_info['suggested_file_name'], download_mirrors
|
||||
)
|
||||
if claim_info:
|
||||
lbry_file.set_claim_info(claim_info)
|
||||
try:
|
||||
# restore will raise an Exception if status is unknown
|
||||
lbry_file.restore(file_info['status'])
|
||||
self.storage.content_claim_callbacks[lbry_file.stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
if len(self.lbry_files) % 500 == 0:
|
||||
log.info("Started %i files", len(self.lbry_files))
|
||||
except Exception:
|
||||
log.warning("Failed to start %i", file_info.get('rowid'))
|
||||
|
||||
async def _start_lbry_files(self):
|
||||
files = await self.storage.get_all_lbry_files()
|
||||
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
for file_info in files:
|
||||
claim_info = claim_infos.get(file_info['stream_hash'])
|
||||
self._start_lbry_file(file_info, prm, claim_info)
|
||||
|
||||
log.info("Started %i lbry files", len(self.lbry_files))
|
||||
if self.auto_re_reflect is True:
|
||||
safe_start_looping_call(self.lbry_file_reflector, self.auto_re_reflect_interval / 10)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop_lbry_file(self, lbry_file):
|
||||
def wait_for_finished(lbry_file, count=2):
|
||||
if count or lbry_file.saving_status is not False:
|
||||
return task.deferLater(reactor, 1, self._stop_lbry_file, lbry_file,
|
||||
count=count - 1)
|
||||
try:
|
||||
yield lbry_file.stop(change_status=False)
|
||||
self.lbry_files.remove(lbry_file)
|
||||
except CurrentlyStoppingError:
|
||||
yield wait_for_finished(lbry_file)
|
||||
except AlreadyStoppedError:
|
||||
pass
|
||||
finally:
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop_lbry_files(self):
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
rowid = await self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate, status
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
await lbry_file.get_claim_info()
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = await self.storage.save_downloaded_file(
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
await lbry_file.get_claim_info(include_supports=False)
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_lbry_file(self, lbry_file, delete_file=False):
|
||||
if lbry_file not in self.lbry_files:
|
||||
raise ValueError("Could not find that LBRY file")
|
||||
|
||||
def wait_for_finished(count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
return True
|
||||
else:
|
||||
return task.deferLater(reactor, 1, wait_for_finished, count=count - 1)
|
||||
|
||||
full_path = os.path.join(lbry_file.download_directory, lbry_file.file_name)
|
||||
|
||||
try:
|
||||
yield lbry_file.stop()
|
||||
except (AlreadyStoppedError, CurrentlyStoppingError):
|
||||
yield wait_for_finished()
|
||||
|
||||
self.lbry_files.remove(lbry_file)
|
||||
|
||||
if lbry_file.stream_hash in self.storage.content_claim_callbacks:
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
|
||||
defer.returnValue(True)
|
||||
|
||||
def toggle_lbry_file_running(self, lbry_file):
|
||||
"""Toggle whether a stream reader is currently running"""
|
||||
for l in self.lbry_files:
|
||||
if l == lbry_file:
|
||||
return l.toggle_running()
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def reflect_lbry_files(self):
|
||||
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
|
||||
ds = []
|
||||
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
|
||||
for lbry_file in self.lbry_files:
|
||||
if lbry_file.sd_hash in sd_hashes_to_reflect:
|
||||
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
|
||||
yield defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
safe_stop_looping_call(self.lbry_file_reflector)
|
||||
yield self._stop_lbry_files()
|
||||
log.info("Stopped encrypted file manager")
|
||||
defer.returnValue(True)
|
||||
@@ -1,6 +0,0 @@
|
||||
class EncryptedFileStatusReport:
|
||||
def __init__(self, name, num_completed, num_known, running_status):
|
||||
self.name = name
|
||||
self.num_completed = num_completed
|
||||
self.num_known = num_known
|
||||
self.running_status = running_status
|
||||
@@ -0,0 +1,6 @@
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
+289
-171
@@ -1,26 +1,29 @@
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
import typing
|
||||
import contextlib
|
||||
from io import BytesIO
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
|
||||
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
|
||||
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
|
||||
|
||||
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.blob.reader import HashBlobReader
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@@ -28,190 +31,305 @@ def is_valid_blobhash(blobhash):
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
|
||||
|
||||
|
||||
class BlobFile:
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
|
||||
digest = get_lbry_hash_obj()
|
||||
digest.update(encrypted)
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
|
||||
if len(data) != length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
|
||||
class AbstractBlob:
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
|
||||
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we already know the blob hash before hand (i.e., when downloading blobs)
|
||||
Also can be used for reading from blobs on the local filesystem
|
||||
This class is non-io specific
|
||||
"""
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_hash',
|
||||
'length',
|
||||
'blob_completed_callback',
|
||||
'blob_directory',
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return self.blob_hash[:16]
|
||||
|
||||
def __repr__(self):
|
||||
return '<{}({})>'.format(self.__class__.__name__, str(self))
|
||||
|
||||
def __init__(self, blob_dir, blob_hash, length=None):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.writers = {} # {Peer: writer, finished_deferred}
|
||||
self._verified = False
|
||||
self.readers = 0
|
||||
self.blob_dir = blob_dir
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.blob_write_lock = defer.DeferredLock()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
self.set_length(os.path.getsize(self.file_path))
|
||||
# This assumes that the hash of the blob has already been
|
||||
# checked as part of the blob creation process. It might
|
||||
# be worth having a function that checks the actual hash;
|
||||
# its probably too expensive to have that check be part of
|
||||
# this call.
|
||||
self._verified = True
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
self.blob_directory = blob_directory
|
||||
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
"""
|
||||
open a blob file to be written by peer, supports concurrent
|
||||
writers, as long as they are from different peers.
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
|
||||
returns tuple of (writer, finished_deferred)
|
||||
def __del__(self):
|
||||
if self.writers or self.readers:
|
||||
log.warning("%s not closed before being garbage collected", self.blob_hash)
|
||||
self.close()
|
||||
|
||||
writer - a file like object with a write() function, close() when finished
|
||||
finished_deferred - deferred that is fired when write is finished and returns
|
||||
a instance of itself as HashBlob
|
||||
"""
|
||||
if peer not in self.writers:
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(self.get_length, self.writer_finished)
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return writer, finished_deferred
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
raise NotImplementedError()
|
||||
|
||||
def open_for_reading(self):
|
||||
"""
|
||||
open blob for reading
|
||||
|
||||
returns a file like object that can be read() from, and closed() when
|
||||
finished
|
||||
"""
|
||||
if self._verified is True:
|
||||
f = open(self.file_path, 'rb')
|
||||
reader = HashBlobReader(f, self.reader_finished)
|
||||
self.readers += 1
|
||||
return reader
|
||||
return None
|
||||
|
||||
def delete(self):
|
||||
"""
|
||||
delete blob file from file system, prevent deletion
|
||||
if a blob is being read from or written to
|
||||
|
||||
returns a deferred that firesback when delete is completed
|
||||
"""
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
self.saved_verified_blob = False
|
||||
@contextlib.contextmanager
|
||||
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
|
||||
with self._reader_context() as reader:
|
||||
try:
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
except Exception as e:
|
||||
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
|
||||
else:
|
||||
raise ValueError("File is currently being read or written and cannot be deleted")
|
||||
self.readers.append(reader)
|
||||
yield reader
|
||||
finally:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
"""
|
||||
Protect verified from being modified by other classes.
|
||||
verified is True if a write to a blob has completed successfully,
|
||||
or a blob has been read to have the same length as specified
|
||||
in init
|
||||
"""
|
||||
return self._verified
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length):
|
||||
def set_length(self, length) -> None:
|
||||
if self.length is not None and length == self.length:
|
||||
return True
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
self.length = length
|
||||
return True
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
|
||||
self.length, length)
|
||||
return False
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self):
|
||||
def get_length(self) -> typing.Optional[int]:
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self):
|
||||
return self.verified
|
||||
def get_is_verified(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_downloading(self):
|
||||
if self.writers:
|
||||
return True
|
||||
return False
|
||||
def is_readable(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def reader_finished(self, reader):
|
||||
self.readers -= 1
|
||||
return defer.succeed(True)
|
||||
def is_writeable(self) -> bool:
|
||||
return not self.writing.is_set()
|
||||
|
||||
def writer_finished(self, writer, err=None):
|
||||
def fire_finished_deferred():
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.callback(self)
|
||||
return True
|
||||
log.warning(
|
||||
"Somehow, the writer that was accepted as being valid was already removed: %s",
|
||||
writer)
|
||||
return False
|
||||
def write_blob(self, blob_bytes: bytes):
|
||||
if not self.is_writeable():
|
||||
raise OSError("cannot open blob for writing")
|
||||
try:
|
||||
self.writing.set()
|
||||
self._write_blob(blob_bytes)
|
||||
finally:
|
||||
self.writing.clear()
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.errback(err)
|
||||
def close(self):
|
||||
while self.writers:
|
||||
peer, writer = self.writers.popitem()
|
||||
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
|
||||
writer.finished.cancel()
|
||||
while self.readers:
|
||||
reader = self.readers.pop()
|
||||
if reader:
|
||||
reader.close()
|
||||
|
||||
def cancel_other_downloads():
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
w.close()
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.verified.clear()
|
||||
self.length = None
|
||||
|
||||
if err is None:
|
||||
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
|
||||
if self._verified is False:
|
||||
d = self.save_verified_blob(writer)
|
||||
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
|
||||
d.addCallback(lambda _: cancel_other_downloads())
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
fire_finished_deferred()
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with self.reader_context() as reader:
|
||||
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
|
||||
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers[(peer_address, peer_port)] = writer
|
||||
|
||||
def remove_writer(_):
|
||||
if (peer_address, peer_port) in self.writers:
|
||||
del self.writers[(peer_address, peer_port)]
|
||||
|
||||
fut.add_done_callback(remove_writer)
|
||||
|
||||
def writer_finished_callback(finished: asyncio.Future):
|
||||
try:
|
||||
err = finished.exception()
|
||||
if err:
|
||||
raise err
|
||||
verified_bytes = finished.result()
|
||||
while self.writers:
|
||||
_, other = self.writers.popitem()
|
||||
if other is not writer:
|
||||
other.close_handle()
|
||||
self.save_verified_blob(verified_bytes)
|
||||
except (InvalidBlobHashError, InvalidDataError) as error:
|
||||
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
|
||||
fut.add_done_callback(writer_finished_callback)
|
||||
return writer
|
||||
|
||||
|
||||
class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError("cannot open blob for reading")
|
||||
try:
|
||||
yield self._verified_bytes
|
||||
finally:
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
return super().delete()
|
||||
|
||||
def __del__(self):
|
||||
super().__del__()
|
||||
if self._verified_bytes:
|
||||
self.delete()
|
||||
|
||||
|
||||
class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
if self.file_exists:
|
||||
file_size = int(os.stat(self.file_path).st_size)
|
||||
if length and length != file_size:
|
||||
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
|
||||
self.delete()
|
||||
else:
|
||||
if writer.len_so_far != self.length:
|
||||
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
|
||||
else:
|
||||
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(None)
|
||||
else:
|
||||
errback_finished_deferred(err)
|
||||
d = defer.succeed(None)
|
||||
d.addBoth(lambda _: writer.close_handle())
|
||||
return d
|
||||
self.length = file_size
|
||||
self.verified.set()
|
||||
|
||||
def save_verified_blob(self, writer):
|
||||
# we cannot have multiple _save_verified_blob interrupting
|
||||
# each other, can happen since startProducing is a deferred
|
||||
return self.blob_write_lock.run(self._save_verified_blob, writer)
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _save_verified_blob(self, writer):
|
||||
if self.saved_verified_blob is False:
|
||||
writer.write_handle.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(writer.write_handle)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self.saved_verified_blob = True
|
||||
defer.returnValue(True)
|
||||
else:
|
||||
raise DownloadCanceledError()
|
||||
def is_writeable(self) -> bool:
|
||||
return super().is_writeable() and not os.path.isfile(self.file_path)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
return super().get_blob_writer(peer_address, peer_port)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
handle = open(self.file_path, 'rb')
|
||||
try:
|
||||
yield handle
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
__slots__ = [
|
||||
'blob_hash',
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.iv = iv
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
'length': self.length,
|
||||
'blob_num': self.blob_num,
|
||||
'iv': self.iv,
|
||||
}
|
||||
if self.blob_hash: # non-terminator blobs have a blob hash
|
||||
d['blob_hash'] = self.blob_hash
|
||||
return d
|
||||
@@ -0,0 +1,131 @@
|
||||
import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbrynet.utils import LRUCache
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
|
||||
blob_dir - directory where blobs are stored
|
||||
storage - SQLiteStorage object
|
||||
"""
|
||||
self.loop = loop
|
||||
self.blob_dir = blob_dir
|
||||
self.storage = storage
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs:
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
else:
|
||||
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise ValueError(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
return self.blobs[blob_hash].get_is_verified()
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
return self._get_blob(blob_hash, length).get_is_verified()
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
if not self.blob_dir:
|
||||
return set()
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
|
||||
for blob_hash in blob_hashes:
|
||||
self.delete_blob(blob_hash)
|
||||
|
||||
if delete_from_db:
|
||||
await self.storage.delete_blobs_from_db(blob_hashes)
|
||||
@@ -1,22 +0,0 @@
|
||||
import binascii
|
||||
from twisted.internet import defer
|
||||
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
|
||||
|
||||
|
||||
class CryptBlobHandler:
|
||||
#implements(IBlobHandler)
|
||||
|
||||
def __init__(self, key, write_func):
|
||||
self.key = key
|
||||
self.write_func = write_func
|
||||
|
||||
######## IBlobHandler #########
|
||||
|
||||
def handle_blob(self, blob, blob_info):
|
||||
try:
|
||||
blob_decryptor = StreamBlobDecryptor(blob, self.key, binascii.unhexlify(blob_info.iv),
|
||||
blob_info.length)
|
||||
except ValueError as err:
|
||||
return defer.fail(err)
|
||||
d = blob_decryptor.decrypt(self.write_func)
|
||||
return d
|
||||
@@ -1,221 +0,0 @@
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.p2p.client.DownloadManager import DownloadManager
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartFailedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AlreadyRunningError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AlreadyStoppedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CurrentlyStoppingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CurrentlyStartingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CryptStreamDownloader:
|
||||
|
||||
#implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
key, stream_name):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@param peer_finder: An object which implements the IPeerFinder
|
||||
interface. Used to look up peers by a hashsum.
|
||||
|
||||
@param rate_limiter: An object which implements the IRateLimiter interface
|
||||
|
||||
@param blob_manager: A BlobManager object
|
||||
|
||||
@param payment_rate_manager: A NegotiatedPaymentRateManager object
|
||||
|
||||
@param wallet: An object which implements the IWallet interface
|
||||
|
||||
@return:
|
||||
|
||||
"""
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.key = unhexlify(key)
|
||||
self.stream_name = unhexlify(stream_name).decode()
|
||||
self.completed = False
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
self.starting = False
|
||||
self.download_manager = None
|
||||
self.finished_deferred = None
|
||||
self.points_paid = 0.0
|
||||
self.blob_requester = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stream_name)
|
||||
|
||||
def toggle_running(self):
|
||||
if self.stopped is True:
|
||||
return self.start()
|
||||
else:
|
||||
return self.stop()
|
||||
|
||||
def start(self):
|
||||
if self.starting is True:
|
||||
raise CurrentlyStartingError()
|
||||
if self.stopping is True:
|
||||
raise CurrentlyStoppingError()
|
||||
if self.stopped is False:
|
||||
raise AlreadyRunningError()
|
||||
assert self.download_manager is None
|
||||
self.starting = True
|
||||
self.completed = False
|
||||
self.finished_deferred = defer.Deferred()
|
||||
d = self._start()
|
||||
d.addCallback(lambda _: self.finished_deferred)
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None):
|
||||
if self.stopped is True:
|
||||
raise AlreadyStoppedError()
|
||||
if self.stopping is True:
|
||||
raise CurrentlyStoppingError()
|
||||
assert self.download_manager is not None
|
||||
self.stopping = True
|
||||
success = yield self.download_manager.stop_downloading()
|
||||
self.stopping = False
|
||||
if success is True:
|
||||
self.stopped = True
|
||||
self._remove_download_manager()
|
||||
yield self._fire_completed_deferred(err)
|
||||
|
||||
def _start_failed(self):
|
||||
|
||||
def set_stopped():
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
self.starting = False
|
||||
|
||||
if self.download_manager is not None:
|
||||
d = self.download_manager.stop_downloading()
|
||||
d.addCallback(lambda _: self._remove_download_manager())
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: set_stopped())
|
||||
d.addCallback(lambda _: Failure(StartFailedError()))
|
||||
return d
|
||||
|
||||
def _start(self):
|
||||
|
||||
def check_start_succeeded(success):
|
||||
if success:
|
||||
self.starting = False
|
||||
self.stopped = False
|
||||
self.completed = False
|
||||
return True
|
||||
else:
|
||||
return self._start_failed()
|
||||
|
||||
self.download_manager = self._get_download_manager()
|
||||
d = self.download_manager.start_downloading()
|
||||
d.addCallbacks(check_start_succeeded)
|
||||
return d
|
||||
|
||||
def _get_download_manager(self):
|
||||
assert self.blob_requester is None
|
||||
download_manager = DownloadManager(self.blob_manager)
|
||||
# TODO: can we get rid of these circular references. I'm not
|
||||
# smart enough to handle thinking about the interactions
|
||||
# between them and have hope that there is a simpler way
|
||||
# to accomplish what we want
|
||||
download_manager.blob_info_finder = self._get_metadata_handler(download_manager)
|
||||
download_manager.progress_manager = self._get_progress_manager(download_manager)
|
||||
download_manager.blob_handler = self._get_blob_handler(download_manager)
|
||||
download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
|
||||
# blob_requester needs to be set before the connection manager is setup
|
||||
self.blob_requester = self._get_blob_requester(download_manager)
|
||||
download_manager.connection_manager = self._get_connection_manager(download_manager)
|
||||
return download_manager
|
||||
|
||||
def _remove_download_manager(self):
|
||||
self.download_manager.blob_info_finder = None
|
||||
self.download_manager.progress_manager = None
|
||||
self.download_manager.blob_handler = None
|
||||
self.download_manager.wallet_info_exchanger = None
|
||||
self.blob_requester = None
|
||||
self.download_manager.connection_manager = None
|
||||
self.download_manager = None
|
||||
|
||||
def _get_primary_request_creators(self, download_manager):
|
||||
return [self.blob_requester]
|
||||
|
||||
def _get_secondary_request_creators(self, download_manager):
|
||||
return [download_manager.wallet_info_exchanger]
|
||||
|
||||
def _get_metadata_handler(self, download_manager):
|
||||
pass
|
||||
|
||||
def _get_blob_requester(self, download_manager):
|
||||
return BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
download_manager)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _get_write_func(self):
|
||||
pass
|
||||
|
||||
def _get_blob_handler(self, download_manager):
|
||||
return CryptBlobHandler(self.key, self._get_write_func())
|
||||
|
||||
def _get_connection_manager(self, download_manager):
|
||||
return ConnectionManager(self, self.rate_limiter,
|
||||
self._get_primary_request_creators(download_manager),
|
||||
self._get_secondary_request_creators(download_manager))
|
||||
|
||||
def _fire_completed_deferred(self, err=None):
|
||||
self.finished_deferred, d = None, self.finished_deferred
|
||||
if d is not None:
|
||||
if err is not None:
|
||||
d.errback(err)
|
||||
else:
|
||||
value = self._get_finished_deferred_callback_value()
|
||||
d.callback(value)
|
||||
else:
|
||||
log.debug("Not firing the completed deferred because d is None")
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
return None
|
||||
|
||||
def _finished_downloading(self, finished):
|
||||
if finished is True:
|
||||
self.completed = True
|
||||
return self.stop()
|
||||
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop(err=err)
|
||||
@@ -1,199 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.Error import FileOpenError
|
||||
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
storage, payment_rate_manager, wallet, key, stream_name, file_name):
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
self.file_name = os.path.basename(unhexlify(file_name).decode())
|
||||
self._calculated_total_bytes = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_data(self):
|
||||
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
|
||||
blob_hashes.append(sd_hash)
|
||||
yield self.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
def stop(self, err=None):
|
||||
self._close_output()
|
||||
return super().stop(err=err)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _start(self):
|
||||
def check_start_succeeded(success):
|
||||
if success:
|
||||
self.starting = False
|
||||
self.stopped = False
|
||||
self.completed = False
|
||||
return True
|
||||
else:
|
||||
return self._start_failed()
|
||||
|
||||
self.download_manager = self._get_download_manager()
|
||||
d = self._setup_output()
|
||||
d.addCallback(lambda _: self.download_manager.start_downloading())
|
||||
d.addCallbacks(check_start_succeeded)
|
||||
return d
|
||||
|
||||
def _setup_output(self):
|
||||
pass
|
||||
|
||||
def _close_output(self):
|
||||
pass
|
||||
|
||||
async def get_total_bytes(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
if self.download_manager is None:
|
||||
return 0
|
||||
else:
|
||||
self._calculated_total_bytes = self.download_manager.calculate_total_bytes()
|
||||
return self._calculated_total_bytes
|
||||
|
||||
def get_bytes_left_to_output(self):
|
||||
if self.download_manager is not None:
|
||||
return self.download_manager.calculate_bytes_left_to_output()
|
||||
else:
|
||||
return 0
|
||||
|
||||
def get_bytes_left_to_download(self):
|
||||
if self.download_manager is not None:
|
||||
return self.download_manager.calculate_bytes_left_to_download()
|
||||
else:
|
||||
return 0
|
||||
|
||||
def _get_metadata_handler(self, download_manager):
|
||||
return EncryptedFileMetadataHandler(self.stream_hash,
|
||||
self.storage, download_manager)
|
||||
|
||||
|
||||
class EncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.storage = storage
|
||||
self.wallet = wallet
|
||||
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
|
||||
assert len(options) == 1
|
||||
data_rate = options[0]
|
||||
payment_rate_manager.min_blob_data_payment_rate = data_rate
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
return defer.succeed(metadata.source_blob_hash)
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager,
|
||||
metadata.validator.raw_info)
|
||||
return defer.succeed(downloader)
|
||||
|
||||
d = save_sd_info(self.blob_manager, metadata.source_blob_hash, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
pass
|
||||
|
||||
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
|
||||
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = unhexlify(download_directory).decode()
|
||||
self.file_written_to = os.path.join(self.download_directory, unhexlify(file_name).decode())
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.file_written_to)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
download_manager)
|
||||
|
||||
def _setup_output(self):
|
||||
def open_file():
|
||||
if self.file_handle is None:
|
||||
file_written_to = os.path.join(self.download_directory, self.file_name)
|
||||
try:
|
||||
self.file_handle = open(file_written_to, 'wb')
|
||||
self.file_written_to = file_written_to
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise FileOpenError(
|
||||
"Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." % file_written_to
|
||||
)
|
||||
return threads.deferToThread(open_file)
|
||||
|
||||
def _close_output(self):
|
||||
self.file_handle, file_handle = None, self.file_handle
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
def _get_write_func(self):
|
||||
def write_func(data):
|
||||
if self.stopped is False and self.file_handle is not None:
|
||||
self.file_handle.write(data)
|
||||
return write_func
|
||||
|
||||
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = hexlify(download_directory.encode())
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
stream_name = stream_info.raw_info['stream_name']
|
||||
key = stream_info.raw_info['key']
|
||||
suggested_file_name = stream_info.raw_info['suggested_file_name']
|
||||
return EncryptedFileSaver(
|
||||
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
|
||||
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
|
||||
file_name=suggested_file_name
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save"
|
||||
@@ -1,40 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileMetadataHandler:
|
||||
|
||||
def __init__(self, stream_hash, storage, download_manager):
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
self.download_manager = download_manager
|
||||
self._final_blob_num = None
|
||||
|
||||
######### IMetadataHandler #########
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_initial_blobs(self):
|
||||
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
return self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
|
||||
def final_blob_num(self):
|
||||
return self._final_blob_num
|
||||
|
||||
######### internal calls #########
|
||||
|
||||
def _format_initial_blobs_for_download_manager(self, blob_infos):
|
||||
infos = []
|
||||
for i, crypt_blob in enumerate(blob_infos):
|
||||
if crypt_blob.blob_hash is not None and crypt_blob.length:
|
||||
infos.append(crypt_blob)
|
||||
else:
|
||||
if i != len(blob_infos) - 1:
|
||||
raise Exception("Invalid stream terminator: %i of %i" %
|
||||
(i, len(blob_infos) - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(crypt_blob.blob_num - 1))
|
||||
self._final_blob_num = crypt_blob.blob_num - 1
|
||||
return infos
|
||||
@@ -1,47 +0,0 @@
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.p2p.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
sd_identifier.add_stream_type(EncryptedFileStreamType, EncryptedFileStreamDescriptorValidator,
|
||||
EncryptedFileOptions())
|
||||
|
||||
|
||||
class EncryptedFileOptions:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.base.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.base.min_blob_data_payment_rate
|
||||
|
||||
rate_choices = []
|
||||
rate_choices.append(DownloadOptionChoice(
|
||||
prm.base.min_blob_data_payment_rate,
|
||||
"No change - %s" % get_default_data_rate_description(),
|
||||
"No change - %s" % get_default_data_rate_description()))
|
||||
if prm.base.min_blob_data_payment_rate is not None:
|
||||
rate_choices.append(DownloadOptionChoice(
|
||||
None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)))
|
||||
rate_choices.append(DownloadOptionChoice(float,
|
||||
"Enter rate in LBC/MB",
|
||||
"Enter rate in LBC/MB"))
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
rate_choices,
|
||||
"Rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.base.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
]
|
||||
return options
|
||||
@@ -1,51 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileCreator:
|
||||
"""
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we do not know the blob hash beforehand (i.e, when creating
|
||||
a new stream)
|
||||
"""
|
||||
def __init__(self, blob_dir):
|
||||
self.blob_dir = blob_dir
|
||||
self.buffer = BytesIO()
|
||||
self._is_open = True
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.blob_hash = None
|
||||
self.length = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def close(self):
|
||||
self.length = self.len_so_far
|
||||
self.blob_hash = self._hashsum.hexdigest()
|
||||
if self.blob_hash and self._is_open and self.length > 0:
|
||||
# do not save 0 length files (empty tail blob in streams)
|
||||
# or if its been closed already
|
||||
self.buffer.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(self.buffer)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self._is_open = False
|
||||
if self.length > 0:
|
||||
defer.returnValue(self.blob_hash)
|
||||
else:
|
||||
# 0 length files (empty tail blob in streams )
|
||||
# must return None as their blob_hash for
|
||||
# it to be saved properly by EncryptedFileMetadataManagers
|
||||
defer.returnValue(None)
|
||||
|
||||
def write(self, data):
|
||||
if not self._is_open:
|
||||
raise IOError
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
self.buffer.write(data)
|
||||
@@ -1,30 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader:
|
||||
"""
|
||||
This is a file like reader class that supports
|
||||
read(size) and close()
|
||||
"""
|
||||
def __init__(self, read_handle, finished_cb):
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
self.read_handle = read_handle
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warning("Garbage collection was called, but reader for %s was not closed yet",
|
||||
self.read_handle.name)
|
||||
self.close()
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.read_handle.read(size)
|
||||
|
||||
def close(self):
|
||||
# if we've already closed and called finished_cb, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
self.read_handle.close()
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
+45
-35
@@ -1,58 +1,68 @@
|
||||
import typing
|
||||
import logging
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobWriter:
|
||||
def __init__(self, length_getter, finished_cb):
|
||||
self.write_handle = BytesIO()
|
||||
self.length_getter = length_getter
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
|
||||
finished: asyncio.Future):
|
||||
self.expected_blob_hash = expected_blob_hash
|
||||
self.get_length = get_length
|
||||
self.buffer = BytesIO()
|
||||
self.finished = finished
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
if self.buffer is not None:
|
||||
log.warning("Garbage collection was called, but writer was not closed yet")
|
||||
self.close()
|
||||
self.close_handle()
|
||||
|
||||
@property
|
||||
def blob_hash(self):
|
||||
def calculate_blob_hash(self) -> str:
|
||||
return self._hashsum.hexdigest()
|
||||
|
||||
def write(self, data):
|
||||
if self.write_handle is None:
|
||||
def closed(self):
|
||||
return self.buffer is None or self.buffer.closed
|
||||
|
||||
def write(self, data: bytes):
|
||||
expected_length = self.get_length()
|
||||
if not expected_length:
|
||||
raise IOError("unknown blob length")
|
||||
if self.buffer is None:
|
||||
log.warning("writer has already been closed")
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
return
|
||||
raise IOError('I/O operation on closed file')
|
||||
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(
|
||||
self,
|
||||
Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (self.len_so_far,
|
||||
self.length_getter()))))
|
||||
else:
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
if self.len_so_far > expected_length:
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
self.close_handle()
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
self.close_handle()
|
||||
|
||||
def close_handle(self):
|
||||
if self.write_handle is not None:
|
||||
self.write_handle.close()
|
||||
self.write_handle = None
|
||||
|
||||
def close(self, reason=None):
|
||||
# if we've already called finished_cb because we either finished writing
|
||||
# or closed already, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
if reason is None:
|
||||
reason = Failure(DownloadCanceledError())
|
||||
self.finished_cb_d = self.finished_cb(self, reason)
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
File renamed without changes.
@@ -0,0 +1,224 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
from lbrynet.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: typing.Optional[asyncio.Future] = None
|
||||
self.buf = b''
|
||||
|
||||
# this is here to handle the race when the downloader is closed right as response_fut gets a result
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if not self._response_fut:
|
||||
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
|
||||
return self.close()
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
response = BlobResponse.deserialize(self.buf + data)
|
||||
if not response.responses and not self._response_fut.done():
|
||||
self.buf += data
|
||||
return
|
||||
else:
|
||||
self.buf = b''
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
|
||||
def _write(self, data: bytes):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
else:
|
||||
data = data
|
||||
self._blob_bytes_received += len(data)
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except (asyncio.TimeoutError) as err: # TODO: is this needed?
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if self.closed.is_set():
|
||||
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
|
||||
log.warning(msg)
|
||||
raise asyncio.CancelledError(msg)
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return self._blob_bytes_received, self.close()
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
log.info(msg)
|
||||
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
|
||||
return self._blob_bytes_received, self.transport
|
||||
except asyncio.TimeoutError:
|
||||
return self._blob_bytes_received, self.close()
|
||||
except (InvalidBlobHashError, InvalidDataError):
|
||||
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
def close(self):
|
||||
self.closed.set()
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
self.closed.clear()
|
||||
blob_hash = blob.blob_hash
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self._blob_bytes_received = 0
|
||||
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
self.close()
|
||||
return self._blob_bytes_received, None
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
finally:
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
|
||||
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.connection_made(connected_transport)
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
try:
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return 0, None
|
||||
@@ -0,0 +1,128 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import cache_concurrent
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
|
||||
)
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = self.failures.get(peer, 0) + 1
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = 0
|
||||
self.connections[peer] = transport
|
||||
elapsed = self.loop.time() - start
|
||||
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
|
||||
|
||||
async def new_peer_or_finished(self):
|
||||
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
|
||||
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
|
||||
def cleanup_active(self):
|
||||
if not self.active_connections and not self.connections:
|
||||
self.clearbanned()
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
self.ignored = dict((
|
||||
(peer, when) for (peer, when) in self.ignored.items()
|
||||
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
|
||||
))
|
||||
|
||||
@cache_concurrent
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
|
||||
connection_id: int = 0) -> 'AbstractBlob':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
try:
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set()
|
||||
while not self.peer_queue.empty():
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
blob.close()
|
||||
|
||||
def close(self):
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
self.is_running.clear()
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
|
||||
blob_hash: str) -> 'AbstractBlob':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
downloader.close()
|
||||
@@ -0,0 +1,283 @@
|
||||
import typing
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobMessage:
|
||||
key = ''
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class BlobPriceRequest(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobPriceResponse(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
rate_accepted = 'RATE_ACCEPTED'
|
||||
rate_too_low = 'RATE_TOO_LOW'
|
||||
rate_unset = 'RATE_UNSET'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
|
||||
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
|
||||
raise ValueError(blob_data_payment_rate)
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityRequest(BlobMessage):
|
||||
key = 'requested_blobs'
|
||||
|
||||
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
|
||||
**kwargs) -> None:
|
||||
assert len(requested_blobs)
|
||||
self.requested_blobs = requested_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blobs,
|
||||
'lbrycrd_address': self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityResponse(BlobMessage):
|
||||
key = 'available_blobs'
|
||||
|
||||
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
|
||||
**kwargs) -> None:
|
||||
self.available_blobs = available_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
self.key: self.available_blobs
|
||||
}
|
||||
if self.lbrycrd_address:
|
||||
d['lbrycrd_address'] = self.lbrycrd_address
|
||||
return d
|
||||
|
||||
|
||||
class BlobDownloadRequest(BlobMessage):
|
||||
key = 'requested_blob'
|
||||
|
||||
def __init__(self, requested_blob: str, **kwargs) -> None:
|
||||
self.requested_blob = requested_blob
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blob
|
||||
}
|
||||
|
||||
|
||||
class BlobDownloadResponse(BlobMessage):
|
||||
key = 'incoming_blob'
|
||||
|
||||
def __init__(self, **response: typing.Dict) -> None:
|
||||
incoming_blob = response[self.key]
|
||||
self.error = None
|
||||
self.incoming_blob = None
|
||||
if 'error' in incoming_blob:
|
||||
self.error = incoming_blob['error']
|
||||
else:
|
||||
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
|
||||
self.length = None if not self.incoming_blob else self.incoming_blob['length']
|
||||
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key if not self.error else 'error': self.incoming_blob or self.error,
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressRequest(BlobMessage):
|
||||
key = 'lbrycrd_address'
|
||||
|
||||
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
|
||||
pass
|
||||
|
||||
|
||||
class BlobErrorResponse(BlobMessage):
|
||||
key = 'error'
|
||||
|
||||
def __init__(self, error: str, **kwargs) -> None:
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.error
|
||||
}
|
||||
|
||||
|
||||
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest]
|
||||
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse]
|
||||
|
||||
|
||||
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
|
||||
# scenarios:
|
||||
# <json>
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
return None, response_msg
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
except ValueError:
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
def __init__(self, requests: typing.List[blob_request_types]) -> None:
|
||||
self.requests = requests
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for request in self.requests:
|
||||
d.update(request.to_dict())
|
||||
return d
|
||||
|
||||
def _get_request(self, request_type: blob_request_types):
|
||||
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
|
||||
if request:
|
||||
return request[0]
|
||||
|
||||
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
|
||||
response = self._get_request(BlobAvailabilityRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
|
||||
response = self._get_request(BlobPriceRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
|
||||
response = self._get_request(BlobDownloadRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
|
||||
response = self._get_request(BlobPaymentAddressRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobRequest':
|
||||
request = json.loads(data)
|
||||
return cls([
|
||||
request_type(**request)
|
||||
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest)
|
||||
if request_type.key in request
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
|
||||
return cls(
|
||||
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
|
||||
)
|
||||
|
||||
|
||||
class BlobResponse:
|
||||
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
|
||||
self.responses = responses
|
||||
self.blob_data = blob_data
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for response in self.responses:
|
||||
d.update(response.to_dict())
|
||||
return d
|
||||
|
||||
def _get_response(self, response_type: blob_response_types):
|
||||
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
|
||||
if response:
|
||||
return response[0]
|
||||
|
||||
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
|
||||
error = self._get_response(BlobErrorResponse)
|
||||
if error:
|
||||
log.error(error)
|
||||
return error
|
||||
|
||||
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
|
||||
response = self._get_response(BlobAvailabilityResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
|
||||
response = self._get_response(BlobPriceResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
|
||||
response = self._get_response(BlobDownloadResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
|
||||
response = self._get_response(BlobPaymentAddressResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobResponse':
|
||||
response, extra = _parse_blob_response(data)
|
||||
requests = []
|
||||
if response:
|
||||
requests.extend([
|
||||
response_type(**response)
|
||||
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse)
|
||||
if response_type.key in response
|
||||
])
|
||||
return cls(requests, extra)
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
from json.decoder import JSONDecodeError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
|
||||
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.buf = b''
|
||||
self.transport = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
self.transport.write(BlobResponse(to_send).serialize())
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
|
||||
responses = []
|
||||
address_request = request.get_address_request()
|
||||
if address_request:
|
||||
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
|
||||
availability_request = request.get_availability_request()
|
||||
if availability_request:
|
||||
responses.append(BlobAvailabilityResponse(available_blobs=list(set((
|
||||
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
|
||||
availability_request.requested_blobs)
|
||||
)))))
|
||||
price_request = request.get_price_request()
|
||||
if price_request:
|
||||
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
|
||||
download_request = request.get_blob_request()
|
||||
|
||||
if download_request:
|
||||
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
||||
if blob.get_is_verified():
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
return
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
# self.transport.close()
|
||||
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if data:
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.error("failed to decode blob request from %s:%i (%i bytes): %s", peer_address, peer_port,
|
||||
len(data), '' if not data else binascii.hexlify(data).decode())
|
||||
if not request:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.warning("failed to decode blob request from %s:%i", peer_address, peer_port)
|
||||
self.transport.close()
|
||||
return
|
||||
self.loop.create_task(self.handle_request(request))
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
log.info("Blob server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
def stop_server(self):
|
||||
if self.server_task:
|
||||
self.server_task.cancel()
|
||||
self.server_task = None
|
||||
log.info("Stopped blob server")
|
||||
@@ -1,2 +1,3 @@
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
BUILD_COMMIT = "source installation"
|
||||
+223
-111
@@ -2,13 +2,13 @@ import os
|
||||
import re
|
||||
import sys
|
||||
import typing
|
||||
import json
|
||||
import logging
|
||||
import yaml
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
from lbrynet.p2p.Error import InvalidCurrencyError
|
||||
from lbrynet.error import InvalidCurrencyError
|
||||
from lbrynet.dht import constants
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,33 +16,11 @@ log = logging.getLogger(__name__)
|
||||
NOT_SET = type(str('NOT_SET'), (object,), {})
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
API_ADDRESS = 'lbryapi'
|
||||
APP_NAME = 'LBRY'
|
||||
BLOBFILES_DIR = 'blobfiles'
|
||||
CRYPTSD_FILE_EXTENSION = '.cryptsd'
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
|
||||
LOG_FILE_NAME = 'lbrynet.log'
|
||||
LOG_POST_URL = 'https://lbry.io/log-upload'
|
||||
MAX_BLOB_REQUEST_SIZE = 64 * KB
|
||||
MAX_HANDSHAKE_SIZE = 64 * KB
|
||||
MAX_REQUEST_SIZE = 64 * KB
|
||||
MAX_RESPONSE_INFO_SIZE = 64 * KB
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
)
|
||||
HEADERS_FILE_SHA256_CHECKSUM = (
|
||||
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
|
||||
)
|
||||
@@ -51,14 +29,24 @@ HEADERS_FILE_SHA256_CHECKSUM = (
|
||||
class Setting(typing.Generic[T]):
|
||||
|
||||
def __init__(self, doc: str, default: typing.Optional[T] = None,
|
||||
previous_names: typing.Optional[typing.List[str]] = None):
|
||||
previous_names: typing.Optional[typing.List[str]] = None,
|
||||
metavar: typing.Optional[str] = None):
|
||||
self.doc = doc
|
||||
self.default = default
|
||||
self.previous_names = previous_names or []
|
||||
self.metavar = metavar
|
||||
|
||||
def __set_name__(self, owner, name):
|
||||
self.name = name
|
||||
|
||||
@property
|
||||
def cli_name(self):
|
||||
return f"--{self.name.replace('_', '-')}"
|
||||
|
||||
@property
|
||||
def no_cli_name(self):
|
||||
return f"--no-{self.name.replace('_', '-')}"
|
||||
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
|
||||
if obj is None:
|
||||
return self
|
||||
@@ -86,6 +74,14 @@ class Setting(typing.Generic[T]):
|
||||
def serialize(self, value):
|
||||
return value
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
metavar=self.metavar,
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class String(Setting[str]):
|
||||
def validate(self, val):
|
||||
@@ -98,18 +94,39 @@ class Integer(Setting[int]):
|
||||
assert isinstance(val, int), \
|
||||
f"Setting '{self.name}' must be an integer."
|
||||
|
||||
def deserialize(self, value):
|
||||
return int(value)
|
||||
|
||||
|
||||
class Float(Setting[float]):
|
||||
def validate(self, val):
|
||||
assert isinstance(val, float), \
|
||||
f"Setting '{self.name}' must be a decimal."
|
||||
|
||||
def deserialize(self, value):
|
||||
return float(value)
|
||||
|
||||
|
||||
class Toggle(Setting[bool]):
|
||||
def validate(self, val):
|
||||
assert isinstance(val, bool), \
|
||||
f"Setting '{self.name}' must be a true/false value."
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
action="store_true",
|
||||
default=NOT_SET
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help=f"Opposite of --{self.cli_name}",
|
||||
dest=self.name,
|
||||
action="store_false",
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class Path(String):
|
||||
def __init__(self, doc: str, default: str = '', *args, **kwargs):
|
||||
@@ -125,16 +142,54 @@ class Path(String):
|
||||
class MaxKeyFee(Setting[dict]):
|
||||
|
||||
def validate(self, value):
|
||||
assert isinstance(value, dict), \
|
||||
f"Setting '{self.name}' must be of the format \"{'currency': 'USD', 'amount': 50.0}\"."
|
||||
assert set(value) == {'currency', 'amount'}, \
|
||||
f"Setting '{self.name}' must contain a 'currency' and an 'amount' field."
|
||||
currency = str(value["currency"]).upper()
|
||||
if value is not None:
|
||||
assert isinstance(value, dict) and set(value) == {'currency', 'amount'}, \
|
||||
f"Setting '{self.name}' must be a dict like \"{{'amount': 50.0, 'currency': 'USD'}}\"."
|
||||
if value["currency"] not in CURRENCIES:
|
||||
raise InvalidCurrencyError(value["currency"])
|
||||
|
||||
@staticmethod
|
||||
def _parse_list(l):
|
||||
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
|
||||
try:
|
||||
amount = float(l[0])
|
||||
except ValueError:
|
||||
raise AssertionError('First value in max key fee is a decimal: "AMOUNT CURRENCY"')
|
||||
currency = str(l[1]).upper()
|
||||
if currency not in CURRENCIES:
|
||||
raise InvalidCurrencyError(currency)
|
||||
return {'amount': amount, 'currency': currency}
|
||||
|
||||
serialize = staticmethod(json.dumps)
|
||||
deserialize = staticmethod(json.loads)
|
||||
def deserialize(self, value):
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
'currency': value['currency'],
|
||||
'amount': float(value['amount']),
|
||||
}
|
||||
if isinstance(value, str):
|
||||
value = value.split()
|
||||
if isinstance(value, list):
|
||||
return self._parse_list(value)
|
||||
raise AssertionError('Invalid max key fee.')
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
nargs=2,
|
||||
metavar=('AMOUNT', 'CURRENCY'),
|
||||
default=NOT_SET
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help=f"Disable maximum key fee check.",
|
||||
dest=self.name,
|
||||
const=None,
|
||||
action="store_const",
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class Servers(Setting[list]):
|
||||
@@ -170,6 +225,14 @@ class Servers(Setting[list]):
|
||||
return [f"{host}:{port}" for host, port in value]
|
||||
return value
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
nargs="*",
|
||||
help=self.doc,
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class Strings(Setting[list]):
|
||||
|
||||
@@ -197,14 +260,23 @@ class EnvironmentAccess:
|
||||
|
||||
class ArgumentAccess:
|
||||
|
||||
def __init__(self, args: dict):
|
||||
self.args = args
|
||||
def __init__(self, config: 'BaseConfig', args: dict):
|
||||
self.configuration = config
|
||||
self.args = {}
|
||||
if args:
|
||||
self.load(args)
|
||||
|
||||
def load(self, args):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = getattr(args, setting.name, NOT_SET)
|
||||
if value != NOT_SET:
|
||||
self.args[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return getattr(self.args, item, None) is not None
|
||||
return item in self.args
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return getattr(self.args, item)
|
||||
return self.args[item]
|
||||
|
||||
|
||||
class ConfigFileAccess:
|
||||
@@ -270,7 +342,7 @@ class ConfigFileAccess:
|
||||
|
||||
class BaseConfig:
|
||||
|
||||
config = Path("Path to configuration file.")
|
||||
config = Path("Path to configuration file.", metavar='FILE')
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.runtime = {} # set internally or by various API calls
|
||||
@@ -308,7 +380,8 @@ class BaseConfig:
|
||||
|
||||
@classmethod
|
||||
def get_settings(cls):
|
||||
for setting in cls.__dict__.values():
|
||||
for attr in dir(cls):
|
||||
setting = getattr(cls, attr)
|
||||
if isinstance(setting, Setting):
|
||||
yield setting
|
||||
|
||||
@@ -331,22 +404,12 @@ class BaseConfig:
|
||||
return conf
|
||||
|
||||
@classmethod
|
||||
def contribute_args(cls, parser: ArgumentParser):
|
||||
def contribute_to_argparse(cls, parser: ArgumentParser):
|
||||
for setting in cls.get_settings():
|
||||
if isinstance(setting, Toggle):
|
||||
parser.add_argument(
|
||||
f"--{setting.name.replace('_', '-')}",
|
||||
help=setting.doc,
|
||||
action="store_true"
|
||||
)
|
||||
else:
|
||||
parser.add_argument(
|
||||
f"--{setting.name.replace('_', '-')}",
|
||||
help=setting.doc
|
||||
)
|
||||
setting.contribute_to_argparse(parser)
|
||||
|
||||
def set_arguments(self, args):
|
||||
self.arguments = ArgumentAccess(args)
|
||||
self.arguments = ArgumentAccess(self, args)
|
||||
|
||||
def set_environment(self, environ=None):
|
||||
self.environment = EnvironmentAccess(environ or os.environ)
|
||||
@@ -370,7 +433,7 @@ class BaseConfig:
|
||||
|
||||
class CLIConfig(BaseConfig):
|
||||
|
||||
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279')
|
||||
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT')
|
||||
|
||||
@property
|
||||
def api_connection_url(self) -> str:
|
||||
@@ -386,71 +449,121 @@ class CLIConfig(BaseConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
|
||||
data_dir = Path("Directory path to store blobs.")
|
||||
download_dir = Path("Directory path to place assembled files downloaded from LBRY.")
|
||||
# directories
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
"Directory path to place assembled files downloaded from LBRY.",
|
||||
previous_names=['download_directory'], metavar='DIR'
|
||||
)
|
||||
wallet_dir = Path(
|
||||
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
|
||||
previous_names=['lbryum_wallet_dir']
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# routing table
|
||||
split_buckets_under_index = Integer(
|
||||
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
|
||||
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 1
|
||||
)
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
blob_lru_cache_size = Integer(
|
||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||
"replying to a range request. Set to 0 to disable.", 32
|
||||
)
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
)
|
||||
concurrent_blob_announcers = Integer(
|
||||
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
|
||||
previous_names=['reflect_uploads']
|
||||
)
|
||||
concurrent_reflector_uploads = Integer(
|
||||
"Maximum number of streams to upload to a reflector server at a time", 10
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.com', 50001), # US EAST
|
||||
('lbryumx2.lbry.com', 50001), # US EAST
|
||||
('lbryumx4.lbry.com', 50001) #EU
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
('lbrynet2.lbry.com', 4444), # US WEST
|
||||
('lbrynet3.lbry.com', 4444), # EU
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
# comment server uses a single string because requests are made to the /api resource
|
||||
comment_server = String("Server to store user-generated metadata.", "http://comments.lbry.com:2903/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
# claims set to expire within this many blocks will be
|
||||
# automatically renewed after startup (if set to 0, renews
|
||||
# will not be made automatically)
|
||||
auto_renew_claim_height_delta = Integer("", 0)
|
||||
cache_time = Integer("", 150)
|
||||
data_rate = Float("points/megabyte", .0001)
|
||||
delete_blobs_on_remove = Toggle("", True)
|
||||
dht_node_port = Integer("", 4444)
|
||||
download_timeout = Integer("", 180)
|
||||
download_mirrors = Servers("", [
|
||||
('blobs.lbry.io', 80)
|
||||
])
|
||||
is_generous_host = Toggle("", True)
|
||||
announce_head_blobs_only = Toggle("", True)
|
||||
concurrent_announcers = Integer("", 10)
|
||||
known_dht_nodes = Servers("", [
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
max_connections_per_stream = Integer("", 5)
|
||||
seek_head_blob_first = Toggle("", True)
|
||||
# TODO: writing json on the cmd line is a pain, come up with a nicer
|
||||
# parser for this data structure. maybe 'USD:25'
|
||||
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
|
||||
disable_max_key_fee = Toggle("", False)
|
||||
min_info_rate = Float("points/1000 infos", .02)
|
||||
min_valuable_hash_rate = Float("points/1000 infos", .05)
|
||||
min_valuable_info_rate = Float("points/1000 infos", .05)
|
||||
peer_port = Integer("", 3333)
|
||||
pointtrader_server = String("", 'http://127.0.0.1:2424')
|
||||
reflector_port = Integer("", 5566)
|
||||
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
|
||||
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
|
||||
reflect_uploads = Toggle("", True)
|
||||
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
|
||||
reflector_servers = Servers("", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
|
||||
sd_download_timeout = Integer("", 3)
|
||||
peer_search_timeout = Integer("", 60)
|
||||
use_upnp = Toggle("", True)
|
||||
use_keyring = Toggle("", False)
|
||||
blockchain_name = String("", 'lbrycrd_main')
|
||||
lbryum_servers = Servers("", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
# media server
|
||||
|
||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||
'localhost:5280', metavar='HOST:PORT')
|
||||
streaming_get = Toggle("Enable the /get endpoint for the streaming media server. "
|
||||
"Disable to prevent new streams from being added.", True)
|
||||
|
||||
@property
|
||||
def streaming_host(self):
|
||||
return self.streaming_server.split(':')[0]
|
||||
|
||||
@property
|
||||
def streaming_port(self):
|
||||
return int(self.streaming_server.split(':')[1])
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -506,11 +619,10 @@ def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.sub('\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except EnvironmentError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
|
||||
if not download_dir:
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import hashlib
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
|
||||
backend = default_backend()
|
||||
|
||||
|
||||
def get_lbry_hash_obj():
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
Francois Aucamp <faucamp@csir.co.za>
|
||||
|
||||
Thanks goes to the following people for providing patches/suggestions/tests:
|
||||
|
||||
Neil Kleynhans <ntkleynhans@csir.co.za>
|
||||
Haiyang Ma <haiyang.ma@maidsafe.net>
|
||||
Bryan McAlister <bmcalister@csir.co.za>
|
||||
@@ -1,165 +0,0 @@
|
||||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
|
||||
This version of the GNU Lesser General Public License incorporates
|
||||
the terms and conditions of version 3 of the GNU General Public
|
||||
License, supplemented by the additional permissions listed below.
|
||||
|
||||
0. Additional Definitions.
|
||||
|
||||
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||
General Public License.
|
||||
|
||||
"The Library" refers to a covered work governed by this License,
|
||||
other than an Application or a Combined Work as defined below.
|
||||
|
||||
An "Application" is any work that makes use of an interface provided
|
||||
by the Library, but which is not otherwise based on the Library.
|
||||
Defining a subclass of a class defined by the Library is deemed a mode
|
||||
of using an interface provided by the Library.
|
||||
|
||||
A "Combined Work" is a work produced by combining or linking an
|
||||
Application with the Library. The particular version of the Library
|
||||
with which the Combined Work was made is also called the "Linked
|
||||
Version".
|
||||
|
||||
The "Minimal Corresponding Source" for a Combined Work means the
|
||||
Corresponding Source for the Combined Work, excluding any source code
|
||||
for portions of the Combined Work that, considered in isolation, are
|
||||
based on the Application, and not on the Linked Version.
|
||||
|
||||
The "Corresponding Application Code" for a Combined Work means the
|
||||
object code and/or source code for the Application, including any data
|
||||
and utility programs needed for reproducing the Combined Work from the
|
||||
Application, but excluding the System Libraries of the Combined Work.
|
||||
|
||||
1. Exception to Section 3 of the GNU GPL.
|
||||
|
||||
You may convey a covered work under sections 3 and 4 of this License
|
||||
without being bound by section 3 of the GNU GPL.
|
||||
|
||||
2. Conveying Modified Versions.
|
||||
|
||||
If you modify a copy of the Library, and, in your modifications, a
|
||||
facility refers to a function or data to be supplied by an Application
|
||||
that uses the facility (other than as an argument passed when the
|
||||
facility is invoked), then you may convey a copy of the modified
|
||||
version:
|
||||
|
||||
a) under this License, provided that you make a good faith effort to
|
||||
ensure that, in the event an Application does not supply the
|
||||
function or data, the facility still operates, and performs
|
||||
whatever part of its purpose remains meaningful, or
|
||||
|
||||
b) under the GNU GPL, with none of the additional permissions of
|
||||
this License applicable to that copy.
|
||||
|
||||
3. Object Code Incorporating Material from Library Header Files.
|
||||
|
||||
The object code form of an Application may incorporate material from
|
||||
a header file that is part of the Library. You may convey such object
|
||||
code under terms of your choice, provided that, if the incorporated
|
||||
material is not limited to numerical parameters, data structure
|
||||
layouts and accessors, or small macros, inline functions and templates
|
||||
(ten or fewer lines in length), you do both of the following:
|
||||
|
||||
a) Give prominent notice with each copy of the object code that the
|
||||
Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
4. Combined Works.
|
||||
|
||||
You may convey a Combined Work under terms of your choice that,
|
||||
taken together, effectively do not restrict modification of the
|
||||
portions of the Library contained in the Combined Work and reverse
|
||||
engineering for debugging such modifications, if you also do each of
|
||||
the following:
|
||||
|
||||
a) Give prominent notice with each copy of the Combined Work that
|
||||
the Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
c) For a Combined Work that displays copyright notices during
|
||||
execution, include the copyright notice for the Library among
|
||||
these notices, as well as a reference directing the user to the
|
||||
copies of the GNU GPL and this license document.
|
||||
|
||||
d) Do one of the following:
|
||||
|
||||
0) Convey the Minimal Corresponding Source under the terms of this
|
||||
License, and the Corresponding Application Code in a form
|
||||
suitable for, and under terms that permit, the user to
|
||||
recombine or relink the Application with a modified version of
|
||||
the Linked Version to produce a modified Combined Work, in the
|
||||
manner specified by section 6 of the GNU GPL for conveying
|
||||
Corresponding Source.
|
||||
|
||||
1) Use a suitable shared library mechanism for linking with the
|
||||
Library. A suitable mechanism is one that (a) uses at run time
|
||||
a copy of the Library already present on the user's computer
|
||||
system, and (b) will operate properly with a modified version
|
||||
of the Library that is interface-compatible with the Linked
|
||||
Version.
|
||||
|
||||
e) Provide Installation Information, but only if you would otherwise
|
||||
be required to provide such information under section 6 of the
|
||||
GNU GPL, and only to the extent that such information is
|
||||
necessary to install and execute a modified version of the
|
||||
Combined Work produced by recombining or relinking the
|
||||
Application with a modified version of the Linked Version. (If
|
||||
you use option 4d0, the Installation Information must accompany
|
||||
the Minimal Corresponding Source and Corresponding Application
|
||||
Code. If you use option 4d1, you must provide the Installation
|
||||
Information in the manner specified by section 6 of the GNU GPL
|
||||
for conveying Corresponding Source.)
|
||||
|
||||
5. Combined Libraries.
|
||||
|
||||
You may place library facilities that are a work based on the
|
||||
Library side by side in a single library together with other library
|
||||
facilities that are not Applications and are not covered by this
|
||||
License, and convey such a combined library under terms of your
|
||||
choice, if you do both of the following:
|
||||
|
||||
a) Accompany the combined library with a copy of the same work based
|
||||
on the Library, uncombined with any other library facilities,
|
||||
conveyed under the terms of this License.
|
||||
|
||||
b) Give prominent notice with the combined library that part of it
|
||||
is a work based on the Library, and explaining where to find the
|
||||
accompanying uncombined form of the same work.
|
||||
|
||||
6. Revised Versions of the GNU Lesser General Public License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions
|
||||
of the GNU Lesser General Public License from time to time. Such new
|
||||
versions will be similar in spirit to the present version, but may
|
||||
differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Library as you received it specifies that a certain numbered version
|
||||
of the GNU Lesser General Public License "or any later version"
|
||||
applies to it, you have the option of following the terms and
|
||||
conditions either of that published version or of any later version
|
||||
published by the Free Software Foundation. If the Library as you
|
||||
received it does not specify a version number of the GNU Lesser
|
||||
General Public License, you may choose any version of the GNU Lesser
|
||||
General Public License ever published by the Free Software Foundation.
|
||||
|
||||
If the Library as you received it specifies that a proxy can decide
|
||||
whether future versions of the GNU Lesser General Public License shall
|
||||
apply, that proxy's public statement of acceptance of any version is
|
||||
permanent authorization for you to choose that version for the
|
||||
Library.
|
||||
@@ -0,0 +1,59 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
while batch_size:
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
await asyncio.sleep(60, loop=self.loop)
|
||||
if not self.node.protocol.routing_table.get_peers():
|
||||
log.warning("No peers in DHT, announce round skipped")
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue):
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
|
||||
], loop=self.loop)
|
||||
announced = list(filter(None, announced))
|
||||
if announced:
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
assert not self.announce_task or self.announce_task.done(), "already running"
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
def stop(self):
|
||||
if self.announce_task and not self.announce_task.done():
|
||||
self.announce_task.cancel()
|
||||
@@ -1,81 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
MIN_DELAY = 0.0
|
||||
MAX_DELAY = 0.01
|
||||
DELAY_INCREMENT = 0.0001
|
||||
QUEUE_SIZE_THRESHOLD = 100
|
||||
|
||||
|
||||
class CallLaterManager:
|
||||
def __init__(self, callLater):
|
||||
"""
|
||||
:param callLater: (IReactorTime.callLater)
|
||||
"""
|
||||
|
||||
self._callLater = callLater
|
||||
self._pendingCallLaters = []
|
||||
self._delay = MIN_DELAY
|
||||
|
||||
def get_min_delay(self):
|
||||
self._pendingCallLaters = [cl for cl in self._pendingCallLaters if cl.active()]
|
||||
queue_size = len(self._pendingCallLaters)
|
||||
if queue_size > QUEUE_SIZE_THRESHOLD:
|
||||
self._delay = min((self._delay + DELAY_INCREMENT), MAX_DELAY)
|
||||
else:
|
||||
self._delay = max((self._delay - 2.0 * DELAY_INCREMENT), MIN_DELAY)
|
||||
return self._delay
|
||||
|
||||
def _cancel(self, call_later):
|
||||
"""
|
||||
:param call_later: DelayedCall
|
||||
:return: (callable) canceller function
|
||||
"""
|
||||
|
||||
def cancel(reason=None):
|
||||
"""
|
||||
:param reason: reason for cancellation, this is returned after cancelling the DelayedCall
|
||||
:return: reason
|
||||
"""
|
||||
|
||||
if call_later.active():
|
||||
call_later.cancel()
|
||||
if call_later in self._pendingCallLaters:
|
||||
self._pendingCallLaters.remove(call_later)
|
||||
return reason
|
||||
return cancel
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Cancel any callLaters that are still running
|
||||
"""
|
||||
|
||||
from twisted.internet import defer
|
||||
while self._pendingCallLaters:
|
||||
canceller = self._cancel(self._pendingCallLaters[0])
|
||||
try:
|
||||
canceller()
|
||||
except (defer.CancelledError, defer.AlreadyCalledError, ValueError):
|
||||
pass
|
||||
|
||||
def call_later(self, when, what, *args, **kwargs):
|
||||
"""
|
||||
Schedule a call later and get a canceller callback function
|
||||
|
||||
:param when: (float) delay in seconds
|
||||
:param what: (callable)
|
||||
:param args: (*tuple) args to be passed to the callable
|
||||
:param kwargs: (**dict) kwargs to be passed to the callable
|
||||
|
||||
:return: (tuple) twisted.internet.base.DelayedCall object, canceller function
|
||||
"""
|
||||
|
||||
call_later = self._callLater(when, what, *args, **kwargs)
|
||||
canceller = self._cancel(call_later)
|
||||
self._pendingCallLaters.append(call_later)
|
||||
return call_later, canceller
|
||||
|
||||
def call_soon(self, what, *args, **kwargs):
|
||||
delay = self.get_min_delay()
|
||||
return self.call_later(delay, what, *args, **kwargs)
|
||||
+38
-57
@@ -1,61 +1,42 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
""" This module defines the charaterizing constants of the Kademlia network
|
||||
|
||||
C{checkRefreshInterval} and C{udpDatagramMaxSize} are implementation-specific
|
||||
constants, and do not affect general Kademlia operation.
|
||||
"""
|
||||
|
||||
######### KADEMLIA CONSTANTS ###########
|
||||
|
||||
#: Small number Representing the degree of parallelism in network calls
|
||||
alpha = 3
|
||||
|
||||
#: Maximum number of contacts stored in a bucket; this should be an even number
|
||||
hash_class = hashlib.sha384
|
||||
hash_length = hash_class().digest_size
|
||||
hash_bits = hash_length * 8
|
||||
alpha = 5
|
||||
k = 8
|
||||
|
||||
#: Maximum number of contacts stored in the replacement cache
|
||||
replacementCacheSize = 8
|
||||
|
||||
#: Timeout for network operations (in seconds)
|
||||
rpcTimeout = 5
|
||||
|
||||
# number of rpc attempts to make before a timeout results in the node being removed as a contact
|
||||
rpcAttempts = 5
|
||||
# time window to count failures (in seconds)
|
||||
rpcAttemptsPruningTimeWindow = 600
|
||||
|
||||
# Delay between iterations of iterative node lookups (for loose parallelism) (in seconds)
|
||||
iterativeLookupDelay = rpcTimeout / 2
|
||||
|
||||
#: If a k-bucket has not been used for this amount of time, refresh it (in seconds)
|
||||
refreshTimeout = 3600 # 1 hour
|
||||
#: The interval at which nodes replicate (republish/refresh) data they are holding
|
||||
replicateInterval = refreshTimeout
|
||||
# The time it takes for data to expire in the network; the original publisher of the data
|
||||
# will also republish the data at this time if it is still valid
|
||||
dataExpireTimeout = 86400 # 24 hours
|
||||
|
||||
tokenSecretChangeInterval = 300 # 5 minutes
|
||||
|
||||
######## IMPLEMENTATION-SPECIFIC CONSTANTS ###########
|
||||
|
||||
#: The interval for the node to check whether any buckets need refreshing
|
||||
checkRefreshInterval = refreshTimeout / 5
|
||||
|
||||
#: Max size of a single UDP datagram, in bytes. If a message is larger than this, it will
|
||||
#: be spread across several UDP packets.
|
||||
udpDatagramMaxSize = 8192 # 8 KB
|
||||
|
||||
key_bits = 384
|
||||
|
||||
split_buckets_under_index = 1
|
||||
replacement_cache_size = 8
|
||||
rpc_timeout = 5.0
|
||||
rpc_attempts = 5
|
||||
rpc_attempts_pruning_window = 600
|
||||
iterative_lookup_delay = rpc_timeout / 2.0 # TODO: use config val / 2 if rpc timeout is provided
|
||||
refresh_interval = 3600 # 1 hour
|
||||
replicate_interval = refresh_interval
|
||||
data_expiration = 86400 # 24 hours
|
||||
token_secret_refresh_interval = 300 # 5 minutes
|
||||
maybe_ping_delay = 300 # 5 minutes
|
||||
check_refresh_interval = refresh_interval / 5
|
||||
max_datagram_size = 8192 # 8 KB
|
||||
rpc_id_length = 20
|
||||
protocol_version = 1
|
||||
bottom_out_limit = 3
|
||||
msg_size_limit = max_datagram_size - 26
|
||||
|
||||
protocolVersion = 1
|
||||
|
||||
def digest(data: bytes) -> bytes:
|
||||
h = hash_class()
|
||||
h.update(data)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def generate_id(num=None) -> bytes:
|
||||
if num is not None:
|
||||
return digest(str(num).encode())
|
||||
else:
|
||||
return digest(os.urandom(32))
|
||||
|
||||
|
||||
def generate_rpc_id(num=None) -> bytes:
|
||||
return generate_id(num)[:rpc_id_length]
|
||||
@@ -1,189 +0,0 @@
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import reduce
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class _Contact:
|
||||
""" Encapsulation for remote contact
|
||||
|
||||
This class contains information on a single remote contact, and also
|
||||
provides a direct RPC API to the remote node which it represents
|
||||
"""
|
||||
|
||||
def __init__(self, contactManager, id, ipAddress, udpPort, networkProtocol, firstComm):
|
||||
if id is not None:
|
||||
if not len(id) == constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: {}".format(hexlify(id).decode()))
|
||||
if not 0 <= udpPort <= 65536:
|
||||
raise ValueError("invalid port")
|
||||
if not is_valid_ipv4(ipAddress):
|
||||
raise ValueError("invalid ip address")
|
||||
self._contactManager = contactManager
|
||||
self._id = id
|
||||
self.address = ipAddress
|
||||
self.port = udpPort
|
||||
self._networkProtocol = networkProtocol
|
||||
self.commTime = firstComm
|
||||
self.getTime = self._contactManager._get_time
|
||||
self.lastReplied = None
|
||||
self.lastRequested = None
|
||||
self.protocolVersion = 0
|
||||
self._token = (None, 0) # token, timestamp
|
||||
|
||||
def update_token(self, token):
|
||||
self._token = token, self.getTime() if token else 0
|
||||
|
||||
@property
|
||||
def token(self):
|
||||
# expire the token 1 minute early to be safe
|
||||
return self._token[0] if self._token[1] + 240 > self.getTime() else None
|
||||
|
||||
@property
|
||||
def lastInteracted(self):
|
||||
return max(self.lastRequested or 0, self.lastReplied or 0, self.lastFailed or 0)
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
def log_id(self, short=True):
|
||||
if not self.id:
|
||||
return "not initialized"
|
||||
id_hex = hexlify(self.id)
|
||||
return id_hex if not short else id_hex[:8]
|
||||
|
||||
@property
|
||||
def failedRPCs(self):
|
||||
return len(self.failures)
|
||||
|
||||
@property
|
||||
def lastFailed(self):
|
||||
return (self.failures or [None])[-1]
|
||||
|
||||
@property
|
||||
def failures(self):
|
||||
return self._contactManager._rpc_failures.get((self.address, self.port), [])
|
||||
|
||||
@property
|
||||
def contact_is_good(self):
|
||||
"""
|
||||
:return: False if contact is bad, None if contact is unknown, or True if contact is good
|
||||
"""
|
||||
failures = self.failures
|
||||
now = self.getTime()
|
||||
delay = constants.checkRefreshInterval
|
||||
|
||||
if failures:
|
||||
if self.lastReplied and len(failures) >= 2 and self.lastReplied < failures[-2]:
|
||||
return False
|
||||
elif self.lastReplied and len(failures) >= 2 and self.lastReplied > failures[-2]:
|
||||
pass # handled below
|
||||
elif len(failures) >= 2:
|
||||
return False
|
||||
|
||||
if self.lastReplied and self.lastReplied > now - delay:
|
||||
return True
|
||||
if self.lastReplied and self.lastRequested and self.lastRequested > now - delay:
|
||||
return True
|
||||
return None
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, _Contact):
|
||||
raise TypeError("invalid type to compare with Contact: %s" % str(type(other)))
|
||||
return (self.id, self.address, self.port) == (other.id, other.address, other.port)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.id, self.address, self.port))
|
||||
|
||||
def compact_ip(self):
|
||||
compact_ip = reduce(
|
||||
lambda buff, x: buff + bytearray([int(x)]), self.address.split('.'), bytearray())
|
||||
return compact_ip
|
||||
|
||||
def set_id(self, id):
|
||||
if not self._id:
|
||||
self._id = id
|
||||
|
||||
def update_last_replied(self):
|
||||
self.lastReplied = int(self.getTime())
|
||||
|
||||
def update_last_requested(self):
|
||||
self.lastRequested = int(self.getTime())
|
||||
|
||||
def update_last_failed(self):
|
||||
failures = self._contactManager._rpc_failures.get((self.address, self.port), [])
|
||||
failures.append(self.getTime())
|
||||
self._contactManager._rpc_failures[(self.address, self.port)] = failures
|
||||
|
||||
def update_protocol_version(self, version):
|
||||
self.protocolVersion = version
|
||||
|
||||
def __str__(self):
|
||||
return '<%s.%s object; IP address: %s, UDP port: %d>' % (
|
||||
self.__module__, self.__class__.__name__, self.address, self.port)
|
||||
|
||||
def __getattr__(self, name):
|
||||
""" This override allows the host node to call a method of the remote
|
||||
node (i.e. this contact) as if it was a local function.
|
||||
|
||||
For instance, if C{remoteNode} is a instance of C{Contact}, the
|
||||
following will result in C{remoteNode}'s C{test()} method to be
|
||||
called with argument C{123}::
|
||||
remoteNode.test(123)
|
||||
|
||||
Such a RPC method call will return a Deferred, which will callback
|
||||
when the contact responds with the result (or an error occurs).
|
||||
This happens via this contact's C{_networkProtocol} object (i.e. the
|
||||
host Node's C{_protocol} object).
|
||||
"""
|
||||
|
||||
if name not in ['ping', 'findValue', 'findNode', 'store']:
|
||||
raise AttributeError("unknown command: %s" % name)
|
||||
|
||||
def _sendRPC(*args, **kwargs):
|
||||
return self._networkProtocol.sendRPC(self, name.encode(), args)
|
||||
|
||||
return _sendRPC
|
||||
|
||||
|
||||
class ContactManager:
|
||||
def __init__(self, get_time=None):
|
||||
if not get_time:
|
||||
from twisted.internet import reactor
|
||||
get_time = reactor.seconds
|
||||
self._get_time = get_time
|
||||
self._contacts = {}
|
||||
self._rpc_failures = {}
|
||||
|
||||
def get_contact(self, id, address, port):
|
||||
for contact in self._contacts.values():
|
||||
if contact.id == id and contact.address == address and contact.port == port:
|
||||
return contact
|
||||
|
||||
def make_contact(self, id, ipAddress, udpPort, networkProtocol, firstComm=0):
|
||||
contact = self.get_contact(id, ipAddress, udpPort)
|
||||
if contact:
|
||||
return contact
|
||||
contact = _Contact(self, id, ipAddress, udpPort, networkProtocol, firstComm or self._get_time())
|
||||
self._contacts[(id, ipAddress, udpPort)] = contact
|
||||
return contact
|
||||
|
||||
def is_ignored(self, origin_tuple):
|
||||
failed_rpc_count = len(self._prune_failures(origin_tuple))
|
||||
return failed_rpc_count > constants.rpcAttempts
|
||||
|
||||
def _prune_failures(self, origin_tuple):
|
||||
# Prunes recorded failures to the last time window of attempts
|
||||
pruning_limit = self._get_time() - constants.rpcAttemptsPruningTimeWindow
|
||||
pruned = list(filter(lambda t: t >= pruning_limit, self._rpc_failures.get(origin_tuple, [])))
|
||||
self._rpc_failures[origin_tuple] = pruned
|
||||
return pruned
|
||||
@@ -1,66 +0,0 @@
|
||||
from collections import UserDict
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class DictDataStore(UserDict):
|
||||
""" A datastore using an in-memory Python dictionary """
|
||||
#implements(IDataStore)
|
||||
|
||||
def __init__(self, getTime=None):
|
||||
# Dictionary format:
|
||||
# { <key>: (<contact>, <value>, <lastPublished>, <originallyPublished> <originalPublisherID>) }
|
||||
super().__init__()
|
||||
if not getTime:
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self.completed_blobs = set()
|
||||
|
||||
def filter_bad_and_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
"""
|
||||
return filter(
|
||||
lambda peer:
|
||||
self._getTime() - peer[3] < constants.dataExpireTimeout and peer[0].contact_is_good is not False,
|
||||
self[key]
|
||||
)
|
||||
|
||||
def filter_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
return filter(lambda peer: self._getTime() - peer[3] < constants.dataExpireTimeout, self[key])
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
expired_keys = []
|
||||
for key in self.keys():
|
||||
unexpired_peers = list(self.filter_expired_peers(key))
|
||||
if not unexpired_peers:
|
||||
expired_keys.append(key)
|
||||
else:
|
||||
self[key] = unexpired_peers
|
||||
for key in expired_keys:
|
||||
del self[key]
|
||||
|
||||
def hasPeersForBlob(self, key):
|
||||
return bool(key in self and len(tuple(self.filter_bad_and_expired_peers(key))))
|
||||
|
||||
def addPeerToBlob(self, contact, key, compact_address, lastPublished, originallyPublished, originalPublisherID):
|
||||
if key in self:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self[key]):
|
||||
self[key].append(
|
||||
(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)
|
||||
)
|
||||
else:
|
||||
self[key] = [(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)]
|
||||
|
||||
def getPeersForBlob(self, key):
|
||||
return [] if key not in self else [val[1] for val in self.filter_bad_and_expired_peers(key)]
|
||||
|
||||
def getStoringContacts(self):
|
||||
contacts = set()
|
||||
for key in self:
|
||||
for values in self[key]:
|
||||
contacts.add(values[0])
|
||||
return list(contacts)
|
||||
+6
-26
@@ -1,43 +1,23 @@
|
||||
import binascii
|
||||
#import exceptions
|
||||
|
||||
# this is a dict of {"exceptions.<exception class name>": exception class} items used to raise
|
||||
# remote built-in exceptions locally
|
||||
BUILTIN_EXCEPTIONS = {
|
||||
# "exceptions.%s" % e: getattr(exceptions, e) for e in dir(exceptions) if not e.startswith("_")
|
||||
}
|
||||
class BaseKademliaException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DecodeError(Exception):
|
||||
class DecodeError(BaseKademliaException):
|
||||
"""
|
||||
Should be raised by an C{Encoding} implementation if decode operation
|
||||
fails
|
||||
"""
|
||||
|
||||
|
||||
class BucketFull(Exception):
|
||||
class BucketFull(BaseKademliaException):
|
||||
"""
|
||||
Raised when the bucket is full
|
||||
"""
|
||||
|
||||
|
||||
class UnknownRemoteException(Exception):
|
||||
class RemoteException(BaseKademliaException):
|
||||
pass
|
||||
|
||||
|
||||
class TimeoutError(Exception):
|
||||
""" Raised when a RPC times out """
|
||||
|
||||
def __init__(self, remote_contact_id):
|
||||
# remote_contact_id is a binary blob so we need to convert it
|
||||
# into something more readable
|
||||
if remote_contact_id:
|
||||
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
|
||||
else:
|
||||
msg = 'Timeout connecting to uninitialized node'
|
||||
super().__init__(msg)
|
||||
self.remote_contact_id = remote_contact_id
|
||||
|
||||
|
||||
class TransportNotConnected(Exception):
|
||||
class TransportNotConnected(BaseKademliaException):
|
||||
pass
|
||||
@@ -1,116 +0,0 @@
|
||||
from zope.interface import Interface
|
||||
|
||||
|
||||
class IDataStore(Interface):
|
||||
""" Interface for classes implementing physical storage (for data
|
||||
published via the "STORE" RPC) for the Kademlia DHT
|
||||
|
||||
@note: This provides an interface for a dict-like object
|
||||
"""
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
pass
|
||||
|
||||
def hasPeersForBlob(self, key):
|
||||
pass
|
||||
|
||||
def addPeerToBlob(self, key, value, lastPublished, originallyPublished, originalPublisherID):
|
||||
pass
|
||||
|
||||
def getPeersForBlob(self, key):
|
||||
pass
|
||||
|
||||
def removePeer(self, key):
|
||||
pass
|
||||
|
||||
|
||||
class IRoutingTable(Interface):
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
Classes inheriting from this should provide a suitable routing table for
|
||||
a parent Node object (i.e. the local entity in the Kademlia network)
|
||||
"""
|
||||
|
||||
def __init__(self, parentNodeID):
|
||||
"""
|
||||
@param parentNodeID: The n-bit node ID of the node to which this
|
||||
routing table belongs
|
||||
@type parentNodeID: str
|
||||
"""
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add the given contact to the correct k-bucket; if it already
|
||||
exists, its status will be updated
|
||||
|
||||
@param contact: The contact to add to this node's k-buckets
|
||||
@type contact: kademlia.contact.Contact
|
||||
"""
|
||||
|
||||
def findCloseNodes(self, key, count, _rpcNodeID=None):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
@param count: the amount of contacts to return
|
||||
@type count: int
|
||||
@param _rpcNodeID: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type _rpcNodeID: str
|
||||
|
||||
@return: A list of node contacts (C{kademlia.contact.Contact instances})
|
||||
closest to the specified key.
|
||||
This method will return C{k} (or C{count}, if specified)
|
||||
contacts if at all possible; it will only return fewer if the
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
|
||||
def getContact(self, contactID):
|
||||
""" Returns the (known) contact with the specified node ID
|
||||
|
||||
@raise ValueError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
|
||||
def getRefreshList(self, startIndex=0, force=False):
|
||||
""" Finds all k-buckets that need refreshing, starting at the
|
||||
k-bucket with the specified index, and returns IDs to be searched for
|
||||
in order to refresh those k-buckets
|
||||
|
||||
@param startIndex: The index of the bucket to start refreshing at;
|
||||
this bucket and those further away from it will
|
||||
be refreshed. For example, when joining the
|
||||
network, this node will set this to the index of
|
||||
the bucket after the one containing it's closest
|
||||
neighbour.
|
||||
@type startIndex: index
|
||||
@param force: If this is C{True}, all buckets (in the specified range)
|
||||
will be refreshed, regardless of the time they were last
|
||||
accessed.
|
||||
@type force: bool
|
||||
|
||||
@return: A list of node ID's that the parent node should search for
|
||||
in order to refresh the routing Table
|
||||
@rtype: list
|
||||
"""
|
||||
|
||||
def removeContact(self, contactID):
|
||||
""" Remove the contact with the specified node ID from the routing
|
||||
table
|
||||
|
||||
@param contactID: The node ID of the contact to remove
|
||||
@type contactID: str
|
||||
"""
|
||||
|
||||
def touchKBucket(self, key):
|
||||
""" Update the "last accessed" timestamp of the k-bucket which covers
|
||||
the range containing the specified key in the key/ID space
|
||||
|
||||
@param key: A key in the range of the target k-bucket
|
||||
@type key: str
|
||||
"""
|
||||
@@ -1,230 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_contact(contact_list, node_id, address, port):
|
||||
for contact in contact_list:
|
||||
if contact.id == node_id and contact.address == address and contact.port == port:
|
||||
return contact
|
||||
raise IndexError(node_id)
|
||||
|
||||
|
||||
def expand_peer(compact_peer_info):
|
||||
host = "{}.{}.{}.{}".format(*compact_peer_info[:4])
|
||||
port = int.from_bytes(compact_peer_info[4:6], 'big')
|
||||
peer_node_id = compact_peer_info[6:]
|
||||
return (peer_node_id, host, port)
|
||||
|
||||
|
||||
class _IterativeFind:
|
||||
# TODO: use polymorphism to search for a value or node
|
||||
# instead of using a find_value flag
|
||||
def __init__(self, node, shortlist, key, rpc, exclude=None):
|
||||
self.exclude = set(exclude or [])
|
||||
self.node = node
|
||||
self.finished_deferred = defer.Deferred()
|
||||
# all distance operations in this class only care about the distance
|
||||
# to self.key, so this makes it easier to calculate those
|
||||
self.distance = Distance(key)
|
||||
# The closest known and active node yet found
|
||||
self.closest_node = None if not shortlist else shortlist[0]
|
||||
self.prev_closest_node = None
|
||||
# Shortlist of contact objects (the k closest known contacts to the key from the routing table)
|
||||
self.shortlist = shortlist
|
||||
# The search key
|
||||
self.key = key
|
||||
# The rpc method name (findValue or findNode)
|
||||
self.rpc = rpc
|
||||
# List of active queries; len() indicates number of active probes
|
||||
self.active_probes = []
|
||||
# List of contact (address, port) tuples that have already been queried, includes contacts that didn't reply
|
||||
self.already_contacted = []
|
||||
# A list of found and known-to-be-active remote nodes (Contact objects)
|
||||
self.active_contacts = []
|
||||
# Ensure only one searchIteration call is running at a time
|
||||
self._search_iteration_semaphore = defer.DeferredSemaphore(1)
|
||||
self._iteration_count = 0
|
||||
self.find_value_result = {}
|
||||
self.pending_iteration_calls = []
|
||||
|
||||
@property
|
||||
def is_find_node_request(self):
|
||||
return self.rpc == "findNode"
|
||||
|
||||
@property
|
||||
def is_find_value_request(self):
|
||||
return self.rpc == "findValue"
|
||||
|
||||
def is_closer(self, contact):
|
||||
if not self.closest_node:
|
||||
return True
|
||||
return self.distance.is_closer(contact.id, self.closest_node.id)
|
||||
|
||||
def getContactTriples(self, result):
|
||||
if self.is_find_value_request:
|
||||
contact_triples = result[b'contacts']
|
||||
else:
|
||||
contact_triples = result
|
||||
for contact_tup in contact_triples:
|
||||
if not isinstance(contact_tup, (list, tuple)) or len(contact_tup) != 3:
|
||||
raise ValueError("invalid contact triple")
|
||||
contact_tup[1] = contact_tup[1].decode() # ips are strings
|
||||
return contact_triples
|
||||
|
||||
def sortByDistance(self, contact_list):
|
||||
"""Sort the list of contacts in order by distance from key"""
|
||||
contact_list.sort(key=lambda c: self.distance(c.id))
|
||||
|
||||
def extendShortlist(self, contact, result):
|
||||
# The "raw response" tuple contains the response message and the originating address info
|
||||
originAddress = (contact.address, contact.port)
|
||||
if self.finished_deferred.called:
|
||||
return contact.id
|
||||
if self.node.contact_manager.is_ignored(originAddress):
|
||||
raise ValueError("contact is ignored")
|
||||
if contact.id == self.node.node_id:
|
||||
return contact.id
|
||||
|
||||
if contact not in self.active_contacts:
|
||||
self.active_contacts.append(contact)
|
||||
if contact not in self.shortlist:
|
||||
self.shortlist.append(contact)
|
||||
|
||||
# Now grow extend the (unverified) shortlist with the returned contacts
|
||||
# TODO: some validation on the result (for guarding against attacks)
|
||||
# If we are looking for a value, first see if this result is the value
|
||||
# we are looking for before treating it as a list of contact triples
|
||||
if self.is_find_value_request and self.key in result:
|
||||
# We have found the value
|
||||
for peer in result[self.key]:
|
||||
node_id, host, port = expand_peer(peer)
|
||||
if (host, port) not in self.exclude:
|
||||
self.find_value_result.setdefault(self.key, []).append((node_id, host, port))
|
||||
if self.find_value_result:
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
else:
|
||||
if self.is_find_value_request:
|
||||
# We are looking for a value, and the remote node didn't have it
|
||||
# - mark it as the closest "empty" node, if it is
|
||||
# TODO: store to this peer after finding the value as per the kademlia spec
|
||||
if b'closestNodeNoValue' in self.find_value_result:
|
||||
if self.is_closer(contact):
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
else:
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
contactTriples = self.getContactTriples(result)
|
||||
for contactTriple in contactTriples:
|
||||
if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
|
||||
continue
|
||||
elif self.node.contact_manager.is_ignored((contactTriple[1], contactTriple[2])):
|
||||
continue
|
||||
else:
|
||||
found_contact = self.node.contact_manager.make_contact(contactTriple[0], contactTriple[1],
|
||||
contactTriple[2], self.node._protocol)
|
||||
if found_contact not in self.shortlist:
|
||||
self.shortlist.append(found_contact)
|
||||
|
||||
if not self.finished_deferred.called and self.should_stop():
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
|
||||
return contact.id
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def probeContact(self, contact):
|
||||
fn = getattr(contact, self.rpc)
|
||||
try:
|
||||
response = yield fn(self.key)
|
||||
result = self.extendShortlist(contact, response)
|
||||
defer.returnValue(result)
|
||||
except (TimeoutError, defer.CancelledError, ValueError, IndexError):
|
||||
defer.returnValue(contact.id)
|
||||
|
||||
def should_stop(self):
|
||||
if self.is_find_value_request:
|
||||
# search stops when it finds a value, let it run
|
||||
return False
|
||||
if self.prev_closest_node and self.closest_node and self.distance.is_closer(self.prev_closest_node.id,
|
||||
self.closest_node.id):
|
||||
# we're getting further away
|
||||
return True
|
||||
if len(self.active_contacts) >= constants.k:
|
||||
# we have enough results
|
||||
return True
|
||||
return False
|
||||
|
||||
# Send parallel, asynchronous FIND_NODE RPCs to the shortlist of contacts
|
||||
def _searchIteration(self):
|
||||
# Sort the discovered active nodes from closest to furthest
|
||||
if len(self.active_contacts):
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.prev_closest_node = self.closest_node
|
||||
self.closest_node = self.active_contacts[0]
|
||||
|
||||
# Sort the current shortList before contacting other nodes
|
||||
self.sortByDistance(self.shortlist)
|
||||
probes = []
|
||||
already_contacted_addresses = {(c.address, c.port) for c in self.already_contacted}
|
||||
to_remove = []
|
||||
for contact in self.shortlist:
|
||||
if self.node.contact_manager.is_ignored((contact.address, contact.port)):
|
||||
to_remove.append(contact) # a contact became bad during iteration
|
||||
continue
|
||||
if (contact.address, contact.port) not in already_contacted_addresses:
|
||||
self.already_contacted.append(contact)
|
||||
to_remove.append(contact)
|
||||
probe = self.probeContact(contact)
|
||||
probes.append(probe)
|
||||
self.active_probes.append(probe)
|
||||
if len(probes) == constants.alpha:
|
||||
break
|
||||
for contact in to_remove: # these contacts will be re-added to the shortlist when they reply successfully
|
||||
self.shortlist.remove(contact)
|
||||
|
||||
# run the probes
|
||||
if probes:
|
||||
# Schedule the next iteration if there are any active
|
||||
# calls (Kademlia uses loose parallelism)
|
||||
self.searchIteration()
|
||||
|
||||
d = defer.DeferredList(probes, consumeErrors=True)
|
||||
|
||||
def _remove_probes(results):
|
||||
for probe in probes:
|
||||
self.active_probes.remove(probe)
|
||||
return results
|
||||
|
||||
d.addCallback(_remove_probes)
|
||||
|
||||
elif not self.finished_deferred.called and not self.active_probes or self.should_stop():
|
||||
# If no probes were sent, there will not be any improvement, so we're done
|
||||
if self.is_find_value_request:
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
else:
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
elif not self.finished_deferred.called:
|
||||
# Force the next iteration
|
||||
self.searchIteration()
|
||||
|
||||
def searchIteration(self, delay=constants.iterativeLookupDelay):
|
||||
def _cancel_pending_iterations(result):
|
||||
while self.pending_iteration_calls:
|
||||
canceller = self.pending_iteration_calls.pop()
|
||||
canceller()
|
||||
return result
|
||||
self.finished_deferred.addBoth(_cancel_pending_iterations)
|
||||
self._iteration_count += 1
|
||||
call, cancel = self.node.reactor_callLater(delay, self._search_iteration_semaphore.run, self._searchIteration)
|
||||
self.pending_iteration_calls.append(cancel)
|
||||
|
||||
|
||||
def iterativeFind(node, shortlist, key, rpc, exclude=None):
|
||||
helper = _IterativeFind(node, shortlist, key, rpc, exclude)
|
||||
helper.searchIteration(0)
|
||||
return helper.finished_deferred
|
||||
@@ -1,147 +0,0 @@
|
||||
import logging
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import BucketFull
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
def __init__(self, rangeMin, rangeMax, node_id):
|
||||
"""
|
||||
@param rangeMin: The lower boundary for the range in the n-bit ID
|
||||
space covered by this k-bucket
|
||||
@param rangeMax: The upper boundary for the range in the ID space
|
||||
covered by this k-bucket
|
||||
"""
|
||||
self.lastAccessed = 0
|
||||
self.rangeMin = rangeMin
|
||||
self.rangeMax = rangeMax
|
||||
self._contacts = list()
|
||||
self._node_id = node_id
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
contact to the end of the k-bucket if it is already present.
|
||||
|
||||
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
|
||||
the contact isn't in the bucket
|
||||
already
|
||||
|
||||
@param contact: The contact to add
|
||||
@type contact: dht.contact._Contact
|
||||
"""
|
||||
if contact in self._contacts:
|
||||
# Move the existing contact to the end of the list
|
||||
# - using the new contact to allow add-on data
|
||||
# (e.g. optimization-specific stuff) to pe updated as well
|
||||
self._contacts.remove(contact)
|
||||
self._contacts.append(contact)
|
||||
elif len(self._contacts) < constants.k:
|
||||
self._contacts.append(contact)
|
||||
else:
|
||||
raise BucketFull("No space in bucket to insert contact")
|
||||
|
||||
def getContact(self, contactID):
|
||||
"""Get the contact specified node ID
|
||||
|
||||
@raise IndexError: raised if the contact is not in the bucket
|
||||
|
||||
@param contactID: the node id of the contact to retrieve
|
||||
@type contactID: str
|
||||
|
||||
@rtype: dht.contact._Contact
|
||||
"""
|
||||
for contact in self._contacts:
|
||||
if contact.id == contactID:
|
||||
return contact
|
||||
raise IndexError(contactID)
|
||||
|
||||
def getContacts(self, count=-1, excludeContact=None, sort_distance_to=None):
|
||||
""" Returns a list containing up to the first count number of contacts
|
||||
|
||||
@param count: The amount of contacts to return (if 0 or less, return
|
||||
all contacts)
|
||||
@type count: int
|
||||
@param excludeContact: A node id to exclude; if this contact is in
|
||||
the list of returned values, it will be
|
||||
discarded before returning. If a C{str} is
|
||||
passed as this argument, it must be the
|
||||
contact's ID.
|
||||
@type excludeContact: str
|
||||
|
||||
@param sort_distance_to: Sort distance to the id, defaulting to the parent node id. If False don't
|
||||
sort the contacts
|
||||
|
||||
@raise IndexError: If the number of requested contacts is too large
|
||||
|
||||
@return: Return up to the first count number of contacts in a list
|
||||
If no contacts are present an empty is returned
|
||||
@rtype: list
|
||||
"""
|
||||
contacts = [contact for contact in self._contacts if contact.id != excludeContact]
|
||||
|
||||
# Return all contacts in bucket
|
||||
if count <= 0:
|
||||
count = len(contacts)
|
||||
|
||||
# Get current contact number
|
||||
currentLen = len(contacts)
|
||||
|
||||
# If count greater than k - return only k contacts
|
||||
if count > constants.k:
|
||||
count = constants.k
|
||||
|
||||
if not currentLen:
|
||||
return contacts
|
||||
|
||||
if sort_distance_to is False:
|
||||
pass
|
||||
else:
|
||||
sort_distance_to = sort_distance_to or self._node_id
|
||||
contacts.sort(key=lambda c: Distance(sort_distance_to)(c.id))
|
||||
|
||||
return contacts[:min(currentLen, count)]
|
||||
|
||||
def getBadOrUnknownContacts(self):
|
||||
contacts = self.getContacts(sort_distance_to=False)
|
||||
results = [contact for contact in contacts if contact.contact_is_good is False]
|
||||
results.extend(contact for contact in contacts if contact.contact_is_good is None)
|
||||
return results
|
||||
|
||||
def removeContact(self, contact):
|
||||
""" Remove the contact from the bucket
|
||||
|
||||
@param contact: The contact to remove
|
||||
@type contact: dht.contact._Contact
|
||||
|
||||
@raise ValueError: The specified contact is not in this bucket
|
||||
"""
|
||||
self._contacts.remove(contact)
|
||||
|
||||
def keyInRange(self, key):
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
of the n-bit ID space covered by this k-bucket (in otherwords, it
|
||||
returns whether or not the specified key should be placed in this
|
||||
k-bucket)
|
||||
|
||||
@param key: The key to test
|
||||
@type key: str or int
|
||||
|
||||
@return: C{True} if the key is in this k-bucket's range, or C{False}
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
if isinstance(key, bytes):
|
||||
key = int.from_bytes(key, 'big')
|
||||
return self.rangeMin <= key < self.rangeMax
|
||||
|
||||
def __len__(self):
|
||||
return len(self._contacts)
|
||||
|
||||
def __contains__(self, item):
|
||||
return item in self._contacts
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from lbrynet.dht import msgtypes
|
||||
|
||||
|
||||
class MessageTranslator:
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
Classes inheriting from this should provide a translation services between
|
||||
the classes used internally by this Kademlia implementation and the actual
|
||||
data that is transmitted between nodes.
|
||||
"""
|
||||
|
||||
def fromPrimitive(self, msgPrimitive):
|
||||
""" Create an RPC Message from a message's string representation
|
||||
|
||||
@param msgPrimitive: The unencoded primitive representation of a message
|
||||
@type msgPrimitive: str, int, list or dict
|
||||
|
||||
@return: The translated message object
|
||||
@rtype: entangled.kademlia.msgtypes.Message
|
||||
"""
|
||||
|
||||
def toPrimitive(self, message):
|
||||
""" Create a string representation of a message
|
||||
|
||||
@param message: The message object
|
||||
@type message: msgtypes.Message
|
||||
|
||||
@return: The message's primitive representation in a particular
|
||||
messaging format
|
||||
@rtype: str, int, list or dict
|
||||
"""
|
||||
|
||||
|
||||
class DefaultFormat(MessageTranslator):
|
||||
""" The default on-the-wire message format for this library """
|
||||
typeRequest, typeResponse, typeError = range(3)
|
||||
headerType, headerMsgID, headerNodeID, headerPayload, headerArgs = range(5) # TODO: make str
|
||||
|
||||
@staticmethod
|
||||
def get(primitive, key):
|
||||
try:
|
||||
return primitive[key]
|
||||
except KeyError:
|
||||
return primitive[str(key)] # TODO: switch to int()
|
||||
|
||||
def fromPrimitive(self, msgPrimitive):
|
||||
msgType = self.get(msgPrimitive, self.headerType)
|
||||
if msgType == self.typeRequest:
|
||||
msg = msgtypes.RequestMessage(self.get(msgPrimitive, self.headerNodeID),
|
||||
self.get(msgPrimitive, self.headerPayload),
|
||||
self.get(msgPrimitive, self.headerArgs),
|
||||
self.get(msgPrimitive, self.headerMsgID))
|
||||
elif msgType == self.typeResponse:
|
||||
msg = msgtypes.ResponseMessage(self.get(msgPrimitive, self.headerMsgID),
|
||||
self.get(msgPrimitive, self.headerNodeID),
|
||||
self.get(msgPrimitive, self.headerPayload))
|
||||
elif msgType == self.typeError:
|
||||
msg = msgtypes.ErrorMessage(self.get(msgPrimitive, self.headerMsgID),
|
||||
self.get(msgPrimitive, self.headerNodeID),
|
||||
self.get(msgPrimitive, self.headerPayload),
|
||||
self.get(msgPrimitive, self.headerArgs))
|
||||
else:
|
||||
# Unknown message, no payload
|
||||
msg = msgtypes.Message(msgPrimitive[self.headerMsgID], msgPrimitive[self.headerNodeID])
|
||||
return msg
|
||||
|
||||
def toPrimitive(self, message):
|
||||
msg = {self.headerMsgID: message.id,
|
||||
self.headerNodeID: message.nodeID}
|
||||
if isinstance(message, msgtypes.RequestMessage):
|
||||
msg[self.headerType] = self.typeRequest
|
||||
msg[self.headerPayload] = message.request
|
||||
msg[self.headerArgs] = message.args
|
||||
elif isinstance(message, msgtypes.ErrorMessage):
|
||||
msg[self.headerType] = self.typeError
|
||||
msg[self.headerPayload] = message.exceptionType
|
||||
msg[self.headerArgs] = message.response
|
||||
elif isinstance(message, msgtypes.ResponseMessage):
|
||||
msg[self.headerType] = self.typeResponse
|
||||
msg[self.headerPayload] = message.response
|
||||
return msg
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class Message:
|
||||
""" Base class for messages - all "unknown" messages use this class """
|
||||
|
||||
def __init__(self, rpcID, nodeID):
|
||||
if len(rpcID) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id: %i bytes (expected 20)" % len(rpcID))
|
||||
if len(nodeID) != constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: %i bytes (expected 48)" % len(nodeID))
|
||||
self.id = rpcID
|
||||
self.nodeID = nodeID
|
||||
|
||||
|
||||
class RequestMessage(Message):
|
||||
""" Message containing an RPC request """
|
||||
|
||||
def __init__(self, nodeID, method, methodArgs, rpcID=None):
|
||||
if rpcID is None:
|
||||
rpcID = generate_id()[:constants.rpc_id_length]
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.request = method
|
||||
self.args = methodArgs
|
||||
|
||||
|
||||
class ResponseMessage(Message):
|
||||
""" Message containing the result from a successful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, response):
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.response = response
|
||||
|
||||
|
||||
class ErrorMessage(ResponseMessage):
|
||||
""" Message containing the error from an unsuccessful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
|
||||
super().__init__(rpcID, nodeID, errorMessage)
|
||||
if isinstance(exceptionType, type):
|
||||
exceptionType = (f'{exceptionType.__module__}.{exceptionType.__name__}').encode()
|
||||
self.exceptionType = exceptionType
|
||||
+184
-615
@@ -1,649 +1,218 @@
|
||||
import binascii
|
||||
import hashlib
|
||||
import logging
|
||||
from functools import reduce
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
from lbrynet.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
|
||||
from twisted.internet import defer, error, task
|
||||
|
||||
from lbrynet.utils import generate_id, DeferredDict
|
||||
from lbrynet.dht.call_later_manager import CallLaterManager
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants, routingtable, datastore, protocol
|
||||
from lbrynet.dht.contact import ContactManager
|
||||
from lbrynet.dht.iterativefind import iterativeFind
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpcmethod(func):
|
||||
""" Decorator to expose Node methods as remote procedure calls
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
split_buckets_under_index)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
self._refresh_task: asyncio.Task = None
|
||||
|
||||
Apply this decorator to methods in the Node class (or a subclass) in order
|
||||
to make them remotely callable via the DHT's RPC mechanism.
|
||||
"""
|
||||
func.rpcmethod = True
|
||||
return func
|
||||
async def refresh_node(self, force_once=False):
|
||||
while True:
|
||||
# remove peers with expired blob announcements from the datastore
|
||||
self.protocol.data_store.removed_expired_peers()
|
||||
|
||||
total_peers: typing.List['KademliaPeer'] = []
|
||||
# add all peers in the routing table
|
||||
total_peers.extend(self.protocol.routing_table.get_peers())
|
||||
# add all the peers who have announed blobs to us
|
||||
total_peers.extend(self.protocol.data_store.get_storing_contacts())
|
||||
|
||||
class MockKademliaHelper:
|
||||
def __init__(self, clock=None, callLater=None, resolve=None, listenUDP=None):
|
||||
if not listenUDP or not resolve or not callLater or not clock:
|
||||
from twisted.internet import reactor
|
||||
listenUDP = listenUDP or reactor.listenUDP
|
||||
resolve = resolve or reactor.resolve
|
||||
callLater = callLater or reactor.callLater
|
||||
clock = clock or reactor
|
||||
# get ids falling in the midpoint of each bucket that hasn't been recently updated
|
||||
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
|
||||
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
|
||||
# populate/split the buckets further
|
||||
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
|
||||
if buckets_with_contacts <= 3:
|
||||
for i in range(buckets_with_contacts):
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
|
||||
self.clock = clock
|
||||
self.contact_manager = ContactManager(self.clock.seconds)
|
||||
self.reactor_listenUDP = listenUDP
|
||||
self.reactor_resolve = resolve
|
||||
self.call_later_manager = CallLaterManager(callLater)
|
||||
self.reactor_callLater = self.call_later_manager.call_later
|
||||
self.reactor_callSoon = self.call_later_manager.call_soon
|
||||
|
||||
self._listeningPort = None # object implementing Twisted
|
||||
# IListeningPort This will contain a deferred created when
|
||||
# joining the network, to enable publishing/retrieving
|
||||
# information from the DHT as soon as the node is part of the
|
||||
# network (add callbacks to this deferred if scheduling such
|
||||
# operations before the node has finished joining the network)
|
||||
|
||||
def get_looping_call(self, fn, *args, **kwargs):
|
||||
lc = task.LoopingCall(fn, *args, **kwargs)
|
||||
lc.clock = self.clock
|
||||
return lc
|
||||
|
||||
def safe_stop_looping_call(self, lc):
|
||||
if lc and lc.running:
|
||||
return lc.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
def safe_start_looping_call(self, lc, t):
|
||||
if lc and not lc.running:
|
||||
lc.start(t)
|
||||
|
||||
|
||||
class Node(MockKademliaHelper):
|
||||
""" Local node in the Kademlia network
|
||||
|
||||
This class represents a single local node in a Kademlia network; in other
|
||||
words, this class encapsulates an Entangled-using application's "presence"
|
||||
in a Kademlia network.
|
||||
|
||||
In Entangled, all interactions with the Kademlia network by a client
|
||||
application is performed via this class (or a subclass).
|
||||
"""
|
||||
|
||||
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
|
||||
routingTableClass=None, networkProtocol=None,
|
||||
externalIP=None, peerPort=3333, listenUDP=None,
|
||||
callLater=None, resolve=None, clock=None,
|
||||
interface='', externalUDPPort=None):
|
||||
"""
|
||||
@param dataStore: The data store to use. This must be class inheriting
|
||||
from the C{DataStore} interface (or providing the
|
||||
same API). How the data store manages its data
|
||||
internally is up to the implementation of that data
|
||||
store.
|
||||
@type dataStore: entangled.kademlia.datastore.DataStore
|
||||
@param routingTable: The routing table class to use. Since there exists
|
||||
some ambiguity as to how the routing table should be
|
||||
implemented in Kademlia, a different routing table
|
||||
may be used, as long as the appropriate API is
|
||||
exposed. This should be a class, not an object,
|
||||
in order to allow the Node to pass an
|
||||
auto-generated node ID to the routingtable object
|
||||
upon instantiation (if necessary).
|
||||
@type routingTable: entangled.kademlia.routingtable.RoutingTable
|
||||
@param networkProtocol: The network protocol to use. This can be
|
||||
overridden from the default to (for example)
|
||||
change the format of the physical RPC messages
|
||||
being transmitted.
|
||||
@type networkProtocol: entangled.kademlia.protocol.KademliaProtocol
|
||||
@param externalIP: the IP at which this node can be contacted
|
||||
@param peerPort: the port at which this node announces it has a blob for
|
||||
"""
|
||||
|
||||
super().__init__(clock, callLater, resolve, listenUDP)
|
||||
self.node_id = node_id or self._generateID()
|
||||
self.port = udpPort
|
||||
self._listen_interface = interface
|
||||
self._change_token_lc = self.get_looping_call(self.change_token)
|
||||
self._refresh_node_lc = self.get_looping_call(self._refreshNode)
|
||||
self._refresh_contacts_lc = self.get_looping_call(self._refreshContacts)
|
||||
|
||||
# Create k-buckets (for storing contacts)
|
||||
if routingTableClass is None:
|
||||
self._routingTable = routingtable.TreeRoutingTable(self.node_id, self.clock.seconds)
|
||||
else:
|
||||
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
|
||||
|
||||
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
|
||||
self.token_secret = self._generateID()
|
||||
self.old_token_secret = None
|
||||
self.externalIP = externalIP
|
||||
self.peerPort = peerPort
|
||||
self.externalUDPPort = externalUDPPort or self.port
|
||||
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
|
||||
self._join_deferred = None
|
||||
|
||||
#def __del__(self):
|
||||
# log.warning("unclean shutdown of the dht node")
|
||||
# if hasattr(self, "_listeningPort") and self._listeningPort is not None:
|
||||
# self._listeningPort.stopListening()
|
||||
|
||||
def __str__(self):
|
||||
return '<%s.%s object; ID: %s, IP address: %s, UDP port: %d>' % (
|
||||
self.__module__, self.__class__.__name__, binascii.hexlify(self.node_id), self.externalIP, self.port)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
# stop LoopingCalls:
|
||||
yield self.safe_stop_looping_call(self._refresh_node_lc)
|
||||
yield self.safe_stop_looping_call(self._change_token_lc)
|
||||
yield self.safe_stop_looping_call(self._refresh_contacts_lc)
|
||||
if self._listeningPort is not None:
|
||||
yield self._listeningPort.stopListening()
|
||||
self._listeningPort = None
|
||||
|
||||
def start_listening(self):
|
||||
if not self._listeningPort:
|
||||
try:
|
||||
self._listeningPort = self.reactor_listenUDP(self.port, self._protocol,
|
||||
interface=self._listen_interface)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running." % str(e))
|
||||
else:
|
||||
log.warning("Already bound to port %s", self._listeningPort)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def joinNetwork(self, known_node_addresses=(('jack.lbry.tech', 4455), )):
|
||||
"""
|
||||
Attempt to join the dht, retry every 30 seconds if unsuccessful
|
||||
:param known_node_addresses: [(str, int)] list of hostnames and ports for known dht seed nodes
|
||||
"""
|
||||
|
||||
self._join_deferred = defer.Deferred()
|
||||
known_node_resolution = {}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _resolve_seeds():
|
||||
result = {}
|
||||
for host, port in known_node_addresses:
|
||||
node_address = yield self.reactor_resolve(host)
|
||||
result[(host, port)] = node_address
|
||||
defer.returnValue(result)
|
||||
|
||||
if not known_node_resolution:
|
||||
known_node_resolution = yield _resolve_seeds()
|
||||
# we are one of the seed nodes, don't add ourselves
|
||||
if (self.externalIP, self.port) in known_node_resolution.values():
|
||||
del known_node_resolution[(self.externalIP, self.port)]
|
||||
known_node_addresses.remove((self.externalIP, self.port))
|
||||
|
||||
def _ping_contacts(contacts):
|
||||
d = DeferredDict({contact: contact.ping() for contact in contacts}, consumeErrors=True)
|
||||
d.addErrback(lambda err: err.trap(TimeoutError))
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _initialize_routing():
|
||||
bootstrap_contacts = []
|
||||
contact_addresses = {(c.address, c.port): c for c in self.contacts}
|
||||
for (host, port), ip_address in known_node_resolution.items():
|
||||
if (host, port) not in contact_addresses:
|
||||
# Create temporary contact information for the list of addresses of known nodes
|
||||
# The contact node id will be set with the responding node id when we initialize it to None
|
||||
contact = self.contact_manager.make_contact(None, ip_address, port, self._protocol)
|
||||
bootstrap_contacts.append(contact)
|
||||
else:
|
||||
for contact in self.contacts:
|
||||
if contact.address == ip_address and contact.port == port:
|
||||
if not contact.id:
|
||||
bootstrap_contacts.append(contact)
|
||||
break
|
||||
if not bootstrap_contacts:
|
||||
log.warning("no bootstrap contacts to ping")
|
||||
ping_result = yield _ping_contacts(bootstrap_contacts)
|
||||
shortlist = list(ping_result.keys())
|
||||
if not shortlist:
|
||||
log.warning("failed to ping %i bootstrap contacts", len(bootstrap_contacts))
|
||||
defer.returnValue(None)
|
||||
if self.protocol.routing_table.get_peers():
|
||||
# if we have node ids to look up, perform the iterative search until we have k results
|
||||
while node_ids:
|
||||
peers = await self.peer_search(node_ids.pop())
|
||||
total_peers.extend(peers)
|
||||
else:
|
||||
# find the closest peers to us
|
||||
closest = yield self._iterativeFind(self.node_id, shortlist if not self.contacts else None)
|
||||
yield _ping_contacts(closest)
|
||||
# query random hashes in our bucket key ranges to fill or split them
|
||||
random_ids_in_range = self._routingTable.getRefreshList()
|
||||
while random_ids_in_range:
|
||||
yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
defer.returnValue(None)
|
||||
if force_once:
|
||||
break
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
|
||||
await fut
|
||||
continue
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _iterative_join(joined_d=None, last_buckets_with_contacts=None):
|
||||
log.info("Attempting to join the DHT network, %i contacts known so far", len(self.contacts))
|
||||
joined_d = joined_d or defer.Deferred()
|
||||
yield _initialize_routing()
|
||||
buckets_with_contacts = self.bucketsWithContacts()
|
||||
if last_buckets_with_contacts and last_buckets_with_contacts == buckets_with_contacts:
|
||||
if not joined_d.called:
|
||||
joined_d.callback(True)
|
||||
elif buckets_with_contacts < 4:
|
||||
self.reactor_callLater(0, _iterative_join, joined_d, buckets_with_contacts)
|
||||
elif not joined_d.called:
|
||||
joined_d.callback(None)
|
||||
yield joined_d
|
||||
if not self._join_deferred.called:
|
||||
self._join_deferred.callback(True)
|
||||
defer.returnValue(None)
|
||||
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
if force_once:
|
||||
break
|
||||
|
||||
yield _iterative_join()
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
|
||||
await fut
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, known_node_addresses=None, block_on_join=False):
|
||||
""" Causes the Node to attempt to join the DHT network by contacting the
|
||||
known DHT nodes. This can be called multiple times if the previous attempt
|
||||
has failed or if the Node has lost all the contacts.
|
||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
assert len(hash_value) == constants.hash_length
|
||||
peers = await self.peer_search(hash_value)
|
||||
|
||||
@param known_node_addresses: A sequence of tuples containing IP address
|
||||
information for existing nodes on the
|
||||
Kademlia network, in the format:
|
||||
C{(<ip address>, (udp port>)}
|
||||
@type known_node_addresses: list
|
||||
"""
|
||||
|
||||
self.start_listening()
|
||||
yield self._protocol._listening
|
||||
# TODO: Refresh all k-buckets further away than this node's closest neighbour
|
||||
d = self.joinNetwork(known_node_addresses or [])
|
||||
d.addCallback(lambda _: self.start_looping_calls())
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
if block_on_join:
|
||||
yield d
|
||||
|
||||
def start_looping_calls(self):
|
||||
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)
|
||||
# Start refreshing k-buckets periodically, if necessary
|
||||
self.safe_start_looping_call(self._refresh_node_lc, constants.checkRefreshInterval)
|
||||
self.safe_start_looping_call(self._refresh_contacts_lc, 60)
|
||||
|
||||
@property
|
||||
def contacts(self):
|
||||
def _inner():
|
||||
for i in range(len(self._routingTable._buckets)):
|
||||
for contact in self._routingTable._buckets[i]._contacts:
|
||||
yield contact
|
||||
return list(_inner())
|
||||
|
||||
def hasContacts(self):
|
||||
for bucket in self._routingTable._buckets:
|
||||
if bucket._contacts:
|
||||
return True
|
||||
return False
|
||||
|
||||
def bucketsWithContacts(self):
|
||||
return self._routingTable.bucketsWithContacts()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def storeToContact(self, blob_hash, contact):
|
||||
try:
|
||||
if not contact.token:
|
||||
yield contact.findValue(blob_hash)
|
||||
res = yield contact.store(blob_hash, contact.token, self.peerPort, self.node_id, 0)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s (%s)", binascii.hexlify(blob_hash), contact.log_id(), contact.address)
|
||||
return True
|
||||
except protocol.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s",
|
||||
binascii.hexlify(blob_hash), contact.log_id())
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
contact.update_token(None)
|
||||
log.error("Unexpected error while storing blob_hash %s at %s: %s",
|
||||
binascii.hexlify(blob_hash), contact, err)
|
||||
return False
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def announceHaveBlob(self, blob_hash):
|
||||
contacts = yield self.iterativeFindNode(blob_hash)
|
||||
|
||||
if not self.externalIP:
|
||||
raise Exception("Cannot determine external IP: %s" % self.externalIP)
|
||||
stored_to = yield DeferredDict({contact: self.storeToContact(blob_hash, contact) for contact in contacts})
|
||||
contacted_node_ids = [binascii.hexlify(contact.id) for contact in stored_to.keys() if stored_to[contact]]
|
||||
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(blob_hash),
|
||||
len(contacted_node_ids), len(contacts))
|
||||
defer.returnValue(contacted_node_ids)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = self._generateID()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
return h.digest()
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
def iterativeFindNode(self, key):
|
||||
""" The basic Kademlia node lookup operation
|
||||
|
||||
Call this to find a remote node in the P2P overlay network.
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
|
||||
@return: This immediately returns a deferred object, which will return
|
||||
a list of k "closest" contacts (C{kademlia.contact.Contact}
|
||||
objects) to the specified key as soon as the operation is
|
||||
finished.
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
return self._iterativeFind(key)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def iterativeFindValue(self, key, exclude=None):
|
||||
""" The Kademlia search operation (deterministic)
|
||||
|
||||
Call this to retrieve data from the DHT.
|
||||
|
||||
@param key: the n-bit key (i.e. the value ID) to search for
|
||||
@type key: str
|
||||
|
||||
@return: This immediately returns a deferred object, which will return
|
||||
either one of two things:
|
||||
- If the value was found, it will return a Python
|
||||
dictionary containing the searched-for key (the C{key}
|
||||
parameter passed to this method), and its associated
|
||||
value, in the format:
|
||||
C{<str>key: <str>data_value}
|
||||
- If the value was not found, it will return a list of k
|
||||
"closest" contacts (C{kademlia.contact.Contact} objects)
|
||||
to the specified key
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length!")
|
||||
|
||||
# Execute the search
|
||||
find_result = yield self._iterativeFind(key, rpc='findValue', exclude=exclude)
|
||||
if isinstance(find_result, dict):
|
||||
# We have found the value; now see who was the closest contact without it...
|
||||
# ...and store the key/value pair
|
||||
pass
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
for peer in peers:
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
||||
if stored_to:
|
||||
log.info("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
len(stored_to), len(peers))
|
||||
else:
|
||||
# The value wasn't found, but a list of contacts was returned
|
||||
# Now, see if we have the value (it might seem wasteful to search on the network
|
||||
# first, but it ensures that all values are properly propagated through the
|
||||
# network
|
||||
if self._dataStore.hasPeersForBlob(key):
|
||||
# Ok, we have the value locally, so use that
|
||||
# Send this value to the closest node without it
|
||||
peers = self._dataStore.getPeersForBlob(key)
|
||||
find_result = {key: peers}
|
||||
else:
|
||||
pass
|
||||
log.warning("Failed announcing %s, stored to 0 peers", blob_hash[:8])
|
||||
return stored_to
|
||||
|
||||
defer.returnValue(list(set(find_result.get(key, []) if find_result else [])))
|
||||
# TODO: get this working
|
||||
# if 'closestNodeNoValue' in find_result:
|
||||
# closest_node_without_value = find_result['closestNodeNoValue']
|
||||
# try:
|
||||
# response, address = yield closest_node_without_value.findValue(key, rawResponse=True)
|
||||
# yield closest_node_without_value.store(key, response.response['token'], self.peerPort)
|
||||
# except TimeoutError:
|
||||
# pass
|
||||
def stop(self) -> None:
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
if self._join_task:
|
||||
self._join_task.cancel()
|
||||
if self._refresh_task and not (self._refresh_task.done() or self._refresh_task.cancelled()):
|
||||
self._refresh_task.cancel()
|
||||
if self.protocol and self.protocol.ping_queue.running:
|
||||
self.protocol.ping_queue.stop()
|
||||
self.protocol.stop()
|
||||
if self.listening_port is not None:
|
||||
self.listening_port.close()
|
||||
self._join_task = None
|
||||
self.listening_port = None
|
||||
log.info("Stopped DHT node")
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add/update the given contact; simple wrapper for the same method
|
||||
in this object's RoutingTable object
|
||||
|
||||
@param contact: The contact to add to this node's k-buckets
|
||||
@type contact: kademlia.contact.Contact
|
||||
"""
|
||||
return self._routingTable.addContact(contact)
|
||||
|
||||
def removeContact(self, contact):
|
||||
""" Remove the contact with the specified node ID from this node's
|
||||
table of known nodes. This is a simple wrapper for the same method
|
||||
in this object's RoutingTable object
|
||||
|
||||
@param contact: The Contact object to remove
|
||||
@type contact: _Contact
|
||||
"""
|
||||
self._routingTable.removeContact(contact)
|
||||
|
||||
def findContact(self, contactID):
|
||||
""" Find a entangled.kademlia.contact.Contact object for the specified
|
||||
cotact ID
|
||||
|
||||
@param contactID: The contact ID of the required Contact object
|
||||
@type contactID: str
|
||||
|
||||
@return: Contact object of remote node with the specified node ID,
|
||||
or None if the contact was not found
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
try:
|
||||
df = defer.succeed(self._routingTable.getContact(contactID))
|
||||
except (ValueError, IndexError):
|
||||
df = self.iterativeFindNode(contactID)
|
||||
df.addCallback(lambda nodes: ([node for node in nodes if node.id == contactID] or (None,))[0])
|
||||
return df
|
||||
|
||||
@rpcmethod
|
||||
def ping(self):
|
||||
""" Used to verify contact between two Kademlia nodes
|
||||
|
||||
@rtype: str
|
||||
"""
|
||||
return b'pong'
|
||||
|
||||
@rpcmethod
|
||||
def store(self, rpc_contact, blob_hash, token, port, originalPublisherID, age):
|
||||
""" Store the received data in this node's local datastore
|
||||
|
||||
@param blob_hash: The hash of the data
|
||||
@type blob_hash: str
|
||||
|
||||
@param token: The token we previously returned when this contact sent us a findValue
|
||||
@type token: str
|
||||
|
||||
@param port: The TCP port the contact is listening on for requests for this blob (the peerPort)
|
||||
@type port: int
|
||||
|
||||
@param originalPublisherID: The node ID of the node that is the publisher of the data
|
||||
@type originalPublisherID: str
|
||||
|
||||
@param age: The relative age of the data (time in seconds since it was
|
||||
originally published). Note that the original publish time
|
||||
isn't actually given, to compensate for clock skew between
|
||||
different nodes.
|
||||
@type age: int
|
||||
|
||||
@rtype: str
|
||||
"""
|
||||
|
||||
if originalPublisherID is None:
|
||||
originalPublisherID = rpc_contact.id
|
||||
compact_ip = rpc_contact.compact_ip()
|
||||
if self.clock.seconds() - self._protocol.started_listening_time < constants.tokenSecretChangeInterval:
|
||||
pass
|
||||
elif not self.verify_token(token, compact_ip):
|
||||
raise ValueError("Invalid token")
|
||||
if 0 <= port <= 65536:
|
||||
compact_port = port.to_bytes(2, 'big')
|
||||
async def start_listening(self, interface: str = '') -> None:
|
||||
if not self.listening_port:
|
||||
self.listening_port, _ = await self.loop.create_datagram_endpoint(
|
||||
lambda: self.protocol, (interface, self.internal_udp_port)
|
||||
)
|
||||
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
|
||||
self.protocol.start()
|
||||
else:
|
||||
raise TypeError(f'Invalid port: {port}')
|
||||
compact_address = compact_ip + compact_port + rpc_contact.id
|
||||
now = int(self.clock.seconds())
|
||||
originallyPublished = now - age
|
||||
self._dataStore.addPeerToBlob(rpc_contact, blob_hash, compact_address, now, originallyPublished,
|
||||
originalPublisherID)
|
||||
return b'OK'
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
@rpcmethod
|
||||
def findNode(self, rpc_contact, key):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
@return: A list of contact triples closest to the specified key.
|
||||
This method will return C{k} (or C{count}, if specified)
|
||||
contacts if at all possible; it will only return fewer if the
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid contact id length: %i" % len(key))
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses and\
|
||||
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
contacts = self._routingTable.findCloseNodes(key, sender_node_id=rpc_contact.id)
|
||||
contact_triples = []
|
||||
for contact in contacts:
|
||||
contact_triples.append((contact.id, contact.address, contact.port))
|
||||
return contact_triples
|
||||
if known_node_addresses:
|
||||
peers = [
|
||||
KademliaPeer(self.loop, address, udp_port=port)
|
||||
for (address, port) in known_node_addresses
|
||||
]
|
||||
while True:
|
||||
if not self.protocol.routing_table.get_peers():
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
self.protocol.peer_manager.reset()
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*peers, delay=0.0)
|
||||
peers.extend(await self.peer_search(self.protocol.node_id, shortlist=peers, count=32))
|
||||
if self.protocol.routing_table.get_peers():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
else:
|
||||
continue
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
@rpcmethod
|
||||
def findValue(self, rpc_contact, key):
|
||||
""" Return the value associated with the specified key if present in
|
||||
this node's data, otherwise execute FIND_NODE for the key
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
|
||||
@param key: The hashtable key of the data to return
|
||||
@type key: str
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
self.join_network(
|
||||
interface=interface, known_node_urls=known_node_urls
|
||||
)
|
||||
)
|
||||
|
||||
@return: A dictionary containing the requested key/value pair,
|
||||
or a list of contact triples closest to the requested key.
|
||||
@rtype: dict or list
|
||||
"""
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.bottom_out_limit,
|
||||
max_results: int = constants.k) -> IterativeNodeFinder:
|
||||
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid blob hash length: %i" % len(key))
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
}
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = 40,
|
||||
max_results: int = -1) -> IterativeValueFinder:
|
||||
|
||||
if self._protocol._protocolVersion:
|
||||
response[b'protocolVersion'] = self._protocol._protocolVersion
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
# get peers we have stored for this blob
|
||||
has_other_peers = self._dataStore.hasPeersForBlob(key)
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
) -> typing.List['KademliaPeer']:
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self._dataStore.getPeersForBlob(key))
|
||||
async for iteration_peers in self.get_iterative_node_finder(
|
||||
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
|
||||
peers.extend(iteration_peers)
|
||||
distance = Distance(node_id)
|
||||
peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
return peers[:count]
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and key in self._dataStore.completed_blobs:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), self.externalIP.split('.'), bytearray())
|
||||
compact_port = self.peerPort.to_bytes(2, 'big')
|
||||
compact_address = compact_ip + compact_port + self.node_id
|
||||
peers.append(compact_address)
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
tasks = []
|
||||
try:
|
||||
while True:
|
||||
blob_hash = await search_queue.get()
|
||||
tasks.append(self.loop.create_task(self._value_producer(blob_hash, result_queue)))
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response[b'contacts'] = self.findNode(rpc_contact, key)
|
||||
return response
|
||||
async def _value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
|
||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
||||
result_queue.put_nowait(results)
|
||||
|
||||
def _generateID(self):
|
||||
""" Generates an n-bit pseudo-random identifier
|
||||
|
||||
@return: A globally unique n-bit pseudo-random identifier
|
||||
@rtype: str
|
||||
"""
|
||||
return generate_id()
|
||||
|
||||
# from lbrynet.p2p.utils import profile_deferred
|
||||
# @profile_deferred()
|
||||
@defer.inlineCallbacks
|
||||
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode', exclude=None):
|
||||
""" The basic Kademlia iterative lookup operation (for nodes/values)
|
||||
|
||||
This builds a list of k "closest" contacts through iterative use of
|
||||
the "FIND_NODE" RPC, or if C{findValue} is set to C{True}, using the
|
||||
"FIND_VALUE" RPC, in which case the value (if found) may be returned
|
||||
instead of a list of contacts
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
@param startupShortlist: A list of contacts to use as the starting
|
||||
shortlist for this search; this is normally
|
||||
only used when the node joins the network
|
||||
@type startupShortlist: list
|
||||
@param rpc: The name of the RPC to issue to remote nodes during the
|
||||
Kademlia lookup operation (e.g. this sets whether this
|
||||
algorithm should search for a data value (if
|
||||
rpc='findValue') or not. It can thus be used to perform
|
||||
other operations that piggy-back on the basic Kademlia
|
||||
lookup operation (Entangled's "delete" RPC, for instance).
|
||||
@type rpc: str
|
||||
|
||||
@return: If C{findValue} is C{True}, the algorithm will stop as soon
|
||||
as a data value for C{key} is found, and return a dictionary
|
||||
containing the key and the found value. Otherwise, it will
|
||||
return a list of the k closest nodes to the specified key
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
|
||||
if startupShortlist is None:
|
||||
shortlist = self._routingTable.findCloseNodes(key)
|
||||
# if key != self.node_id:
|
||||
# # Update the "last accessed" timestamp for the appropriate k-bucket
|
||||
# self._routingTable.touchKBucket(key)
|
||||
if len(shortlist) == 0:
|
||||
log.warning("This node doesn't know any other nodes")
|
||||
# This node doesn't know of any other nodes
|
||||
fakeDf = defer.Deferred()
|
||||
fakeDf.callback([])
|
||||
result = yield fakeDf
|
||||
defer.returnValue(result)
|
||||
else:
|
||||
# This is used during the bootstrap process
|
||||
shortlist = startupShortlist
|
||||
|
||||
result = yield iterativeFind(self, shortlist, key, rpc, exclude=exclude)
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _refreshNode(self):
|
||||
""" Periodically called to perform k-bucket refreshes and data
|
||||
replication/republishing as necessary """
|
||||
yield self._refreshRoutingTable()
|
||||
self._dataStore.removeExpiredPeers()
|
||||
self._refreshStoringPeers()
|
||||
defer.returnValue(None)
|
||||
|
||||
def _refreshContacts(self):
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self.contacts, delay=0)
|
||||
|
||||
def _refreshStoringPeers(self):
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self._dataStore.getStoringContacts(), delay=0)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _refreshRoutingTable(self):
|
||||
nodeIDs = self._routingTable.getRefreshList(0, False)
|
||||
while nodeIDs:
|
||||
searchID = nodeIDs.pop()
|
||||
yield self.iterativeFindNode(searchID)
|
||||
defer.returnValue(None)
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue(loop=self.loop)
|
||||
return q, self.loop.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
@@ -0,0 +1,198 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import lru_cache
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class PeerManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop):
|
||||
self._loop = loop
|
||||
self._rpc_failures: typing.Dict[
|
||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||
] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
|
||||
def reset(self):
|
||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||
statistic.clear()
|
||||
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
def clear_token(self, node_id: bytes):
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (None, None))
|
||||
if ts and ts > self._loop.time() - constants.token_secret_refresh_interval:
|
||||
return token
|
||||
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
@lru_cache(maxsize=400)
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
:return: False if peer is bad, None if peer is unknown, or True if peer is good
|
||||
"""
|
||||
|
||||
delay = self._loop.time() - constants.check_refresh_interval
|
||||
|
||||
# fixme: find a way to re-enable that without breaking other parts
|
||||
#if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
# return
|
||||
#addr_tup = (address, udp_port)
|
||||
#if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
|
||||
# return
|
||||
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
|
||||
last_requested = self._last_requested.get((address, udp_port))
|
||||
last_replied = self._last_replied.get((address, udp_port))
|
||||
if most_recent_failure and last_replied:
|
||||
if delay < last_replied > most_recent_failure:
|
||||
return True
|
||||
elif last_replied > most_recent_failure:
|
||||
return
|
||||
return False
|
||||
elif previous_failure and most_recent_failure and most_recent_failure > delay:
|
||||
return False
|
||||
elif last_replied and last_replied > delay:
|
||||
return True
|
||||
elif last_requested and last_requested > delay:
|
||||
return None
|
||||
return
|
||||
|
||||
def peer_is_good(self, peer: 'KademliaPeer'):
|
||||
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
|
||||
|
||||
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':
|
||||
node_id, address, tcp_port = decode_compact_address(compact_address)
|
||||
return KademliaPeer(self._loop, address, node_id, tcp_port=tcp_port)
|
||||
|
||||
|
||||
class KademliaPeer:
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'_node_id',
|
||||
'address',
|
||||
'udp_port',
|
||||
'tcp_port',
|
||||
'protocol_version',
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
|
||||
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
|
||||
if node_id is not None:
|
||||
if not len(node_id) == constants.hash_length:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(node_id).decode()))
|
||||
if udp_port is not None and not 0 <= udp_port <= 65536:
|
||||
raise ValueError("invalid udp port")
|
||||
if tcp_port and not 0 <= tcp_port <= 65536:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_ipv4(address):
|
||||
raise ValueError("invalid ip address")
|
||||
self.loop = loop
|
||||
self._node_id = node_id
|
||||
self.address = address
|
||||
self.udp_port = udp_port
|
||||
self.tcp_port = tcp_port
|
||||
self.protocol_version = 1
|
||||
|
||||
def update_tcp_port(self, tcp_port: int):
|
||||
self.tcp_port = tcp_port
|
||||
|
||||
def update_udp_port(self, udp_port: int):
|
||||
self.udp_port = udp_port
|
||||
|
||||
def set_id(self, node_id):
|
||||
if not self._node_id:
|
||||
self._node_id = node_id
|
||||
|
||||
@property
|
||||
def node_id(self) -> bytes:
|
||||
return self._node_id
|
||||
|
||||
def compact_address_udp(self) -> bytearray:
|
||||
return make_compact_address(self.node_id, self.address, self.udp_port)
|
||||
|
||||
def compact_address_tcp(self) -> bytearray:
|
||||
return make_compact_address(self.node_id, self.address, self.tcp_port)
|
||||
|
||||
def compact_ip(self):
|
||||
return make_compact_ip(self.address)
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, KademliaPeer):
|
||||
raise TypeError("invalid type to compare with Peer: %s" % str(type(other)))
|
||||
return (self.node_id, self.address, self.udp_port) == (other.node_id, other.address, other.udp_port)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.node_id, self.address, self.udp_port))
|
||||
@@ -1,493 +0,0 @@
|
||||
import logging
|
||||
import errno
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
from lbrynet.dht import constants, encoding, msgformat, msgtypes
|
||||
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PingQueue:
|
||||
"""
|
||||
Schedules a 15 minute delayed ping after a new node sends us a query. This is so the new node gets added to the
|
||||
routing table after having been given enough time for a pinhole to expire.
|
||||
"""
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._enqueued_contacts = {}
|
||||
self._pending_contacts = {}
|
||||
self._process_lc = node.get_looping_call(self._process)
|
||||
|
||||
def enqueue_maybe_ping(self, *contacts, **kwargs):
|
||||
delay = kwargs.get('delay', constants.checkRefreshInterval)
|
||||
no_op = (defer.succeed(None), lambda: None)
|
||||
for contact in contacts:
|
||||
if delay and contact not in self._enqueued_contacts:
|
||||
self._pending_contacts.setdefault(contact, self._node.clock.seconds() + delay)
|
||||
else:
|
||||
self._enqueued_contacts.setdefault(contact, no_op)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _ping(self, contact):
|
||||
if contact.contact_is_good:
|
||||
return
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
finally:
|
||||
if contact in self._enqueued_contacts:
|
||||
del self._enqueued_contacts[contact]
|
||||
|
||||
def _process(self):
|
||||
# move contacts that are scheduled to join the queue
|
||||
if self._pending_contacts:
|
||||
now = self._node.clock.seconds()
|
||||
for contact in [contact for contact, schedule in self._pending_contacts.items() if schedule <= now]:
|
||||
del self._pending_contacts[contact]
|
||||
self._enqueued_contacts.setdefault(contact, (defer.succeed(None), lambda: None))
|
||||
# spread pings across 60 seconds to avoid flood and/or false negatives
|
||||
step = 60.0/float(len(self._enqueued_contacts)) if self._enqueued_contacts else 0
|
||||
for index, (contact, (call, _)) in enumerate(self._enqueued_contacts.items()):
|
||||
if call.called and not contact.contact_is_good:
|
||||
self._enqueued_contacts[contact] = self._node.reactor_callLater(index*step, self._ping, contact)
|
||||
|
||||
def start(self):
|
||||
return self._node.safe_start_looping_call(self._process_lc, 60)
|
||||
|
||||
def stop(self):
|
||||
map(None, (cancel() for _, (call, cancel) in self._enqueued_contacts.items() if not call.called))
|
||||
return self._node.safe_stop_looping_call(self._process_lc)
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
|
||||
msgSizeLimit = constants.udpDatagramMaxSize - 26
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._translator = msgformat.DefaultFormat()
|
||||
self._sentMessages = {}
|
||||
self._partialMessages = {}
|
||||
self._partialMessagesProgress = {}
|
||||
self._listening = defer.Deferred(None)
|
||||
self._ping_queue = PingQueue(self._node)
|
||||
self._protocolVersion = constants.protocolVersion
|
||||
self.started_listening_time = 0
|
||||
|
||||
def _migrate_incoming_rpc_args(self, contact, method, *args):
|
||||
if method == b'store' and contact.protocolVersion == 0:
|
||||
if isinstance(args[1], dict):
|
||||
blob_hash = args[0]
|
||||
token = args[1].pop(b'token', None)
|
||||
port = args[1].pop(b'port', -1)
|
||||
originalPublisherID = args[1].pop(b'lbryid', None)
|
||||
age = 0
|
||||
return (blob_hash, token, port, originalPublisherID, age), {}
|
||||
return args, {}
|
||||
|
||||
def _migrate_outgoing_rpc_args(self, contact, method, *args):
|
||||
"""
|
||||
This will reformat protocol version 0 arguments for the store function and will add the
|
||||
protocol version keyword argument to calls to contacts who will accept it
|
||||
"""
|
||||
if contact.protocolVersion == 0:
|
||||
if method == b'store':
|
||||
blob_hash, token, port, originalPublisherID, age = args
|
||||
args = (
|
||||
blob_hash, {
|
||||
b'token': token,
|
||||
b'port': port,
|
||||
b'lbryid': originalPublisherID
|
||||
}, originalPublisherID, False
|
||||
)
|
||||
return args
|
||||
return args
|
||||
if args and isinstance(args[-1], dict):
|
||||
args[-1][b'protocolVersion'] = self._protocolVersion
|
||||
return args
|
||||
return args + ({b'protocolVersion': self._protocolVersion},)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def sendRPC(self, contact, method, args):
|
||||
for _ in range(constants.rpcAttempts):
|
||||
try:
|
||||
response = yield self._sendRPC(contact, method, args)
|
||||
return response
|
||||
except TimeoutError:
|
||||
if contact.contact_is_good:
|
||||
log.debug("RETRY %s ON %s", method, contact)
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
def _sendRPC(self, contact, method, args):
|
||||
"""
|
||||
Sends an RPC to the specified contact
|
||||
|
||||
@param contact: The contact (remote node) to send the RPC to
|
||||
@type contact: kademlia.contacts.Contact
|
||||
@param method: The name of remote method to invoke
|
||||
@type method: str
|
||||
@param args: A list of (non-keyword) arguments to pass to the remote
|
||||
method, in the correct order
|
||||
@type args: tuple
|
||||
|
||||
@return: This immediately returns a deferred object, which will return
|
||||
the result of the RPC call, or raise the relevant exception
|
||||
if the remote node raised one. If C{rawResponse} is set to
|
||||
C{True}, however, it will always return the actual response
|
||||
message (which may be a C{ResponseMessage} or an
|
||||
C{ErrorMessage}).
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
msg = msgtypes.RequestMessage(self._node.node_id, method, self._migrate_outgoing_rpc_args(contact, method,
|
||||
*args))
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
|
||||
if args:
|
||||
log.debug("%s:%i SEND CALL %s(%s) TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
hexlify(args[0]), contact.address, contact.port)
|
||||
else:
|
||||
log.debug("%s:%i SEND CALL %s TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
contact.address, contact.port)
|
||||
|
||||
df = defer.Deferred()
|
||||
|
||||
def _remove_contact(failure): # remove the contact from the routing table and track the failure
|
||||
contact.update_last_failed()
|
||||
try:
|
||||
if not contact.contact_is_good:
|
||||
self._node.removeContact(contact)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
return failure
|
||||
|
||||
def _update_contact(result): # refresh the contact in the routing table
|
||||
contact.update_last_replied()
|
||||
if method == b'findValue':
|
||||
if b'token' in result:
|
||||
contact.update_token(result[b'token'])
|
||||
if b'protocolVersion' not in result:
|
||||
contact.update_protocol_version(0)
|
||||
else:
|
||||
contact.update_protocol_version(result.pop(b'protocolVersion'))
|
||||
d = self._node.addContact(contact)
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
|
||||
df.addCallbacks(_update_contact, _remove_contact)
|
||||
|
||||
# Set the RPC timeout timer
|
||||
timeoutCall, cancelTimeout = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, msg.id)
|
||||
|
||||
# Transmit the data
|
||||
self._send(encodedMsg, msg.id, (contact.address, contact.port))
|
||||
self._sentMessages[msg.id] = (contact, df, timeoutCall, cancelTimeout, method, args)
|
||||
|
||||
df.addErrback(cancelTimeout)
|
||||
return df
|
||||
|
||||
def startProtocol(self):
|
||||
log.info("DHT listening on UDP %i (ext port %i)", self._node.port, self._node.externalUDPPort)
|
||||
if self._listening.called:
|
||||
self._listening = defer.Deferred()
|
||||
self._listening.callback(True)
|
||||
self.started_listening_time = self._node.clock.seconds()
|
||||
return self._ping_queue.start()
|
||||
|
||||
def datagramReceived(self, datagram, address):
|
||||
""" Handles and parses incoming RPC messages (and responses)
|
||||
|
||||
@note: This is automatically called by Twisted when the protocol
|
||||
receives a UDP datagram
|
||||
"""
|
||||
if chr(datagram[0]) == '\x00' and chr(datagram[25]) == '\x00':
|
||||
totalPackets = (datagram[1] << 8) | datagram[2]
|
||||
msgID = datagram[5:25]
|
||||
seqNumber = (datagram[3] << 8) | datagram[4]
|
||||
if msgID not in self._partialMessages:
|
||||
self._partialMessages[msgID] = {}
|
||||
self._partialMessages[msgID][seqNumber] = datagram[26:]
|
||||
if len(self._partialMessages[msgID]) == totalPackets:
|
||||
keys = self._partialMessages[msgID].keys()
|
||||
keys.sort()
|
||||
data = b''
|
||||
for key in keys:
|
||||
data += self._partialMessages[msgID][key]
|
||||
datagram = data
|
||||
del self._partialMessages[msgID]
|
||||
else:
|
||||
return
|
||||
try:
|
||||
msgPrimitive = encoding.bdecode(datagram)
|
||||
message = self._translator.fromPrimitive(msgPrimitive)
|
||||
except (encoding.DecodeError, ValueError) as err:
|
||||
# We received some rubbish here
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", hexlify(datagram),
|
||||
address[0], address[1], err)
|
||||
return
|
||||
except (IndexError, KeyError):
|
||||
log.warning("Couldn't decode dht datagram from %s", address)
|
||||
return
|
||||
|
||||
if isinstance(message, msgtypes.RequestMessage):
|
||||
# This is an RPC method request
|
||||
remoteContact = self._node.contact_manager.make_contact(message.nodeID, address[0], address[1], self)
|
||||
remoteContact.update_last_requested()
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
if remoteContact.contact_is_good is True:
|
||||
df = self._node.addContact(remoteContact)
|
||||
else:
|
||||
df = defer.succeed(None)
|
||||
df.addCallback(lambda _: self._handleRPC(remoteContact, message.id, message.request, message.args))
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
if remoteContact.contact_is_good is None and remoteContact.lastReplied is None:
|
||||
df.addCallback(lambda _: self._ping_queue.enqueue_maybe_ping(remoteContact))
|
||||
elif isinstance(message, msgtypes.ErrorMessage):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
if message.exceptionType in BUILTIN_EXCEPTIONS:
|
||||
exception_type = BUILTIN_EXCEPTIONS[message.exceptionType]
|
||||
else:
|
||||
exception_type = UnknownRemoteException
|
||||
remoteException = exception_type(message.response)
|
||||
log.error("DHT RECV REMOTE EXCEPTION FROM %s:%i: %s", address[0],
|
||||
address[1], remoteException)
|
||||
if message.id in self._sentMessages:
|
||||
# Cancel timeout timer for this RPC
|
||||
remoteContact, df, timeoutCall, timeoutCanceller, method = self._sentMessages[message.id][0:5]
|
||||
timeoutCanceller()
|
||||
del self._sentMessages[message.id]
|
||||
|
||||
# reject replies coming from a different address than what we sent our request to
|
||||
if (remoteContact.address, remoteContact.port) != address:
|
||||
log.warning("Sent request to node %s at %s:%i, got reply from %s:%i",
|
||||
remoteContact.log_id(), remoteContact.address,
|
||||
remoteContact.port, address[0], address[1])
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
return
|
||||
|
||||
# this error is returned by nodes that can be contacted but have an old
|
||||
# and broken version of the ping command, if they return it the node can
|
||||
# be contacted, so we'll treat it as a successful ping
|
||||
old_ping_error = "ping() got an unexpected keyword argument '_rpcNodeContact'"
|
||||
if isinstance(remoteException, TypeError) and \
|
||||
remoteException.message == old_ping_error:
|
||||
log.debug("old pong error")
|
||||
df.callback('pong')
|
||||
else:
|
||||
df.errback(remoteException)
|
||||
elif isinstance(message, msgtypes.ResponseMessage):
|
||||
# Find the message that triggered this response
|
||||
if message.id in self._sentMessages:
|
||||
# Cancel timeout timer for this RPC
|
||||
remoteContact, df, timeoutCall, timeoutCanceller, method = self._sentMessages[message.id][0:5]
|
||||
timeoutCanceller()
|
||||
del self._sentMessages[message.id]
|
||||
log.debug("%s:%i RECV response to %s from %s:%i", self._node.externalIP, self._node.port,
|
||||
method, remoteContact.address, remoteContact.port)
|
||||
|
||||
# When joining the network we made Contact objects for the seed nodes with node ids set to None
|
||||
# Thus, the sent_to_id will also be None, and the contact objects need the ids to be manually set.
|
||||
# These replies have be distinguished from those where the node id in the datagram does not match
|
||||
# the node id of the node we sent a message to (these messages are treated as an error)
|
||||
if remoteContact.id and remoteContact.id != message.nodeID: # sent_to_id will be None for bootstrap
|
||||
log.debug("mismatch: (%s) %s:%i (%s vs %s)", method, remoteContact.address, remoteContact.port,
|
||||
remoteContact.log_id(False), hexlify(message.nodeID))
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
return
|
||||
elif not remoteContact.id:
|
||||
remoteContact.set_id(message.nodeID)
|
||||
|
||||
# We got a result from the RPC
|
||||
df.callback(message.response)
|
||||
else:
|
||||
# If the original message isn't found, it must have timed out
|
||||
# TODO: we should probably do something with this...
|
||||
pass
|
||||
|
||||
def _send(self, data, rpcID, address):
|
||||
""" Transmit the specified data over UDP, breaking it up into several
|
||||
packets if necessary
|
||||
|
||||
If the data is spread over multiple UDP datagrams, the packets have the
|
||||
following structure::
|
||||
| | | | | |||||||||||| 0x00 |
|
||||
|Transmission|Total number|Sequence number| RPC ID |Header end|
|
||||
| type ID | of packets |of this packet | | indicator|
|
||||
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
|
||||
| | | | | |||||||||||| |
|
||||
|
||||
@note: The header used for breaking up large data segments will
|
||||
possibly be moved out of the KademliaProtocol class in the
|
||||
future, into something similar to a message translator/encoder
|
||||
class (see C{kademlia.msgformat} and C{kademlia.encoding}).
|
||||
"""
|
||||
|
||||
if len(data) > self.msgSizeLimit:
|
||||
# We have to spread the data over multiple UDP datagrams,
|
||||
# and provide sequencing information
|
||||
#
|
||||
# 1st byte is transmission type id, bytes 2 & 3 are the
|
||||
# total number of packets in this transmission, bytes 4 &
|
||||
# 5 are the sequence number for this specific packet
|
||||
totalPackets = len(data) // self.msgSizeLimit
|
||||
if len(data) % self.msgSizeLimit > 0:
|
||||
totalPackets += 1
|
||||
encTotalPackets = chr(totalPackets >> 8) + chr(totalPackets & 0xff)
|
||||
seqNumber = 0
|
||||
startPos = 0
|
||||
while seqNumber < totalPackets:
|
||||
packetData = data[startPos:startPos + self.msgSizeLimit]
|
||||
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
|
||||
txData = f'\x00{encTotalPackets}{encSeqNumber}{rpcID}\x00{packetData}'
|
||||
self._scheduleSendNext(txData, address)
|
||||
|
||||
startPos += self.msgSizeLimit
|
||||
seqNumber += 1
|
||||
else:
|
||||
self._scheduleSendNext(data, address)
|
||||
|
||||
def _scheduleSendNext(self, txData, address):
|
||||
"""Schedule the sending of the next UDP packet """
|
||||
delayed_call, _ = self._node.reactor_callSoon(self._write, txData, address)
|
||||
|
||||
def _write(self, txData, address):
|
||||
if self.transport:
|
||||
try:
|
||||
self.transport.write(txData, address)
|
||||
except OSError as err:
|
||||
if err.errno == errno.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
elif err.errno == errno.ENETUNREACH:
|
||||
# this should probably try to retransmit when the network connection is back
|
||||
log.error("Network is unreachable")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(txData), address[0], address[1], err.message, err.errno)
|
||||
raise err
|
||||
else:
|
||||
raise TransportNotConnected()
|
||||
|
||||
def _sendResponse(self, contact, rpcID, response):
|
||||
""" Send a RPC response to the specified contact
|
||||
"""
|
||||
msg = msgtypes.ResponseMessage(rpcID, self._node.node_id, response)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _sendError(self, contact, rpcID, exceptionType, exceptionMessage):
|
||||
""" Send an RPC error message to the specified contact
|
||||
"""
|
||||
exceptionMessage = exceptionMessage.encode()
|
||||
msg = msgtypes.ErrorMessage(rpcID, self._node.node_id, exceptionType, exceptionMessage)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _handleRPC(self, senderContact, rpcID, method, args):
|
||||
""" Executes a local function in response to an RPC request """
|
||||
|
||||
# Set up the deferred callchain
|
||||
def handleError(f):
|
||||
self._sendError(senderContact, rpcID, f.type, f.getErrorMessage())
|
||||
|
||||
def handleResult(result):
|
||||
self._sendResponse(senderContact, rpcID, result)
|
||||
|
||||
df = defer.Deferred()
|
||||
df.addCallback(handleResult)
|
||||
df.addErrback(handleError)
|
||||
|
||||
# Execute the RPC
|
||||
func = getattr(self._node, method.decode(), None)
|
||||
if callable(func) and hasattr(func, "rpcmethod"):
|
||||
# Call the exposed Node method and return the result to the deferred callback chain
|
||||
# if args:
|
||||
# log.debug("%s:%i RECV CALL %s(%s) %s:%i", self._node.externalIP, self._node.port, method,
|
||||
# args[0].encode('hex'), senderContact.address, senderContact.port)
|
||||
# else:
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self._node.externalIP, self._node.port, method,
|
||||
senderContact.address, senderContact.port)
|
||||
if args and isinstance(args[-1], dict) and b'protocolVersion' in args[-1]: # args don't need reformatting
|
||||
senderContact.update_protocol_version(int(args[-1].pop(b'protocolVersion')))
|
||||
a, kw = tuple(args[:-1]), args[-1]
|
||||
else:
|
||||
senderContact.update_protocol_version(0)
|
||||
a, kw = self._migrate_incoming_rpc_args(senderContact, method, *args)
|
||||
try:
|
||||
if method != b'ping':
|
||||
result = func(senderContact, *a)
|
||||
else:
|
||||
result = func()
|
||||
except Exception as e:
|
||||
log.error("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
df.errback(e)
|
||||
else:
|
||||
df.callback(result)
|
||||
else:
|
||||
# No such exposed method
|
||||
df.errback(AttributeError('Invalid method: %s' % method))
|
||||
return df
|
||||
|
||||
def _msgTimeout(self, messageID):
|
||||
""" Called when an RPC request message times out """
|
||||
# Find the message that timed out
|
||||
if messageID not in self._sentMessages:
|
||||
# This should never be reached
|
||||
log.error("deferred timed out, but is not present in sent messages list!")
|
||||
return
|
||||
remoteContact, df, timeout_call, timeout_canceller, method, args = self._sentMessages[messageID]
|
||||
if messageID in self._partialMessages:
|
||||
# We are still receiving this message
|
||||
self._msgTimeoutInProgress(messageID, timeout_canceller, remoteContact, df, method, args)
|
||||
return
|
||||
del self._sentMessages[messageID]
|
||||
# The message's destination node is now considered to be dead;
|
||||
# raise an (asynchronous) TimeoutError exception and update the host node
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
|
||||
def _msgTimeoutInProgress(self, messageID, timeoutCanceller, remoteContact, df, method, args):
|
||||
# See if any progress has been made; if not, kill the message
|
||||
if self._hasProgressBeenMade(messageID):
|
||||
# Reset the RPC timeout timer
|
||||
timeoutCanceller()
|
||||
timeoutCall, cancelTimeout = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, messageID)
|
||||
self._sentMessages[messageID] = (remoteContact, df, timeoutCall, cancelTimeout, method, args)
|
||||
else:
|
||||
# No progress has been made
|
||||
if messageID in self._partialMessagesProgress:
|
||||
del self._partialMessagesProgress[messageID]
|
||||
if messageID in self._partialMessages:
|
||||
del self._partialMessages[messageID]
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
|
||||
def _hasProgressBeenMade(self, messageID):
|
||||
return (
|
||||
messageID in self._partialMessagesProgress and
|
||||
(
|
||||
len(self._partialMessagesProgress[messageID]) !=
|
||||
len(self._partialMessages[messageID])
|
||||
)
|
||||
)
|
||||
|
||||
def stopProtocol(self):
|
||||
""" Called when the transport is disconnected.
|
||||
|
||||
Will only be called once, after all ports are disconnected.
|
||||
"""
|
||||
log.info('Stopping DHT')
|
||||
self._ping_queue.stop()
|
||||
self._node.call_later_manager.stop()
|
||||
log.info('DHT stopped')
|
||||
File renamed without changes.
@@ -0,0 +1,70 @@
|
||||
import asyncio
|
||||
import typing
|
||||
|
||||
from lbrynet.dht import constants
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
|
||||
class DictDataStore:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager'):
|
||||
# Dictionary format:
|
||||
# { <key>: [(<contact>, <age>), ...] }
|
||||
self._data_store: typing.Dict[bytes, typing.List[typing.Tuple['KademliaPeer', float]]] = {}
|
||||
|
||||
self.loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self.completed_blobs: typing.Set[str] = set()
|
||||
|
||||
def removed_expired_peers(self):
|
||||
now = self.loop.time()
|
||||
keys = list(self._data_store.keys())
|
||||
for key in keys:
|
||||
to_remove = []
|
||||
for (peer, ts) in self._data_store[key]:
|
||||
if ts + constants.data_expiration < now or self._peer_manager.peer_is_good(peer) is False:
|
||||
to_remove.append((peer, ts))
|
||||
for item in to_remove:
|
||||
self._data_store[key].remove(item)
|
||||
if not self._data_store[key]:
|
||||
del self._data_store[key]
|
||||
|
||||
def filter_bad_and_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
"""
|
||||
for peer in self.filter_expired_peers(key):
|
||||
if self._peer_manager.peer_is_good(peer) is not False:
|
||||
yield peer
|
||||
|
||||
def filter_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
now = self.loop.time()
|
||||
for (peer, ts) in self._data_store.get(key, []):
|
||||
if ts + constants.data_expiration > now:
|
||||
yield peer
|
||||
|
||||
def has_peers_for_blob(self, key: bytes) -> bool:
|
||||
return key in self._data_store
|
||||
|
||||
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes) -> None:
|
||||
now = self.loop.time()
|
||||
if key in self._data_store:
|
||||
current = list(filter(lambda x: x[0] == contact, self._data_store[key]))
|
||||
if len(current):
|
||||
self._data_store[key][self._data_store[key].index(current[0])] = contact, now
|
||||
else:
|
||||
self._data_store[key].append((contact, now))
|
||||
else:
|
||||
self._data_store[key] = [(contact, now)]
|
||||
|
||||
def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
return list(self.filter_bad_and_expired_peers(key))
|
||||
|
||||
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
|
||||
peers = set()
|
||||
for key, stored in self._data_store.items():
|
||||
peers.update(set(map(lambda tup: tup[0], stored)))
|
||||
return list(peers)
|
||||
@@ -8,20 +8,18 @@ class Distance:
|
||||
we pre-calculate the value of that point.
|
||||
"""
|
||||
|
||||
def __init__(self, key):
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
def __init__(self, key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
self.key = key
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two):
|
||||
def __call__(self, key_two: bytes) -> int:
|
||||
if len(key_two) != constants.hash_length:
|
||||
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
def is_closer(self, a, b):
|
||||
def is_closer(self, a: bytes, b: bytes) -> bool:
|
||||
"""Returns true is `a` is closer to `key` than `b` is"""
|
||||
return self(a) < self(b)
|
||||
|
||||
def to_contact(self, contact):
|
||||
"""A convenience function for calculating the distance to a contact"""
|
||||
return self(contact.id)
|
||||
@@ -0,0 +1,347 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException, TransportNotConnected
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
if TYPE_CHECKING:
|
||||
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import PeerManager, KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FindResponse:
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class FindNodeResponse(FindResponse):
|
||||
def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
|
||||
self.key = key
|
||||
self.close_triples = close_triples
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
return self.key in [triple[0] for triple in self.close_triples]
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
return self.close_triples
|
||||
|
||||
|
||||
class FindValueResponse(FindResponse):
|
||||
def __init__(self, key: bytes, result_dict: typing.Dict):
|
||||
self.key = key
|
||||
self.token = result_dict[b'token']
|
||||
self.close_triples: typing.List[typing.Tuple[bytes, bytes, int]] = result_dict.get(b'contacts', [])
|
||||
self.found_compact_addresses = result_dict.get(key, [])
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
return len(self.found_compact_addresses) > 0
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
|
||||
|
||||
|
||||
def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table
|
||||
|
||||
:param routing_table: a TreeRoutingTable
|
||||
:param key: a 48 byte hash
|
||||
:param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
|
||||
peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
|
||||
"""
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
return shortlist or routing_table.find_close_peers(key)
|
||||
|
||||
|
||||
class IterativeFinder:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.loop = loop
|
||||
self.peer_manager = peer_manager
|
||||
self.routing_table = routing_table
|
||||
self.protocol = protocol
|
||||
|
||||
self.key = key
|
||||
self.bottom_out_limit = bottom_out_limit
|
||||
self.max_results = max_results
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set['KademliaPeer'] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
for peer in get_shortlist(routing_table, key, shortlist):
|
||||
if peer.node_id:
|
||||
self._add_active(peer)
|
||||
else:
|
||||
# seed nodes
|
||||
self._schedule_probe(peer)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
Send the rpc request to the peer and return an object with the FindResponse interface
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def search_exhausted(self):
|
||||
"""
|
||||
This method ends the iterator due no more peers to contact.
|
||||
Override to provide last time results.
|
||||
"""
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
Get an initial or cached result to be put into the Queue. Used for findValue requests where the blob
|
||||
has peers in the local data store of blobs announced to us
|
||||
"""
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _add_active(self, peer):
|
||||
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
|
||||
self.active.add(peer)
|
||||
if self._is_closer(peer):
|
||||
self.prev_closest_peer = self.closest_peer
|
||||
self.closest_peer = peer
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
self._add_active(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
self._add_active(self.peer_manager.get_kademlia_peer(node_id, address, udp_port))
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.TimeoutError:
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except TransportNotConnected:
|
||||
return self.aclose()
|
||||
except RemoteException:
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
|
||||
async def _search_round(self):
|
||||
"""
|
||||
Send up to constants.alpha (5) probes to closest active peers
|
||||
"""
|
||||
|
||||
added = 0
|
||||
to_probe = list(self.active - self.contacted)
|
||||
to_probe.sort(key=lambda peer: self.distance(self.key))
|
||||
for peer in to_probe:
|
||||
if added >= constants.alpha:
|
||||
break
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
self._schedule_probe(peer)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
def _schedule_probe(self, peer: 'KademliaPeer'):
|
||||
self.contacted.add(peer)
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
if self.running:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def __aiter__(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
result = self.get_initial_result() or await self.iteration_queue.get()
|
||||
else:
|
||||
result = await self.iteration_queue.get()
|
||||
if not result:
|
||||
raise StopAsyncIteration
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
self.loop.call_soon(self.aclose)
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [
|
||||
peer for peer in from_iter
|
||||
if peer not in self.yielded_peers
|
||||
and peer.node_id != self.protocol.node_id
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
self.yielded_peers.update(to_yield)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.debug("found")
|
||||
return self.put_result(self.active, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
self.bottom_out_count = 0
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active, True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key)
|
||||
return FindValueResponse(self.key, response)
|
||||
|
||||
def check_result_ready(self, response: FindValueResponse):
|
||||
if response.found:
|
||||
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
|
||||
for compact_addr in response.found_compact_addresses]
|
||||
to_yield = []
|
||||
self.bottom_out_count = 0
|
||||
for blob_peer in blob_peers:
|
||||
if blob_peer not in self.blob_peers:
|
||||
self.blob_peers.add(blob_peer)
|
||||
to_yield.append(blob_peer)
|
||||
if to_yield:
|
||||
# log.info("found %i new peers for blob", len(to_yield))
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
# if self.max_results and len(self.blob_peers) >= self.max_results:
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
return self.protocol.data_store.get_peers_for_blob(self.key)
|
||||
return []
|
||||
@@ -0,0 +1,673 @@
|
||||
import logging
|
||||
import socket
|
||||
import functools
|
||||
import hashlib
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbrynet.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE
|
||||
from lbrynet.dht.error import RemoteException, TransportNotConnected
|
||||
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import PeerManager, KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
old_protocol_errors = {
|
||||
"findNode() takes exactly 2 arguments (5 given)": "0.19.1",
|
||||
"findValue() takes exactly 2 arguments (5 given)": "0.19.1"
|
||||
}
|
||||
|
||||
|
||||
class KademliaRPC:
|
||||
def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.BaseEventLoop, peer_port: int = 3333):
|
||||
self.protocol = protocol
|
||||
self.loop = loop
|
||||
self.peer_port = peer_port
|
||||
self.old_token_secret: bytes = None
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def compact_address(self):
|
||||
compact_ip = functools.reduce(lambda buff, x: buff + bytearray([int(x)]),
|
||||
self.protocol.external_ip.split('.'), bytearray())
|
||||
compact_port = self.peer_port.to_bytes(2, 'big')
|
||||
return compact_ip + compact_port + self.protocol.node_id
|
||||
|
||||
@staticmethod
|
||||
def ping():
|
||||
return b'pong'
|
||||
|
||||
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int) -> bytes:
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not 0 < port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {port}")
|
||||
rpc_contact.update_tcp_port(port)
|
||||
if not self.verify_token(token, rpc_contact.compact_ip()):
|
||||
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
|
||||
pass
|
||||
else:
|
||||
raise ValueError("Invalid token")
|
||||
self.protocol.data_store.add_peer_to_blob(
|
||||
rpc_contact, blob_hash
|
||||
)
|
||||
return b'OK'
|
||||
|
||||
def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid contact node_id length: %i" % len(key))
|
||||
|
||||
contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id)
|
||||
contact_triples = []
|
||||
for contact in contacts:
|
||||
contact_triples.append((contact.node_id, contact.address, contact.udp_port))
|
||||
return contact_triples
|
||||
|
||||
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid blob_exchange hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
b'contacts': self.find_node(rpc_contact, key)
|
||||
}
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
|
||||
# get peers we have stored for this blob_exchange
|
||||
peers = [
|
||||
peer.compact_address_tcp()
|
||||
for peer in self.protocol.data_store.get_peers_for_blob(key)
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if peers:
|
||||
response[key] = peers
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
return h.digest()
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class RemoteKademliaRPC:
|
||||
"""
|
||||
Encapsulates RPC calls to remote Peers
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_tracker: 'PeerManager', protocol: 'KademliaProtocol',
|
||||
peer: 'KademliaPeer'):
|
||||
self.loop = loop
|
||||
self.peer_tracker = peer_tracker
|
||||
self.protocol = protocol
|
||||
self.peer = peer
|
||||
|
||||
async def ping(self) -> bytes:
|
||||
"""
|
||||
:return: b'pong'
|
||||
"""
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_ping(self.protocol.node_id)
|
||||
)
|
||||
return response.response
|
||||
|
||||
async def store(self, blob_hash: bytes) -> bytes:
|
||||
"""
|
||||
:param blob_hash: blob hash as bytes
|
||||
:return: b'OK'
|
||||
"""
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not self.protocol.peer_port or not 0 < self.protocol.peer_port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {self.protocol.peer_port}")
|
||||
token = self.peer_tracker.get_node_token(self.peer.node_id)
|
||||
if not token:
|
||||
find_value_resp = await self.find_value(blob_hash)
|
||||
token = find_value_resp[b'token']
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_store(self.protocol.node_id, blob_hash, token, self.protocol.peer_port)
|
||||
)
|
||||
return response.response
|
||||
|
||||
async def find_node(self, key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
"""
|
||||
:return: [(node_id, address, udp_port), ...]
|
||||
"""
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of find node key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_node(self.protocol.node_id, key)
|
||||
)
|
||||
return [(node_id, address.decode(), udp_port) for node_id, address, udp_port in response.response]
|
||||
|
||||
async def find_value(self, key: bytes) -> typing.Union[typing.Dict]:
|
||||
"""
|
||||
:return: {
|
||||
b'token': <token bytes>,
|
||||
b'contacts': [(node_id, address, udp_port), ...]
|
||||
<key bytes>: [<blob_peer_compact_address, ...]
|
||||
}
|
||||
"""
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of find value key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
)
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
class PingQueue:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
|
||||
self._loop = loop
|
||||
self._protocol = protocol
|
||||
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
|
||||
self._process_task: asyncio.Task = None
|
||||
self._running = False
|
||||
self._running_pings: typing.Set[asyncio.Task] = set()
|
||||
self._default_delay = constants.maybe_ping_delay
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = delay if delay is not None else self._default_delay
|
||||
now = self._loop.time()
|
||||
for peer in peers:
|
||||
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
|
||||
self._pending_contacts[peer] = delay + now
|
||||
|
||||
def maybe_ping(self, peer: 'KademliaPeer'):
|
||||
async def ping_task():
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(peer):
|
||||
if peer not in self._protocol.routing_table.get_peers():
|
||||
self._protocol.add_peer(peer)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
task = self._loop.create_task(ping_task())
|
||||
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
|
||||
self._running_pings.add(task)
|
||||
|
||||
async def _process(self): # send up to 1 ping per second
|
||||
while True:
|
||||
enqueued = list(self._pending_contacts.keys())
|
||||
now = self._loop.time()
|
||||
for peer in enqueued:
|
||||
if self._pending_contacts[peer] <= now:
|
||||
del self._pending_contacts[peer]
|
||||
self.maybe_ping(peer)
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
def start(self):
|
||||
assert not self._running
|
||||
self._running = True
|
||||
if not self._process_task:
|
||||
self._process_task = self._loop.create_task(self._process())
|
||||
|
||||
def stop(self):
|
||||
assert self._running
|
||||
self._running = False
|
||||
if self._process_task:
|
||||
self._process_task.cancel()
|
||||
self._process_task = None
|
||||
while self._running_pings:
|
||||
self._running_pings.pop().cancel()
|
||||
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
self.external_ip = external_ip
|
||||
self.udp_port = udp_port
|
||||
self.peer_port = peer_port
|
||||
self.is_seed_node = False
|
||||
self.partial_messages: typing.Dict[bytes, typing.Dict[bytes, bytes]] = {}
|
||||
self.sent_messages: typing.Dict[bytes, typing.Tuple['KademliaPeer', asyncio.Future, RequestDatagram]] = {}
|
||||
self.protocol_version = constants.protocol_version
|
||||
self.started_listening_time = 0
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
self.token_secret = constants.generate_id()
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
self.rpc_timeout = rpc_timeout
|
||||
self._split_lock = asyncio.Lock(loop=self.loop)
|
||||
self._to_remove: typing.Set['KademliaPeer'] = set()
|
||||
self._to_add: typing.Set['KademliaPeer'] = set()
|
||||
self._wakeup_routing_task = asyncio.Event(loop=self.loop)
|
||||
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@functools.lru_cache(128)
|
||||
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
|
||||
return RemoteKademliaRPC(self.loop, self.peer_manager, self, peer)
|
||||
|
||||
def start(self):
|
||||
self.maintaing_routing_task = self.loop.create_task(self.routing_table_task())
|
||||
|
||||
def stop(self):
|
||||
if self.maintaing_routing_task:
|
||||
self.maintaing_routing_task.cancel()
|
||||
if self.transport:
|
||||
self.disconnect()
|
||||
|
||||
def disconnect(self):
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport: DatagramTransport):
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc):
|
||||
self.stop()
|
||||
|
||||
@staticmethod
|
||||
def _migrate_incoming_rpc_args(peer: 'KademliaPeer', method: bytes, *args) -> typing.Tuple[typing.Tuple,
|
||||
typing.Dict]:
|
||||
if method == b'store' and peer.protocol_version == 0:
|
||||
if isinstance(args[1], dict):
|
||||
blob_hash = args[0]
|
||||
token = args[1].pop(b'token', None)
|
||||
port = args[1].pop(b'port', -1)
|
||||
original_publisher_id = args[1].pop(b'lbryid', None)
|
||||
age = 0
|
||||
return (blob_hash, token, port, original_publisher_id, age), {}
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
for p in self.routing_table.get_peers():
|
||||
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(p)
|
||||
self.routing_table.join_buckets()
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self.routing_table.should_split(bucket_index, peer.node_id):
|
||||
self.routing_table.split_bucket(bucket_index)
|
||||
# Retry the insertion attempt
|
||||
result = await self._add_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
return result
|
||||
else:
|
||||
# We can't split the k-bucket
|
||||
#
|
||||
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
|
||||
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
|
||||
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
|
||||
#
|
||||
# A reasonable extension to this is BEP 0005, which extends the above:
|
||||
#
|
||||
# Not all nodes that we learn about are equal. Some are "good" and some are not.
|
||||
# Many nodes using the DHT are able to send queries and receive responses,
|
||||
# but are not able to respond to queries from other nodes. It is important that
|
||||
# each node's routing table must contain only known good nodes. A good node is
|
||||
# a node has responded to one of our queries within the last 15 minutes. A node
|
||||
# is also good if it has ever responded to one of our queries and has sent us a
|
||||
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
|
||||
# questionable. Nodes become bad when they fail to respond to multiple queries
|
||||
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
|
||||
#
|
||||
# When there are bad or questionable nodes in the bucket, the least recent is selected for
|
||||
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
|
||||
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
|
||||
# is ignored if the pinged node replies.
|
||||
|
||||
not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
|
||||
not_recently_replied = []
|
||||
for p in not_good_contacts:
|
||||
last_replied = self.peer_manager.get_last_replied(p.address, p.udp_port)
|
||||
if not last_replied or last_replied + 60 < self.loop.time():
|
||||
not_recently_replied.append(p)
|
||||
if not_recently_replied:
|
||||
to_replace = not_recently_replied[0]
|
||||
else:
|
||||
to_replace = self.routing_table.buckets[bucket_index].peers[0]
|
||||
last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
|
||||
if last_replied and last_replied + 60 > self.loop.time():
|
||||
return False
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
|
||||
try:
|
||||
to_replace_rpc = self.get_rpc_peer(to_replace)
|
||||
await to_replace_rpc.ping()
|
||||
return False
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
|
||||
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
|
||||
if to_replace in self.routing_table.buckets[bucket_index]:
|
||||
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
|
||||
return await self._add_peer(peer)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer'):
|
||||
if peer.node_id == self.node_id:
|
||||
return False
|
||||
self._to_add.add(peer)
|
||||
self._wakeup_routing_task.set()
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer'):
|
||||
self._to_remove.add(peer)
|
||||
self._wakeup_routing_task.set()
|
||||
|
||||
async def routing_table_task(self):
|
||||
while True:
|
||||
while self._to_remove:
|
||||
async with self._split_lock:
|
||||
peer = self._to_remove.pop()
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
while self._to_add:
|
||||
async with self._split_lock:
|
||||
await self._add_peer(self._to_add.pop())
|
||||
await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop)
|
||||
self._wakeup_routing_task.clear()
|
||||
|
||||
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
|
||||
binascii.hexlify(self.node_id)[:8].decode())
|
||||
method = message.method
|
||||
if method not in [b'ping', b'store', b'findNode', b'findValue']:
|
||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||
if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]:
|
||||
# args don't need reformatting
|
||||
a, kw = tuple(message.args[:-1]), message.args[-1]
|
||||
else:
|
||||
a, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
|
||||
sender_contact.address, sender_contact.udp_port)
|
||||
|
||||
if method == b'ping':
|
||||
result = self.node_rpc.ping()
|
||||
elif method == b'store':
|
||||
blob_hash, token, port, original_publisher_id, age = a
|
||||
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
|
||||
elif method == b'findNode':
|
||||
key, = a
|
||||
result = self.node_rpc.find_node(sender_contact, key)
|
||||
else:
|
||||
assert method == b'findValue'
|
||||
key, = a
|
||||
result = self.node_rpc.find_value(sender_contact, key)
|
||||
|
||||
self.send_response(
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
)
|
||||
|
||||
def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
if is_good is None:
|
||||
self.ping_queue.enqueue_maybe_ping(peer)
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
elif is_good is True:
|
||||
self.add_peer(peer)
|
||||
except ValueError as err:
|
||||
log.debug("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
except Exception as err:
|
||||
log.warning("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
|
||||
def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
|
||||
# Find the message that triggered this response
|
||||
if response_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages[response_datagram.rpc_id]
|
||||
if peer.address != address[0]:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}, expected {peer.address}")
|
||||
)
|
||||
return
|
||||
|
||||
# We got a result from the RPC
|
||||
if peer.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("node has our node id"))
|
||||
return
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
peer.set_id(response_datagram.node_id)
|
||||
peer.update_udp_port(address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
self.add_peer(peer)
|
||||
else:
|
||||
log.warning("%s:%i replied, but after we cancelled the request attempt",
|
||||
peer.address, peer.udp_port)
|
||||
else:
|
||||
# If the original message isn't found, it must have timed out
|
||||
# TODO: we should probably do something with this...
|
||||
pass
|
||||
|
||||
def handle_error_datagram(self, address, error_datagram: ErrorDatagram):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
|
||||
if error_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
)
|
||||
return
|
||||
error_msg = f"" \
|
||||
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
|
||||
f"Args: {request.args}\n" \
|
||||
f"Raised: {str(remote_exception)}"
|
||||
if 'Invalid token' in error_msg:
|
||||
log.debug(error_msg)
|
||||
elif error_datagram.response not in old_protocol_errors:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
|
||||
df.set_exception(remote_exception)
|
||||
return
|
||||
else:
|
||||
if error_datagram.response not in old_protocol_errors:
|
||||
msg = f"Received error from {address[0]}:{address[1]}, but it isn't in response to a " \
|
||||
f"pending request: {str(remote_exception)}"
|
||||
log.warning(msg)
|
||||
else:
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
address[0], address[1], old_protocol_errors[error_datagram.response])
|
||||
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.handle_request_datagram(address, message)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.handle_error_datagram(address, message)
|
||||
else:
|
||||
assert isinstance(message, ResponseDatagram), "sanity"
|
||||
self.handle_response_datagram(address, message)
|
||||
|
||||
async def send_request(self, peer: 'KademliaPeer', request: RequestDatagram) -> ResponseDatagram:
|
||||
self._send(peer, request)
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except asyncio.CancelledError:
|
||||
if not response_fut.done():
|
||||
response_fut.cancel()
|
||||
raise
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.remove_peer(peer)
|
||||
raise
|
||||
|
||||
def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
self._send(peer, response)
|
||||
|
||||
def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
|
||||
self._send(peer, error)
|
||||
|
||||
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
|
||||
ErrorDatagram]):
|
||||
if not self.transport or self.transport.is_closing():
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
if len(data) > constants.msg_size_limit:
|
||||
log.exception("unexpected: %i vs %i", len(data), constants.msg_size_limit)
|
||||
raise ValueError()
|
||||
if isinstance(message, (RequestDatagram, ResponseDatagram)):
|
||||
assert message.node_id == self.node_id, message
|
||||
if isinstance(message, RequestDatagram):
|
||||
assert self.node_id != peer.node_id
|
||||
|
||||
def pop_from_sent_messages(_):
|
||||
if message.rpc_id in self.sent_messages:
|
||||
self.sent_messages.pop(message.rpc_id)
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
return constants.digest(self.token_secret + compact_ip)
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = constants.hash_class()
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = constants.hash_class()
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer') -> typing.Tuple[bytes, bool]:
|
||||
async def __store():
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
|
||||
try:
|
||||
return await __store()
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
try:
|
||||
return await __store()
|
||||
except:
|
||||
return peer.node_id, False
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
def _write(self, data: bytes, address: typing.Tuple[str, int]):
|
||||
if self.transport:
|
||||
try:
|
||||
self.transport.sendto(data, address)
|
||||
except OSError as err:
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
# elif err.errno == socket.ENETUNREACH:
|
||||
# # this should probably try to retransmit when the network connection is back
|
||||
# log.error("Network is unreachable")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), address[0], address[1], str(err), err.errno)
|
||||
raise err
|
||||
else:
|
||||
raise TransportNotConnected()
|
||||
@@ -0,0 +1,319 @@
|
||||
import asyncio
|
||||
import random
|
||||
import logging
|
||||
import typing
|
||||
import itertools
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
|
||||
"""
|
||||
@param range_min: The lower boundary for the range in the n-bit ID
|
||||
space covered by this k-bucket
|
||||
@param range_max: The upper boundary for the range in the ID space
|
||||
covered by this k-bucket
|
||||
"""
|
||||
self._peer_manager = peer_manager
|
||||
self.last_accessed = 0
|
||||
self.range_min = range_min
|
||||
self.range_max = range_max
|
||||
self.peers: typing.List['KademliaPeer'] = []
|
||||
self._node_id = node_id
|
||||
self._distance_to_self = Distance(node_id)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
contact to the end of the k-bucket if it is already present.
|
||||
|
||||
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
|
||||
the contact isn't in the bucket
|
||||
already
|
||||
|
||||
@param peer: The contact to add
|
||||
@type peer: dht.contact._Contact
|
||||
"""
|
||||
if peer in self.peers:
|
||||
# Move the existing contact to the end of the list
|
||||
# - using the new contact to allow add-on data
|
||||
# (e.g. optimization-specific stuff) to pe updated as well
|
||||
self.peers.remove(peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
p = self.peers[i]
|
||||
if p.node_id == peer.node_id:
|
||||
self.peers.remove(p)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
if len(self.peers) < constants.k:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
# raise BucketFull("No space in bucket to insert contact")
|
||||
|
||||
def get_peer(self, node_id: bytes) -> 'KademliaPeer':
|
||||
for peer in self.peers:
|
||||
if peer.node_id == node_id:
|
||||
return peer
|
||||
raise IndexError(node_id)
|
||||
|
||||
def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
|
||||
""" Returns a list containing up to the first count number of contacts
|
||||
|
||||
@param count: The amount of contacts to return (if 0 or less, return
|
||||
all contacts)
|
||||
@type count: int
|
||||
@param exclude_contact: A node node_id to exclude; if this contact is in
|
||||
the list of returned values, it will be
|
||||
discarded before returning. If a C{str} is
|
||||
passed as this argument, it must be the
|
||||
contact's ID.
|
||||
@type exclude_contact: str
|
||||
|
||||
@param sort_distance_to: Sort distance to the node_id, defaulting to the parent node node_id. If False don't
|
||||
sort the contacts
|
||||
|
||||
@raise IndexError: If the number of requested contacts is too large
|
||||
|
||||
@return: Return up to the first count number of contacts in a list
|
||||
If no contacts are present an empty is returned
|
||||
@rtype: list
|
||||
"""
|
||||
peers = [peer for peer in self.peers if peer.node_id != exclude_contact]
|
||||
|
||||
# Return all contacts in bucket
|
||||
if count <= 0:
|
||||
count = len(peers)
|
||||
|
||||
# Get current contact number
|
||||
current_len = len(peers)
|
||||
|
||||
# If count greater than k - return only k contacts
|
||||
if count > constants.k:
|
||||
count = constants.k
|
||||
|
||||
if not current_len:
|
||||
return peers
|
||||
|
||||
if sort_distance_to is False:
|
||||
pass
|
||||
else:
|
||||
sort_distance_to = sort_distance_to or self._node_id
|
||||
peers.sort(key=lambda c: Distance(sort_distance_to)(c.node_id))
|
||||
|
||||
return peers[:min(current_len, count)]
|
||||
|
||||
def get_bad_or_unknown_peers(self) -> typing.List['KademliaPeer']:
|
||||
peer = self.get_peers(sort_distance_to=False)
|
||||
return [
|
||||
peer for peer in peer
|
||||
if self._peer_manager.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port) is not True
|
||||
]
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
self.peers.remove(peer)
|
||||
|
||||
def key_in_range(self, key: bytes) -> bool:
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
of the n-bit ID space covered by this k-bucket (in otherwords, it
|
||||
returns whether or not the specified key should be placed in this
|
||||
k-bucket)
|
||||
|
||||
@param key: The key to test
|
||||
@type key: str or int
|
||||
|
||||
@return: C{True} if the key is in this k-bucket's range, or C{False}
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
return self.range_min <= self._distance_to_self(key) < self.range_max
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.peers)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.peers
|
||||
|
||||
|
||||
class TreeRoutingTable:
|
||||
""" This class implements a routing table used by a Node class.
|
||||
|
||||
The Kademlia routing table is a binary tree whose leaves are k-buckets,
|
||||
where each k-bucket contains nodes with some common prefix of their IDs.
|
||||
This prefix is the k-bucket's position in the binary tree; it therefore
|
||||
covers some range of ID values, and together all of the k-buckets cover
|
||||
the entire n-bit ID (or key) space (with no overlap).
|
||||
|
||||
@note: In this implementation, nodes in the tree (the k-buckets) are
|
||||
added dynamically, as needed; this technique is described in the 13-page
|
||||
version of the Kademlia paper, in section 2.4. It does, however, use the
|
||||
ping RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self._split_buckets_under_index = split_buckets_under_index
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
|
||||
)
|
||||
]
|
||||
|
||||
def get_peers(self) -> typing.List['KademliaPeer']:
|
||||
return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
|
||||
|
||||
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if bucket_index < self._split_buckets_under_index:
|
||||
return True
|
||||
contacts = self.get_peers()
|
||||
distance = Distance(self._parent_node_id)
|
||||
contacts.sort(key=lambda c: distance(c.node_id))
|
||||
kth_contact = contacts[-1] if len(contacts) < constants.k else contacts[constants.k - 1]
|
||||
return distance(to_add) < distance(kth_contact.node_id)
|
||||
|
||||
def find_close_peers(self, key: bytes, count: typing.Optional[int] = None,
|
||||
sender_node_id: typing.Optional[bytes] = None) -> typing.List['KademliaPeer']:
|
||||
exclude = [self._parent_node_id]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
count = count or constants.k
|
||||
distance = Distance(key)
|
||||
contacts = self.get_peers()
|
||||
contacts = [c for c in contacts if c.node_id not in exclude]
|
||||
if contacts:
|
||||
contacts.sort(key=lambda c: distance(c.node_id))
|
||||
return contacts[:min(count, len(contacts))]
|
||||
return []
|
||||
|
||||
def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
|
||||
"""
|
||||
@raise IndexError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
|
||||
|
||||
def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
|
||||
bucket_index = start_index
|
||||
refresh_ids = []
|
||||
now = int(self._loop.time())
|
||||
for bucket in self.buckets[start_index:]:
|
||||
if force or now - bucket.last_accessed >= constants.refresh_interval:
|
||||
to_search = self.midpoint_id_in_bucket_range(bucket_index)
|
||||
refresh_ids.append(to_search)
|
||||
bucket_index += 1
|
||||
return refresh_ids
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
if not peer.node_id:
|
||||
return
|
||||
bucket_index = self.kbucket_index(peer.node_id)
|
||||
try:
|
||||
self.buckets[bucket_index].remove_peer(peer)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
def touch_kbucket(self, key: bytes) -> None:
|
||||
self.touch_kbucket_by_index(self.kbucket_index(key))
|
||||
|
||||
def touch_kbucket_by_index(self, bucket_index: int):
|
||||
self.buckets[bucket_index].last_accessed = int(self._loop.time())
|
||||
|
||||
def kbucket_index(self, key: bytes) -> int:
|
||||
i = 0
|
||||
for bucket in self.buckets:
|
||||
if bucket.key_in_range(key):
|
||||
return i
|
||||
else:
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return Distance(
|
||||
self._parent_node_id
|
||||
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return Distance(self._parent_node_id)(
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
cover the same range in the key/ID space
|
||||
|
||||
@param old_bucket_index: The index of k-bucket to split (in this table's
|
||||
list of k-buckets)
|
||||
@type old_bucket_index: int
|
||||
"""
|
||||
# Resize the range of the current (old) k-bucket
|
||||
old_bucket = self.buckets[old_bucket_index]
|
||||
split_point = old_bucket.range_max - (old_bucket.range_max - old_bucket.range_min) // 2
|
||||
# Create a new k-bucket to cover the range split off from the old bucket
|
||||
new_bucket = KBucket(self._peer_manager, split_point, old_bucket.range_max, self._parent_node_id)
|
||||
old_bucket.range_max = split_point
|
||||
# Now, add the new bucket into the routing table tree
|
||||
self.buckets.insert(old_bucket_index + 1, new_bucket)
|
||||
# Finally, copy all nodes that belong to the new k-bucket into it...
|
||||
for contact in old_bucket.peers:
|
||||
if new_bucket.key_in_range(contact.node_id):
|
||||
new_bucket.add_peer(contact)
|
||||
# ...and remove them from the old bucket
|
||||
for contact in new_bucket.peers:
|
||||
old_bucket.remove_peer(contact)
|
||||
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
return
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
|
||||
if not to_pop:
|
||||
return
|
||||
log.info("join buckets %i", len(to_pop))
|
||||
bucket_index_to_pop = to_pop[0]
|
||||
assert len(self.buckets[bucket_index_to_pop]) == 0
|
||||
can_go_lower = bucket_index_to_pop - 1 >= 0
|
||||
can_go_higher = bucket_index_to_pop + 1 < len(self.buckets)
|
||||
assert can_go_higher or can_go_lower
|
||||
bucket = self.buckets[bucket_index_to_pop]
|
||||
if can_go_lower and can_go_higher:
|
||||
midpoint = ((bucket.range_max - bucket.range_min) // 2) + bucket.range_min
|
||||
self.buckets[bucket_index_to_pop - 1].range_max = midpoint - 1
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = midpoint
|
||||
elif can_go_lower:
|
||||
self.buckets[bucket_index_to_pop - 1].range_max = bucket.range_max
|
||||
elif can_go_higher:
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
|
||||
self.buckets.remove(bucket)
|
||||
return self.join_buckets()
|
||||
|
||||
def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
|
||||
for bucket in self.buckets:
|
||||
for contact in bucket.get_peers(sort_distance_to=False):
|
||||
if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
|
||||
return True
|
||||
return False
|
||||
|
||||
def buckets_with_contacts(self) -> int:
|
||||
count = 0
|
||||
for bucket in self.buckets:
|
||||
if len(bucket):
|
||||
count += 1
|
||||
return count
|
||||
@@ -1,320 +0,0 @@
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import random
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.dht import constants, kbucket
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht.distance import Distance
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TreeRoutingTable:
|
||||
""" This class implements a routing table used by a Node class.
|
||||
|
||||
The Kademlia routing table is a binary tree whFose leaves are k-buckets,
|
||||
where each k-bucket contains nodes with some common prefix of their IDs.
|
||||
This prefix is the k-bucket's position in the binary tree; it therefore
|
||||
covers some range of ID values, and together all of the k-buckets cover
|
||||
the entire n-bit ID (or key) space (with no overlap).
|
||||
|
||||
@note: In this implementation, nodes in the tree (the k-buckets) are
|
||||
added dynamically, as needed; this technique is described in the 13-page
|
||||
version of the Kademlia paper, in section 2.4. It does, however, use the
|
||||
C{PING} RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
#implements(IRoutingTable)
|
||||
|
||||
def __init__(self, parentNodeID, getTime=None):
|
||||
"""
|
||||
@param parentNodeID: The n-bit node ID of the node to which this
|
||||
routing table belongs
|
||||
@type parentNodeID: str
|
||||
"""
|
||||
# Create the initial (single) k-bucket covering the range of the entire n-bit ID space
|
||||
self._parentNodeID = parentNodeID
|
||||
self._buckets = [kbucket.KBucket(rangeMin=0, rangeMax=2 ** constants.key_bits, node_id=self._parentNodeID)]
|
||||
if not getTime:
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self._ongoing_replacements = set()
|
||||
|
||||
def get_contacts(self):
|
||||
contacts = []
|
||||
for i in range(len(self._buckets)):
|
||||
for contact in self._buckets[i]._contacts:
|
||||
contacts.append(contact)
|
||||
return contacts
|
||||
|
||||
def _shouldSplit(self, bucketIndex, toAdd):
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if self._buckets[bucketIndex].keyInRange(self._parentNodeID):
|
||||
return True
|
||||
contacts = self.get_contacts()
|
||||
distance = Distance(self._parentNodeID)
|
||||
contacts.sort(key=lambda c: distance(c.id))
|
||||
kth_contact = contacts[-1] if len(contacts) < constants.k else contacts[constants.k-1]
|
||||
return distance(toAdd) < distance(kth_contact.id)
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add the given contact to the correct k-bucket; if it already
|
||||
exists, its status will be updated
|
||||
|
||||
@param contact: The contact to add to this node's k-buckets
|
||||
@type contact: kademlia.contact.Contact
|
||||
|
||||
@rtype: defer.Deferred
|
||||
"""
|
||||
|
||||
if contact.id == self._parentNodeID:
|
||||
return defer.succeed(None)
|
||||
bucketIndex = self._kbucketIndex(contact.id)
|
||||
try:
|
||||
self._buckets[bucketIndex].addContact(contact)
|
||||
except kbucket.BucketFull:
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's id)
|
||||
if self._shouldSplit(bucketIndex, contact.id):
|
||||
self._splitBucket(bucketIndex)
|
||||
# Retry the insertion attempt
|
||||
return self.addContact(contact)
|
||||
else:
|
||||
# We can't split the k-bucket
|
||||
#
|
||||
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
|
||||
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
|
||||
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
|
||||
#
|
||||
# A reasonable extension to this is BEP 0005, which extends the above:
|
||||
#
|
||||
# Not all nodes that we learn about are equal. Some are "good" and some are not.
|
||||
# Many nodes using the DHT are able to send queries and receive responses,
|
||||
# but are not able to respond to queries from other nodes. It is important that
|
||||
# each node's routing table must contain only known good nodes. A good node is
|
||||
# a node has responded to one of our queries within the last 15 minutes. A node
|
||||
# is also good if it has ever responded to one of our queries and has sent us a
|
||||
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
|
||||
# questionable. Nodes become bad when they fail to respond to multiple queries
|
||||
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
|
||||
#
|
||||
# When there are bad or questionable nodes in the bucket, the least recent is selected for
|
||||
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
|
||||
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
|
||||
# is ignored if the pinged node replies.
|
||||
|
||||
def replaceContact(failure, deadContact):
|
||||
"""
|
||||
Callback for the deferred PING RPC to see if the node to be replaced in the k-bucket is still
|
||||
responding
|
||||
|
||||
@type failure: twisted.python.failure.Failure
|
||||
"""
|
||||
failure.trap(TimeoutError)
|
||||
log.debug("Replacing dead contact in bucket %i: %s:%i (%s) with %s:%i (%s)", bucketIndex,
|
||||
deadContact.address, deadContact.port, deadContact.log_id(), contact.address,
|
||||
contact.port, contact.log_id())
|
||||
try:
|
||||
self._buckets[bucketIndex].removeContact(deadContact)
|
||||
except ValueError:
|
||||
# The contact has already been removed (probably due to a timeout)
|
||||
pass
|
||||
return self.addContact(contact)
|
||||
|
||||
not_good_contacts = self._buckets[bucketIndex].getBadOrUnknownContacts()
|
||||
if not_good_contacts:
|
||||
to_replace = not_good_contacts[0]
|
||||
else:
|
||||
to_replace = self._buckets[bucketIndex]._contacts[0]
|
||||
if to_replace not in self._ongoing_replacements:
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.port)
|
||||
self._ongoing_replacements.add(to_replace)
|
||||
df = to_replace.ping()
|
||||
df.addErrback(replaceContact, to_replace)
|
||||
df.addBoth(lambda _: self._ongoing_replacements.remove(to_replace))
|
||||
else:
|
||||
df = defer.succeed(None)
|
||||
return df
|
||||
else:
|
||||
self.touchKBucketByIndex(bucketIndex)
|
||||
return defer.succeed(None)
|
||||
|
||||
def findCloseNodes(self, key, count=None, sender_node_id=None):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
@param count: the amount of contacts to return, default of k (8)
|
||||
@type count: int
|
||||
@param sender_node_id: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type sender_node_id: str
|
||||
|
||||
@return: A list of node contacts (C{kademlia.contact.Contact instances})
|
||||
closest to the specified key.
|
||||
This method will return C{k} (or C{count}, if specified)
|
||||
contacts if at all possible; it will only return fewer if the
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
exclude = [self._parentNodeID]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
if key in exclude:
|
||||
exclude.remove(key)
|
||||
count = count or constants.k
|
||||
distance = Distance(key)
|
||||
contacts = self.get_contacts()
|
||||
contacts = [c for c in contacts if c.id not in exclude]
|
||||
contacts.sort(key=lambda c: distance(c.id))
|
||||
return contacts[:min(count, len(contacts))]
|
||||
|
||||
def getContact(self, contactID):
|
||||
""" Returns the (known) contact with the specified node ID
|
||||
|
||||
@raise ValueError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
bucketIndex = self._kbucketIndex(contactID)
|
||||
return self._buckets[bucketIndex].getContact(contactID)
|
||||
|
||||
def getRefreshList(self, startIndex=0, force=False):
|
||||
""" Finds all k-buckets that need refreshing, starting at the
|
||||
k-bucket with the specified index, and returns IDs to be searched for
|
||||
in order to refresh those k-buckets
|
||||
|
||||
@param startIndex: The index of the bucket to start refreshing at;
|
||||
this bucket and those further away from it will
|
||||
be refreshed. For example, when joining the
|
||||
network, this node will set this to the index of
|
||||
the bucket after the one containing it's closest
|
||||
neighbour.
|
||||
@type startIndex: index
|
||||
@param force: If this is C{True}, all buckets (in the specified range)
|
||||
will be refreshed, regardless of the time they were last
|
||||
accessed.
|
||||
@type force: bool
|
||||
|
||||
@return: A list of node ID's that the parent node should search for
|
||||
in order to refresh the routing Table
|
||||
@rtype: list
|
||||
"""
|
||||
bucketIndex = startIndex
|
||||
refreshIDs = []
|
||||
now = int(self._getTime())
|
||||
for bucket in self._buckets[startIndex:]:
|
||||
if force or now - bucket.lastAccessed >= constants.refreshTimeout:
|
||||
searchID = self.midpoint_id_in_bucket_range(bucketIndex)
|
||||
refreshIDs.append(searchID)
|
||||
bucketIndex += 1
|
||||
return refreshIDs
|
||||
|
||||
def removeContact(self, contact):
|
||||
"""
|
||||
Remove the contact from the routing table
|
||||
|
||||
@param contact: The contact to remove
|
||||
@type contact: dht.contact._Contact
|
||||
"""
|
||||
bucketIndex = self._kbucketIndex(contact.id)
|
||||
try:
|
||||
self._buckets[bucketIndex].removeContact(contact)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
def touchKBucket(self, key):
|
||||
""" Update the "last accessed" timestamp of the k-bucket which covers
|
||||
the range containing the specified key in the key/ID space
|
||||
|
||||
@param key: A key in the range of the target k-bucket
|
||||
@type key: str
|
||||
"""
|
||||
self.touchKBucketByIndex(self._kbucketIndex(key))
|
||||
|
||||
def touchKBucketByIndex(self, bucketIndex):
|
||||
self._buckets[bucketIndex].lastAccessed = int(self._getTime())
|
||||
|
||||
def _kbucketIndex(self, key):
|
||||
""" Calculate the index of the k-bucket which is responsible for the
|
||||
specified key (or ID)
|
||||
|
||||
@param key: The key for which to find the appropriate k-bucket index
|
||||
@type key: str
|
||||
|
||||
@return: The index of the k-bucket responsible for the specified key
|
||||
@rtype: int
|
||||
"""
|
||||
i = 0
|
||||
for bucket in self._buckets:
|
||||
if bucket.keyInRange(key):
|
||||
return i
|
||||
else:
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def random_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns a random ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
|
||||
random_id = int(random.randrange(self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax))
|
||||
return random_id.to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns the middle ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
|
||||
half = int((self._buckets[bucketIndex].rangeMax - self._buckets[bucketIndex].rangeMin) // 2)
|
||||
return int(self._buckets[bucketIndex].rangeMin + half).to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def _splitBucket(self, oldBucketIndex):
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
cover the same range in the key/ID space
|
||||
|
||||
@param oldBucketIndex: The index of k-bucket to split (in this table's
|
||||
list of k-buckets)
|
||||
@type oldBucketIndex: int
|
||||
"""
|
||||
# Resize the range of the current (old) k-bucket
|
||||
oldBucket = self._buckets[oldBucketIndex]
|
||||
splitPoint = oldBucket.rangeMax - (oldBucket.rangeMax - oldBucket.rangeMin) / 2
|
||||
# Create a new k-bucket to cover the range split off from the old bucket
|
||||
newBucket = kbucket.KBucket(splitPoint, oldBucket.rangeMax, self._parentNodeID)
|
||||
oldBucket.rangeMax = splitPoint
|
||||
# Now, add the new bucket into the routing table tree
|
||||
self._buckets.insert(oldBucketIndex + 1, newBucket)
|
||||
# Finally, copy all nodes that belong to the new k-bucket into it...
|
||||
for contact in oldBucket._contacts:
|
||||
if newBucket.keyInRange(contact.id):
|
||||
newBucket.addContact(contact)
|
||||
# ...and remove them from the old bucket
|
||||
for contact in newBucket._contacts:
|
||||
oldBucket.removeContact(contact)
|
||||
|
||||
def contactInRoutingTable(self, address_tuple):
|
||||
for bucket in self._buckets:
|
||||
for contact in bucket.getContacts(sort_distance_to=False):
|
||||
if address_tuple[0] == contact.address and address_tuple[1] == contact.port:
|
||||
return True
|
||||
return False
|
||||
|
||||
def bucketsWithContacts(self):
|
||||
count = 0
|
||||
for bucket in self._buckets:
|
||||
if len(bucket):
|
||||
count += 1
|
||||
return count
|
||||
File renamed without changes.
@@ -1,8 +1,8 @@
|
||||
import typing
|
||||
from lbrynet.dht.error import DecodeError
|
||||
|
||||
|
||||
def bencode(data):
|
||||
""" Encoder implementation of the Bencode algorithm (Bittorrent). """
|
||||
def _bencode(data: typing.Union[int, bytes, bytearray, str, list, tuple, dict]) -> bytes:
|
||||
if isinstance(data, int):
|
||||
return b'i%de' % data
|
||||
elif isinstance(data, (bytes, bytearray)):
|
||||
@@ -12,31 +12,20 @@ def bencode(data):
|
||||
elif isinstance(data, (list, tuple)):
|
||||
encoded_list_items = b''
|
||||
for item in data:
|
||||
encoded_list_items += bencode(item)
|
||||
encoded_list_items += _bencode(item)
|
||||
return b'l%se' % encoded_list_items
|
||||
elif isinstance(data, dict):
|
||||
encoded_dict_items = b''
|
||||
keys = data.keys()
|
||||
for key in sorted(keys):
|
||||
encoded_dict_items += bencode(key)
|
||||
encoded_dict_items += bencode(data[key])
|
||||
encoded_dict_items += _bencode(key)
|
||||
encoded_dict_items += _bencode(data[key])
|
||||
return b'd%se' % encoded_dict_items
|
||||
else:
|
||||
raise TypeError("Cannot bencode '%s' object" % type(data))
|
||||
raise TypeError(f"Cannot bencode {type(data)}")
|
||||
|
||||
|
||||
def bdecode(data):
|
||||
""" Decoder implementation of the Bencode algorithm. """
|
||||
assert type(data) == bytes # fixme: _maybe_ remove this after porting
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
try:
|
||||
return _decode_recursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError(str(e))
|
||||
|
||||
|
||||
def _decode_recursive(data, start_index=0):
|
||||
def _bdecode(data: bytes, start_index: int = 0) -> typing.Tuple[typing.Union[int, bytes, list, tuple, dict], int]:
|
||||
if data[start_index] == ord('i'):
|
||||
end_pos = data[start_index:].find(b'e') + start_index
|
||||
return int(data[start_index + 1:end_pos]), end_pos + 1
|
||||
@@ -44,32 +33,44 @@ def _decode_recursive(data, start_index=0):
|
||||
start_index += 1
|
||||
decoded_list = []
|
||||
while data[start_index] != ord('e'):
|
||||
list_data, start_index = _decode_recursive(data, start_index)
|
||||
list_data, start_index = _bdecode(data, start_index)
|
||||
decoded_list.append(list_data)
|
||||
return decoded_list, start_index + 1
|
||||
elif data[start_index] == ord('d'):
|
||||
start_index += 1
|
||||
decoded_dict = {}
|
||||
while data[start_index] != ord('e'):
|
||||
key, start_index = _decode_recursive(data, start_index)
|
||||
value, start_index = _decode_recursive(data, start_index)
|
||||
key, start_index = _bdecode(data, start_index)
|
||||
value, start_index = _bdecode(data, start_index)
|
||||
decoded_dict[key] = value
|
||||
return decoded_dict, start_index
|
||||
elif data[start_index] == ord('f'):
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
end_pos = data[start_index:].find(b'e') + start_index
|
||||
return float(data[start_index + 1:end_pos]), end_pos + 1
|
||||
elif data[start_index] == ord('n'):
|
||||
# This (None/NULL data type) is a non-standard extension
|
||||
# to the original Bencode algorithm
|
||||
return None, start_index + 1
|
||||
else:
|
||||
split_pos = data[start_index:].find(b':') + start_index
|
||||
try:
|
||||
length = int(data[start_index:split_pos])
|
||||
except ValueError:
|
||||
raise DecodeError()
|
||||
except (ValueError, TypeError) as err:
|
||||
raise DecodeError(err)
|
||||
start_index = split_pos + 1
|
||||
end_pos = start_index + length
|
||||
b = data[start_index:end_pos]
|
||||
return b, end_pos
|
||||
|
||||
|
||||
def bencode(data: typing.Dict) -> bytes:
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError()
|
||||
return _bencode(data)
|
||||
|
||||
|
||||
def bdecode(data: bytes, allow_non_dict_return: typing.Optional[bool] = False) -> typing.Dict:
|
||||
assert type(data) == bytes, DecodeError(f"invalid data type: {str(type(data))}")
|
||||
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
try:
|
||||
result = _bdecode(data)[0]
|
||||
if not allow_non_dict_return and not isinstance(result, dict):
|
||||
raise ValueError(f'expected dict, got {type(result)}')
|
||||
return result
|
||||
except (ValueError, TypeError) as err:
|
||||
raise DecodeError(err)
|
||||
@@ -0,0 +1,178 @@
|
||||
import typing
|
||||
from functools import reduce
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.bencoding import bencode, bdecode
|
||||
|
||||
REQUEST_TYPE = 0
|
||||
RESPONSE_TYPE = 1
|
||||
ERROR_TYPE = 2
|
||||
|
||||
|
||||
class KademliaDatagramBase:
|
||||
"""
|
||||
field names are used to unwrap/wrap the argument names to index integers that replace them in a datagram
|
||||
all packets have an argument dictionary when bdecoded starting with {0: <int>, 1: <bytes>, 2: <bytes>, ...}
|
||||
these correspond to the packet_type, rpc_id, and node_id args
|
||||
"""
|
||||
|
||||
fields = [
|
||||
'packet_type',
|
||||
'rpc_id',
|
||||
'node_id'
|
||||
]
|
||||
|
||||
expected_packet_type = -1
|
||||
|
||||
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes):
|
||||
self.packet_type = packet_type
|
||||
if self.expected_packet_type != packet_type:
|
||||
raise ValueError(f"invalid packet type: {packet_type}, expected {self.expected_packet_type}")
|
||||
if len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError(f"invalid rpc node_id: {len(rpc_id)} bytes (expected 20)")
|
||||
if not len(node_id) == constants.hash_length:
|
||||
raise ValueError(f"invalid node node_id: {len(node_id)} bytes (expected 48)")
|
||||
self.rpc_id = rpc_id
|
||||
self.node_id = node_id
|
||||
|
||||
def bencode(self) -> bytes:
|
||||
return bencode({
|
||||
i: getattr(self, k) for i, k in enumerate(self.fields)
|
||||
})
|
||||
|
||||
|
||||
class RequestDatagram(KademliaDatagramBase):
|
||||
fields = [
|
||||
'packet_type',
|
||||
'rpc_id',
|
||||
'node_id',
|
||||
'method',
|
||||
'args'
|
||||
]
|
||||
|
||||
expected_packet_type = REQUEST_TYPE
|
||||
|
||||
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, method: bytes,
|
||||
args: typing.Optional[typing.List] = None):
|
||||
super().__init__(packet_type, rpc_id, node_id)
|
||||
self.method = method
|
||||
self.args = args or []
|
||||
if not self.args:
|
||||
self.args.append({})
|
||||
if isinstance(self.args[-1], dict):
|
||||
self.args[-1][b'protocolVersion'] = 1
|
||||
else:
|
||||
self.args.append({b'protocolVersion': 1})
|
||||
|
||||
@classmethod
|
||||
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
|
||||
|
||||
@classmethod
|
||||
def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid blob hash length: {len(blob_hash)}")
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f"invalid port: {port}")
|
||||
if len(token) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid token length: {len(token)}")
|
||||
store_args = [blob_hash, token, port, from_node_id, 0]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args)
|
||||
|
||||
@classmethod
|
||||
def make_find_node(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key])
|
||||
|
||||
@classmethod
|
||||
def make_find_value(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
|
||||
|
||||
|
||||
class ResponseDatagram(KademliaDatagramBase):
|
||||
fields = [
|
||||
'packet_type',
|
||||
'rpc_id',
|
||||
'node_id',
|
||||
'response'
|
||||
]
|
||||
|
||||
expected_packet_type = RESPONSE_TYPE
|
||||
|
||||
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, response):
|
||||
super().__init__(packet_type, rpc_id, node_id)
|
||||
self.response = response
|
||||
|
||||
|
||||
class ErrorDatagram(KademliaDatagramBase):
|
||||
fields = [
|
||||
'packet_type',
|
||||
'rpc_id',
|
||||
'node_id',
|
||||
'exception_type',
|
||||
'response',
|
||||
]
|
||||
|
||||
expected_packet_type = ERROR_TYPE
|
||||
|
||||
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, exception_type: bytes, response: bytes):
|
||||
super().__init__(packet_type, rpc_id, node_id)
|
||||
self.exception_type = exception_type.decode()
|
||||
self.response = response.decode()
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
msg_types = {
|
||||
REQUEST_TYPE: RequestDatagram,
|
||||
RESPONSE_TYPE: ResponseDatagram,
|
||||
ERROR_TYPE: ErrorDatagram
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
raise ValueError("invalid datagram type")
|
||||
datagram_class = msg_types[datagram_type]
|
||||
return datagram_class(**{
|
||||
k: primitive[i] # pylint: disable=unsubscriptable-object
|
||||
for i, k in enumerate(datagram_class.fields)
|
||||
if i in primitive # pylint: disable=unsupported-membership-test
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def make_compact_ip(address: str) -> bytearray:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
if len(compact_ip) != 4:
|
||||
raise ValueError(f"invalid IPv4 length")
|
||||
return compact_ip
|
||||
|
||||
|
||||
def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
compact_ip = make_compact_ip(address)
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
|
||||
def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, int]:
|
||||
address = "{}.{}.{}.{}".format(*compact_address[:4])
|
||||
port = int.from_bytes(compact_address[4:6], 'big')
|
||||
node_id = compact_address[6:]
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return node_id, address, port
|
||||
@@ -10,7 +10,7 @@ class DuplicateStreamHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DownloadCanceledError(Exception):
|
||||
class DownloadCancelledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@@ -28,11 +28,16 @@ class DownloadTimeoutError(Exception):
|
||||
|
||||
class DownloadDataTimeout(Exception):
|
||||
def __init__(self, download):
|
||||
super().__init__('Failed to download data blobs for sd hash '
|
||||
'{} within timeout'.format(download))
|
||||
super().__init__(f'Failed to download data blobs for sd hash {download} within timeout')
|
||||
self.download = download
|
||||
|
||||
|
||||
class ResolveTimeout(Exception):
|
||||
def __init__(self, uri):
|
||||
super().__init__(f'Failed to resolve "{uri}" within the timeout')
|
||||
self.uri = uri
|
||||
|
||||
|
||||
class RequestCanceledError(Exception):
|
||||
pass
|
||||
|
||||
+201
-115
@@ -1,19 +1,20 @@
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
import logging
|
||||
import logging.handlers
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
import aiohttp
|
||||
from lbrynet.extras.compat import force_asyncioreactor_install
|
||||
force_asyncioreactor_install()
|
||||
from aiohttp.web import GracefulExit
|
||||
|
||||
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
|
||||
from lbrynet import __name__ as lbrynet_name, __version__ as lbrynet_version
|
||||
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbrynet.conf import Config, CLIConfig
|
||||
from lbrynet.utils import check_connection
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(lbrynet_name)
|
||||
@@ -24,7 +25,7 @@ def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
async def execute_command(conf, method, params, callback=display):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
@@ -32,55 +33,15 @@ async def execute_command(conf, method, params):
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
return callback(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
return callback(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
|
||||
|
||||
def print_help():
|
||||
print(dedent("""
|
||||
NAME
|
||||
lbrynet - LBRY command line client.
|
||||
|
||||
USAGE
|
||||
lbrynet [--data_dir=<blob and database directory>] [--wallet_dir=<wallet directory>]
|
||||
[--download_dir=<downloads directory>] <command> [<args>]
|
||||
|
||||
EXAMPLES
|
||||
lbrynet start # starts the daemon and listens for jsonrpc commands
|
||||
lbrynet help # display this message
|
||||
lbrynet help <command_name> # get help for a command(doesn't need the daemon to be running)
|
||||
lbrynet commands # list available commands
|
||||
lbrynet status # get the running status of the daemon
|
||||
lbrynet resolve what # resolve a name
|
||||
|
||||
lbrynet --wallet_dir=~/wallet2 start # start the daemon using an alternative wallet directory
|
||||
lbrynet --data_dir=~/lbry start # start the daemon using an alternative data directory
|
||||
|
||||
lbrynet --data_dir=~/lbry <command_name> # run a command on a daemon using an alternative data directory,
|
||||
# which can contain a full daemon_settings.yml config file.
|
||||
# Note: since the daemon is what runs the wallet and
|
||||
# downloads files, only the --data_dir setting is needed when
|
||||
# running commands. The wallet_dir and download_dir would only
|
||||
# by used when starting the daemon.
|
||||
"""))
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print(dedent(fn.__doc__))
|
||||
else:
|
||||
print("Invalid command name")
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
@@ -115,21 +76,113 @@ def set_kwargs(parsed_args):
|
||||
return kwargs
|
||||
|
||||
|
||||
def split_subparser_argument(parent, original, name, condition):
|
||||
new_sub_parser = argparse._SubParsersAction(
|
||||
original.option_strings,
|
||||
original._prog_prefix,
|
||||
original._parser_class,
|
||||
metavar=original.metavar
|
||||
)
|
||||
new_sub_parser._name_parser_map = original._name_parser_map
|
||||
new_sub_parser._choices_actions = [
|
||||
a for a in original._choices_actions if condition(original._name_parser_map[a.dest])
|
||||
]
|
||||
group = argparse._ArgumentGroup(parent, name)
|
||||
group._group_actions = [new_sub_parser]
|
||||
return group
|
||||
|
||||
|
||||
class ArgumentParser(argparse.ArgumentParser):
|
||||
def __init__(self, *args, group_name=None, **kwargs):
|
||||
super().__init__(*args, formatter_class=HelpFormatter, add_help=False, **kwargs)
|
||||
self.add_argument(
|
||||
'--help', dest='help', action='store_true', default=False,
|
||||
help='Show this help message and exit.'
|
||||
)
|
||||
self._optionals.title = 'Options'
|
||||
if group_name is None:
|
||||
self.epilog = (
|
||||
f"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
)
|
||||
else:
|
||||
self.epilog = (
|
||||
f"Run 'lbrynet {group_name} COMMAND --help' for more information on a command."
|
||||
)
|
||||
self.set_defaults(group=group_name, group_parser=self)
|
||||
|
||||
def format_help(self):
|
||||
formatter = self._get_formatter()
|
||||
formatter.add_usage(
|
||||
self.usage, self._actions, self._mutually_exclusive_groups
|
||||
)
|
||||
formatter.add_text(self.description)
|
||||
|
||||
# positionals, optionals and user-defined groups
|
||||
for action_group in self._granular_action_groups:
|
||||
formatter.start_section(action_group.title)
|
||||
formatter.add_text(action_group.description)
|
||||
formatter.add_arguments(action_group._group_actions)
|
||||
formatter.end_section()
|
||||
|
||||
formatter.add_text(self.epilog)
|
||||
return formatter.format_help()
|
||||
|
||||
@property
|
||||
def _granular_action_groups(self):
|
||||
if self.prog != 'lbrynet':
|
||||
yield from self._action_groups
|
||||
return
|
||||
yield self._optionals
|
||||
action: argparse._SubParsersAction = self._positionals._group_actions[0]
|
||||
yield split_subparser_argument(
|
||||
self, action, "Grouped Commands", lambda parser: 'group' in parser._defaults
|
||||
)
|
||||
yield split_subparser_argument(
|
||||
self, action, "Commands", lambda parser: 'group' not in parser._defaults
|
||||
)
|
||||
|
||||
def error(self, message):
|
||||
self.print_help(argparse._sys.stderr)
|
||||
self.exit(2, '\n'+message+'\n')
|
||||
|
||||
|
||||
class HelpFormatter(argparse.HelpFormatter):
|
||||
|
||||
def add_usage(self, usage, actions, groups, prefix='Usage: '):
|
||||
super().add_usage(
|
||||
usage, [a for a in actions if a.option_strings != ['--help']], groups, prefix
|
||||
)
|
||||
|
||||
|
||||
def add_command_parser(parent, command):
|
||||
subcommand = parent.add_parser(
|
||||
command['name'],
|
||||
help=command['doc'].strip().splitlines()[0]
|
||||
)
|
||||
subcommand.set_defaults(
|
||||
api_method_name=command['api_method_name'],
|
||||
command=command['name'],
|
||||
doc=command['doc'],
|
||||
replaced_by=command.get('replaced_by', None)
|
||||
)
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
main = argparse.ArgumentParser('lbrynet', add_help=False)
|
||||
main = ArgumentParser(
|
||||
'lbrynet', description='An interface to the LBRY Network.'
|
||||
)
|
||||
main.add_argument(
|
||||
'--version', dest='cli_version', action="store_true",
|
||||
'-v', '--version', dest='cli_version', action="store_true",
|
||||
help='Show lbrynet CLI version and exit.'
|
||||
)
|
||||
main.add_argument(
|
||||
'-h', '--help', dest='help', action="store_true",
|
||||
help='Show this help message and exit'
|
||||
main.set_defaults(group=None, command=None)
|
||||
CLIConfig.contribute_to_argparse(main)
|
||||
sub = main.add_subparsers(metavar='COMMAND')
|
||||
start = sub.add_parser(
|
||||
'start',
|
||||
usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...',
|
||||
help='Start LBRY Network interface.'
|
||||
)
|
||||
CLIConfig.contribute_args(main)
|
||||
sub = main.add_subparsers(dest='command')
|
||||
help = sub.add_parser('help', help='Detailed help for remote commands.')
|
||||
help.add_argument('help_command', nargs='*')
|
||||
start = sub.add_parser('start', help='Start lbrynet server.')
|
||||
start.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
@@ -139,28 +192,43 @@ def get_argument_parser():
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
Config.contribute_args(start)
|
||||
start.set_defaults(command='start', start_parser=start)
|
||||
Config.contribute_to_argparse(start)
|
||||
|
||||
api = Daemon.get_api_definitions()
|
||||
for group in sorted(api):
|
||||
group_command = sub.add_parser(group, help=api[group]['doc'])
|
||||
group_command.set_defaults(group_doc=group_command)
|
||||
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
continue
|
||||
commands = group_command.add_subparsers(dest='subcommand')
|
||||
for command in api[group]['commands']:
|
||||
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
|
||||
for deprecated in Daemon.deprecated_methods:
|
||||
group_command = sub.add_parser(deprecated)
|
||||
group_command.add_subparsers(dest='subcommand')
|
||||
groups = {}
|
||||
for group_name in sorted(api['groups']):
|
||||
group_parser = sub.add_parser(group_name, group_name=group_name, help=api['groups'][group_name])
|
||||
groups[group_name] = group_parser.add_subparsers(metavar='COMMAND')
|
||||
|
||||
nicer_order = ['stop', 'get', 'publish', 'resolve']
|
||||
for command_name in sorted(api['commands']):
|
||||
if command_name not in nicer_order:
|
||||
nicer_order.append(command_name)
|
||||
|
||||
for command_name in nicer_order:
|
||||
command = api['commands'][command_name]
|
||||
if command['group'] is None:
|
||||
add_command_parser(sub, command)
|
||||
else:
|
||||
add_command_parser(groups[command['group']], command)
|
||||
|
||||
return main
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
@@ -168,63 +236,81 @@ def main(argv=None):
|
||||
|
||||
elif args.command == 'start':
|
||||
|
||||
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
|
||||
if args.help:
|
||||
args.start_parser.print_help()
|
||||
return 0
|
||||
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
conf.log_file_path, maxBytes=2097152, backupCount=5
|
||||
)
|
||||
file_handler.setFormatter(default_formatter)
|
||||
log.addHandler(file_handler)
|
||||
logging.getLogger('torba').addHandler(file_handler)
|
||||
|
||||
if not args.quiet:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(default_formatter)
|
||||
log.addHandler(handler)
|
||||
logging.getLogger('torba').addHandler(handler)
|
||||
logging.getLogger('torba').setLevel(logging.INFO)
|
||||
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
loop.set_debug(True)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
log.debug('Final Settings: %s', conf.settings_dict)
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
daemon = Daemon(conf)
|
||||
|
||||
if check_connection():
|
||||
from twisted.internet import reactor
|
||||
reactor._asyncioEventloop.create_task(daemon.start())
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
elif args.command == 'help':
|
||||
|
||||
if args.help_command:
|
||||
method = '_'.join(args.help_command)
|
||||
else:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
if method not in Daemon.callable_methods:
|
||||
print('Invalid command name: {method}')
|
||||
return 1
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
print(fn.__doc__)
|
||||
started = False
|
||||
def __exit():
|
||||
if started:
|
||||
daemon.stop_event.set()
|
||||
else:
|
||||
raise GracefulExit()
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGINT, __exit)
|
||||
loop.add_signal_handler(signal.SIGTERM, __exit)
|
||||
except NotImplementedError:
|
||||
pass # Not implemented on Windows
|
||||
try:
|
||||
loop.run_until_complete(daemon.start())
|
||||
started = True
|
||||
loop.run_until_complete(daemon.stop_event.wait())
|
||||
except (GracefulExit, KeyboardInterrupt):
|
||||
pass
|
||||
finally:
|
||||
loop.run_until_complete(daemon.stop())
|
||||
if hasattr(loop, 'shutdown_asyncgens'):
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
|
||||
elif args.command is not None:
|
||||
|
||||
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
method = args.command
|
||||
elif args.subcommand is not None:
|
||||
method = f'{args.command}_{args.subcommand}'
|
||||
doc = args.doc
|
||||
api_method_name = args.api_method_name
|
||||
if args.replaced_by:
|
||||
print(f"{args.api_method_name} is deprecated, using {args.replaced_by['api_method_name']}.")
|
||||
doc = args.replaced_by['doc']
|
||||
api_method_name = args.replaced_by['api_method_name']
|
||||
|
||||
if args.help:
|
||||
print(doc)
|
||||
else:
|
||||
args.group_doc.print_help()
|
||||
return 0
|
||||
parsed = docopt(doc, command_args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(conf, api_method_name, params))
|
||||
|
||||
if method in Daemon.deprecated_methods:
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
return 0
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, command_args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(conf, method, params))
|
||||
elif args.group is not None:
|
||||
args.group_parser.print_help()
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import asyncio
|
||||
|
||||
|
||||
def force_asyncioreactor_install():
|
||||
import sys
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
from twisted.internet import defer
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
@@ -1,7 +1,5 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted._threads import AlreadyQuit
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
@@ -57,7 +55,7 @@ class Component(metaclass=ComponentType):
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
@@ -68,7 +66,7 @@ class Component(metaclass=ComponentType):
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error stopping %s", self.__class__.__name__)
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from lbrynet.p2p.Error import ComponentStartConditionNotMet
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
|
||||
import asyncio
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.error import ComponentStartConditionNotMet
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -35,16 +34,16 @@ class RequiredCondition(metaclass=RequiredConditionType):
|
||||
class ComponentManager:
|
||||
default_component_classes = {}
|
||||
|
||||
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None,
|
||||
peer_manager=None, peer_finder=None, **override_components):
|
||||
def __init__(self, conf: Config, analytics_manager=None, skip_components=None,
|
||||
peer_manager=None, **override_components):
|
||||
self.conf = conf
|
||||
self.skip_components = skip_components or []
|
||||
self.reactor = reactor
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.analytics_manager = analytics_manager
|
||||
self.component_classes = {}
|
||||
self.components = set()
|
||||
self.analytics_manager = analytics_manager
|
||||
self.peer_manager = peer_manager or PeerManager()
|
||||
self.peer_finder = peer_finder or DHTPeerFinder(self)
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop())
|
||||
|
||||
for component_name, component_class in self.default_component_classes.items():
|
||||
if component_name in override_components:
|
||||
@@ -57,7 +56,6 @@ class ComponentManager:
|
||||
|
||||
for component_class in self.component_classes.values():
|
||||
self.components.add(component_class(self))
|
||||
self.daemon = None
|
||||
|
||||
def evaluate_condition(self, condition_name):
|
||||
if condition_name not in RegisteredConditions.conditions:
|
||||
@@ -113,35 +111,19 @@ class ComponentManager:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
async def setup(self, **callbacks):
|
||||
async def start(self):
|
||||
""" Start Components in sequence sorted by requirements """
|
||||
for component_name, cb in callbacks.items():
|
||||
if component_name not in self.component_classes:
|
||||
if component_name not in self.skip_components:
|
||||
raise NameError("unknown component: %s" % component_name)
|
||||
if not callable(cb):
|
||||
raise ValueError("%s is not callable" % cb)
|
||||
|
||||
async def _setup(component):
|
||||
await component._setup()
|
||||
if component.component_name in callbacks:
|
||||
maybe_coro = callbacks[component.component_name](component)
|
||||
if asyncio.iscoroutine(maybe_coro):
|
||||
asyncio.create_task(maybe_coro)
|
||||
|
||||
stages = self.sort_components()
|
||||
for stage in stages:
|
||||
for stage in self.sort_components():
|
||||
needing_start = [
|
||||
_setup(component) for component in stage if not component.running
|
||||
component._setup() for component in stage if not component.running
|
||||
]
|
||||
if needing_start:
|
||||
await asyncio.wait(needing_start)
|
||||
self.started.set()
|
||||
|
||||
async def stop(self):
|
||||
"""
|
||||
Stop Components in reversed startup order
|
||||
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
stages = self.sort_components(reverse=True)
|
||||
for stage in stages:
|
||||
|
||||
+143
-260
@@ -1,40 +1,29 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import treq
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
from twisted.internet import defer, reactor, error
|
||||
import base58
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.p2p.RateLimiter import RateLimiter
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
|
||||
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
|
||||
from lbrynet.wallet import LbryWalletManager
|
||||
from lbrynet.wallet import Network
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -47,13 +36,10 @@ HEADERS_COMPONENT = "blockchain_headers"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
REFLECTOR_COMPONENT = "reflector"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
RATE_LIMITER_COMPONENT = "rate_limiter"
|
||||
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
|
||||
|
||||
|
||||
async def gather_dict(tasks: dict):
|
||||
@@ -64,17 +50,6 @@ async def gather_dict(tasks: dict):
|
||||
)))
|
||||
|
||||
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
@@ -88,11 +63,11 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 9
|
||||
return 11
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
return self.component_manager.daemon.db_revision_file_path
|
||||
return os.path.join(self.conf.data_dir, 'db_revision')
|
||||
|
||||
def _write_db_revision_file(self, version_num):
|
||||
with open(self.revision_filename, mode='w') as db_revision:
|
||||
@@ -118,7 +93,7 @@ class DatabaseComponent(Component):
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
|
||||
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
@@ -158,39 +133,32 @@ class HeadersComponent(Component):
|
||||
'download_progress': self._headers_progress_percent
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def fetch_headers_from_s3(self):
|
||||
def collector(data, h_file):
|
||||
h_file.write(data)
|
||||
local_size = float(h_file.tell())
|
||||
final_size = float(final_size_after_download)
|
||||
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
response = yield treq.get(HEADERS_URL, headers=resume_header)
|
||||
got_406 = response.code == 406 # our file is bigger
|
||||
final_size_after_download = response.length + local_header_size
|
||||
if got_406:
|
||||
log.warning("s3 is more out of date than we are")
|
||||
# should have something to download and a final length divisible by the header size
|
||||
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
|
||||
s3_height = (final_size_after_download / HEADER_SIZE) - 1
|
||||
local_height = self.local_header_file_height()
|
||||
if s3_height > local_height:
|
||||
if local_header_size:
|
||||
log.info("Resuming download of %i bytes from s3", response.length)
|
||||
with open(self.headers_file, "a+b") as headers_file:
|
||||
yield treq.collect(response, lambda d: collector(d, headers_file))
|
||||
else:
|
||||
with open(self.headers_file, "wb") as headers_file:
|
||||
yield treq.collect(response, lambda d: collector(d, headers_file))
|
||||
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
|
||||
self._check_header_file_integrity()
|
||||
else:
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
else:
|
||||
log.error("invalid size for headers from s3")
|
||||
return
|
||||
if response.content_length % HEADER_SIZE != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while True:
|
||||
chunk = await response.content.read(512)
|
||||
if not chunk:
|
||||
break
|
||||
fd.write(chunk)
|
||||
self._headers_progress_percent = math.ceil(
|
||||
float(fd.tell()) / float(final_size_after_download) * 100
|
||||
)
|
||||
log.info("fetched headers from s3, now verifying integrity after download.")
|
||||
self._check_header_file_integrity()
|
||||
|
||||
def local_header_file_height(self):
|
||||
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
|
||||
@@ -259,7 +227,7 @@ class HeadersComponent(Component):
|
||||
self._downloading_headers = await self.should_download_headers_from_s3()
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
await d2f(self.fetch_headers_from_s3())
|
||||
await self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
@@ -283,23 +251,20 @@ class WalletComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
local_height = self.wallet_manager.ledger.headers.height
|
||||
remote_height = await self.wallet_manager.ledger.network.get_server_height()
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet_manager.wallet.use_encryption,
|
||||
'is_encrypted': self.wallet_manager.use_encryption,
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
|
||||
await self.wallet_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
@@ -313,29 +278,32 @@ class BlobComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager = None
|
||||
self.blob_manager: BlobManager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
def component(self) -> typing.Optional[BlobManager]:
|
||||
return self.blob_manager
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
datastore = None
|
||||
data_store = None
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
datastore = dht_node._dataStore
|
||||
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore)
|
||||
return self.blob_manager.setup()
|
||||
data_store = dht_node.protocol.data_store
|
||||
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
|
||||
if not os.path.isdir(blob_dir):
|
||||
os.mkdir(blob_dir)
|
||||
self.blob_manager = BlobManager(asyncio.get_event_loop(), blob_dir, storage, self.conf, data_store)
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
def stop(self):
|
||||
return self.blob_manager.stop()
|
||||
async def stop(self):
|
||||
self.blob_manager.stop()
|
||||
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = await self.blob_manager.storage.count_finished_blobs()
|
||||
count = len(self.blob_manager.completed_blob_hashes)
|
||||
return {'finished_blobs': count}
|
||||
|
||||
|
||||
@@ -345,48 +313,61 @@ class DHTComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.dht_node = None
|
||||
self.dht_node: Node = None
|
||||
self.upnp_component = None
|
||||
self.external_udp_port = None
|
||||
self.external_peer_port = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
def component(self) -> typing.Optional[Node]:
|
||||
return self.dht_node
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
|
||||
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
|
||||
}
|
||||
|
||||
def get_node_id(self):
|
||||
node_id_filename = os.path.join(self.conf.data_dir, "node_id")
|
||||
if os.path.isfile(node_id_filename):
|
||||
with open(node_id_filename, "r") as node_id_file:
|
||||
return base58.b58decode(str(node_id_file.read()).strip())
|
||||
node_id = utils.generate_id()
|
||||
with open(node_id_filename, "w") as node_id_file:
|
||||
node_id_file.write(base58.b58encode(node_id).decode())
|
||||
return node_id
|
||||
|
||||
async def start(self):
|
||||
log.info("start the dht")
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
node_id = self.component_manager.daemon.node_id
|
||||
if node_id is None:
|
||||
node_id = generate_id()
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = await get_external_ip()
|
||||
external_ip = await utils.get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
self.dht_node = Node(
|
||||
node_id=node_id,
|
||||
udpPort=self.conf.dht_node_port,
|
||||
externalUDPPort=self.external_udp_port,
|
||||
externalIP=external_ip,
|
||||
peerPort=self.external_peer_port
|
||||
asyncio.get_event_loop(),
|
||||
self.component_manager.peer_manager,
|
||||
node_id=self.get_node_id(),
|
||||
internal_udp_port=self.conf.udp_port,
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
)
|
||||
|
||||
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
|
||||
log.info("Started the dht")
|
||||
|
||||
def stop(self):
|
||||
return d2f(self.dht_node.stop())
|
||||
async def stop(self):
|
||||
self.dht_node.stop()
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
@@ -395,195 +376,94 @@ class HashAnnouncerComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.hash_announcer = None
|
||||
self.hash_announcer: BlobAnnouncer = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
def component(self) -> typing.Optional[BlobAnnouncer]:
|
||||
return self.hash_announcer
|
||||
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage)
|
||||
self.hash_announcer.start()
|
||||
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
|
||||
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
|
||||
log.info("Started blob announcer")
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
self.hash_announcer.stop()
|
||||
log.info("Stopped blob announcer")
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.announce_queue)
|
||||
}
|
||||
|
||||
|
||||
class RateLimiterComponent(Component):
|
||||
component_name = RATE_LIMITER_COMPONENT
|
||||
class StreamManagerComponent(Component):
|
||||
component_name = STREAM_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.rate_limiter = RateLimiter()
|
||||
self.stream_manager: StreamManager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.rate_limiter
|
||||
|
||||
async def start(self):
|
||||
self.rate_limiter.start()
|
||||
|
||||
async def stop(self):
|
||||
self.rate_limiter.stop()
|
||||
|
||||
|
||||
class PaymentRateComponent(Component):
|
||||
component_name = PAYMENT_RATE_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.payment_rate_manager = OnlyFreePaymentsManager()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.payment_rate_manager
|
||||
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
|
||||
PAYMENT_RATE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.file_manager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.file_manager
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
return self.stream_manager
|
||||
|
||||
async def get_status(self):
|
||||
if not self.file_manager:
|
||||
if not self.stream_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.lbry_files)
|
||||
'managed_files': len(self.stream_manager.streams)
|
||||
}
|
||||
|
||||
def start(self):
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
async def start(self):
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
|
||||
sd_identifier = StreamDescriptorIdentifier()
|
||||
add_lbry_file_to_sd_identifier(sd_identifier)
|
||||
file_saver_factory = EncryptedFileSaverFactory(
|
||||
self.conf,
|
||||
self.component_manager.peer_finder,
|
||||
rate_limiter,
|
||||
blob_manager,
|
||||
storage,
|
||||
wallet,
|
||||
self.conf.download_dir
|
||||
)
|
||||
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
|
||||
try:
|
||||
node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
except NameError:
|
||||
node = None
|
||||
log.info('Starting the file manager')
|
||||
self.file_manager = EncryptedFileManager(
|
||||
self.conf, self.component_manager.peer_finder, rate_limiter,
|
||||
blob_manager, wallet, payment_rate_manager, storage, sd_identifier
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
)
|
||||
return self.file_manager.setup()
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
def stop(self):
|
||||
return d2f(self.file_manager.stop())
|
||||
async def stop(self):
|
||||
self.stream_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
component_name = PEER_PROTOCOL_SERVER_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
|
||||
PAYMENT_RATE_COMPONENT]
|
||||
depends_on = [UPNP_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.lbry_server_port = None
|
||||
self.blob_server: BlobServer = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.lbry_server_port
|
||||
def component(self) -> typing.Optional[BlobServer]:
|
||||
return self.blob_server
|
||||
|
||||
async def start(self):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
log.info("start blob server")
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
peer_port = self.conf.peer_port
|
||||
query_handlers = {
|
||||
handler.get_primary_query_identifier(): handler for handler in [
|
||||
BlobRequestHandlerFactory(
|
||||
self.component_manager.get_component(BLOB_COMPONENT),
|
||||
wallet,
|
||||
self.component_manager.get_component(PAYMENT_RATE_COMPONENT),
|
||||
self.component_manager.analytics_manager
|
||||
),
|
||||
wallet.get_wallet_info_query_handler_factory(),
|
||||
]
|
||||
}
|
||||
server_factory = ServerProtocolFactory(
|
||||
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
|
||||
self.component_manager.peer_manager
|
||||
)
|
||||
|
||||
try:
|
||||
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
|
||||
upnp.upnp_redirects.get("TCP", peer_port))
|
||||
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
|
||||
" more details.", peer_port)
|
||||
log.error("%s", traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
|
||||
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
|
||||
await self.blob_server.started_listening.wait()
|
||||
|
||||
async def stop(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, old_port = None, self.lbry_server_port
|
||||
log.info('Stop listening on port %s', old_port.port)
|
||||
await d2f(old_port.stopListening())
|
||||
|
||||
|
||||
class ReflectorComponent(Component):
|
||||
component_name = REFLECTOR_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.reflector_server_port = self.conf.reflector_port
|
||||
self.reflector_server = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.reflector_server
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting reflector server")
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
|
||||
try:
|
||||
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
|
||||
log.info('Started reflector on port %s', self.reflector_server_port)
|
||||
except error.CannotListenError as e:
|
||||
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
|
||||
raise ValueError(f"{e} lbrynet may already be running on your computer.")
|
||||
|
||||
async def stop(self):
|
||||
if self.reflector_server is not None:
|
||||
log.info("Stopping reflector server")
|
||||
self.reflector_server, p = None, self.reflector_server
|
||||
await d2f(p.stopListening())
|
||||
if self.blob_server:
|
||||
self.blob_server.stop_server()
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
@@ -591,8 +471,8 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self._int_peer_port = self.conf.tcp_port
|
||||
self._int_dht_node_port = self.conf.udp_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
@@ -600,14 +480,14 @@ class UPnPComponent(Component):
|
||||
self._maintain_redirects_task = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
def component(self) -> 'UPnPComponent':
|
||||
return self
|
||||
|
||||
async def _repeatedly_maintain_redirects(self, now=True):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
await asyncio.sleep(360, loop=self.component_manager.loop)
|
||||
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
@@ -631,7 +511,7 @@ class UPnPComponent(Component):
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = await get_external_ip()
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
@@ -650,7 +530,7 @@ class UPnPComponent(Component):
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
self.upnp = None
|
||||
return self._maintain_redirects()
|
||||
return
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = await self.upnp.get_redirects()
|
||||
@@ -682,7 +562,7 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await get_external_ip()
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
@@ -696,7 +576,10 @@ class UPnPComponent(Component):
|
||||
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
if self.component_manager.analytics_manager:
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
async def stop(self):
|
||||
@@ -704,7 +587,7 @@ class UPnPComponent(Component):
|
||||
await asyncio.wait([
|
||||
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
|
||||
])
|
||||
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done():
|
||||
if self._maintain_redirects_task and not self._maintain_redirects_task.done():
|
||||
self._maintain_redirects_task.cancel()
|
||||
|
||||
async def get_status(self):
|
||||
@@ -722,11 +605,11 @@ class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
super().__init__(component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
def component(self) -> ExchangeRateManager:
|
||||
return self.exchange_rate_manager
|
||||
|
||||
async def start(self):
|
||||
|
||||
+1987
-2082
File diff suppressed because it is too large.
Load diff
@@ -1,209 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
|
||||
from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError
|
||||
from lbrynet.p2p.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from torba.client.constants import COIN
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
INITIALIZING_CODE = 'initializing'
|
||||
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
|
||||
DOWNLOAD_TIMEOUT_CODE = 'timeout'
|
||||
DOWNLOAD_RUNNING_CODE = 'running'
|
||||
DOWNLOAD_STOPPED_CODE = 'stopped'
|
||||
STREAM_STAGES = [
|
||||
(INITIALIZING_CODE, 'Initializing'),
|
||||
(DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
|
||||
(DOWNLOAD_RUNNING_CODE, 'Started stream'),
|
||||
(DOWNLOAD_STOPPED_CODE, 'Paused stream'),
|
||||
(DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
|
||||
]
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GetStream:
|
||||
def __init__(self, conf: Config, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None,
|
||||
timeout=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.conf = conf
|
||||
self.reactor = reactor
|
||||
self.timeout = timeout or conf.download_timeout
|
||||
self.data_rate = data_rate or conf.data_rate
|
||||
self.max_key_fee = max_key_fee or conf.max_key_fee
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.disable_max_key_fee
|
||||
self.download_directory = conf.download_dir
|
||||
self.timeout_counter = 0
|
||||
self.code = None
|
||||
self.sd_hash = None
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.wallet = wallet
|
||||
self.exchange_rate_manager = exchange_rate_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.storage = storage
|
||||
self.downloader = None
|
||||
|
||||
# fired when the download is complete
|
||||
self.finished_deferred = None
|
||||
# fired after the metadata and the first data blob have been downloaded
|
||||
self.data_downloading_deferred = defer.Deferred(None)
|
||||
self.wrote_data = False
|
||||
|
||||
@property
|
||||
def download_path(self):
|
||||
return os.path.join(self.download_directory, self.downloader.file_name)
|
||||
|
||||
def convert_max_fee(self):
|
||||
currency, amount = self.max_key_fee['currency'], self.max_key_fee['amount']
|
||||
return self.exchange_rate_manager.convert_currency(currency, "LBC", amount)
|
||||
|
||||
def set_status(self, status, name):
|
||||
log.info("Download lbry://%s status changed to %s" % (name, status))
|
||||
self.code = next(s for s in STREAM_STAGES if s[0] == status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def check_fee_and_convert(self, fee):
|
||||
max_key_fee_amount = self.convert_max_fee()
|
||||
converted_fee_amount = self.exchange_rate_manager.convert_currency(fee.currency, "LBC",
|
||||
fee.amount)
|
||||
if converted_fee_amount > (yield f2d(self.wallet.default_account.get_balance())):
|
||||
raise InsufficientFundsError('Unable to pay the key fee of %s' % converted_fee_amount)
|
||||
if converted_fee_amount > max_key_fee_amount and not self.disable_max_key_fee:
|
||||
raise KeyFeeAboveMaxAllowed('Key fee {} above max allowed {}'.format(converted_fee_amount,
|
||||
max_key_fee_amount))
|
||||
converted_fee = {
|
||||
'currency': 'LBC',
|
||||
'amount': converted_fee_amount,
|
||||
'address': fee.address
|
||||
}
|
||||
return Fee(converted_fee)
|
||||
|
||||
def get_downloader_factory(self, factories):
|
||||
for factory in factories:
|
||||
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
|
||||
return factory
|
||||
raise Exception(f'No suitable factory was found in {factories}')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_downloader(self, factory, stream_metadata, file_name=None):
|
||||
# TODO: we should use stream_metadata.options.get_downloader_options
|
||||
# instead of hard-coding the options to be [self.data_rate]
|
||||
downloader = yield factory.make_downloader(
|
||||
stream_metadata,
|
||||
self.data_rate,
|
||||
self.payment_rate_manager,
|
||||
self.download_directory,
|
||||
file_name=file_name
|
||||
)
|
||||
defer.returnValue(downloader)
|
||||
|
||||
def _pay_key_fee(self, address, fee_lbc, name):
|
||||
log.info("Pay key fee %s --> %s", dewies_to_lbc(fee_lbc), address)
|
||||
reserved_points = self.wallet.reserve_points(address, fee_lbc)
|
||||
if reserved_points is None:
|
||||
raise InsufficientFundsError(
|
||||
'Unable to pay the key fee of {} for {}'.format(dewies_to_lbc(fee_lbc), name)
|
||||
)
|
||||
return f2d(self.wallet.send_points_to_address(reserved_points, fee_lbc))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def pay_key_fee(self, fee, name):
|
||||
if fee is not None:
|
||||
yield self._pay_key_fee(fee.address.decode(), int(fee.amount * COIN), name)
|
||||
else:
|
||||
defer.returnValue(None)
|
||||
|
||||
def finish(self, results, name):
|
||||
self.set_status(DOWNLOAD_STOPPED_CODE, name)
|
||||
log.info("Finished downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
|
||||
self.download_path)
|
||||
return defer.succeed(self.download_path)
|
||||
|
||||
def fail(self, err):
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _initialize(self, stream_info):
|
||||
# Set sd_hash and return key_fee from stream_info
|
||||
self.sd_hash = stream_info.source_hash.decode()
|
||||
key_fee = None
|
||||
if stream_info.has_fee:
|
||||
key_fee = yield self.check_fee_and_convert(stream_info.source_fee)
|
||||
defer.returnValue(key_fee)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _create_downloader(self, sd_blob, file_name=None):
|
||||
stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
|
||||
factory = self.get_downloader_factory(stream_metadata.factories)
|
||||
downloader = yield self.get_downloader(factory, stream_metadata, file_name)
|
||||
defer.returnValue(downloader)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _download_sd_blob(self):
|
||||
sd_blob = yield download_sd_blob(
|
||||
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, self.conf.download_mirrors
|
||||
)
|
||||
defer.returnValue(sd_blob)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
|
||||
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
|
||||
yield self.pay_key_fee(key_fee, name)
|
||||
yield f2d(self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout)))
|
||||
self.finished_deferred = self.downloader.start()
|
||||
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
|
||||
self.data_downloading_deferred.callback
|
||||
)
|
||||
self.finished_deferred.addCallbacks(lambda result: self.finish(result, name), self.fail)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, stream_info, name, txid, nout, file_name=None):
|
||||
"""
|
||||
Start download
|
||||
|
||||
Returns:
|
||||
(tuple) Tuple containing (downloader, finished_deferred)
|
||||
|
||||
downloader - instance of ManagedEncryptedFileDownloader
|
||||
finished_deferred - deferred callbacked when download is finished
|
||||
"""
|
||||
self.set_status(INITIALIZING_CODE, name)
|
||||
key_fee = yield self._initialize(stream_info)
|
||||
self.set_status(DOWNLOAD_METADATA_CODE, name)
|
||||
try:
|
||||
sd_blob = yield self._download_sd_blob()
|
||||
yield self._download(sd_blob, name, key_fee, txid, nout, file_name)
|
||||
self.set_status(DOWNLOAD_RUNNING_CODE, name)
|
||||
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
|
||||
self.data_downloading_deferred.addTimeout(self.timeout, self.reactor)
|
||||
try:
|
||||
yield self.data_downloading_deferred
|
||||
self.wrote_data = True
|
||||
except defer.TimeoutError:
|
||||
raise DownloadDataTimeout("data download timed out")
|
||||
except (DownloadDataTimeout, InvalidStreamDescriptorError) as err:
|
||||
raise err
|
||||
|
||||
defer.returnValue((self.downloader, self.finished_deferred))
|
||||
|
||||
def cancel(self, reason=None):
|
||||
if reason:
|
||||
msg = "download stream cancelled: %s" % reason
|
||||
else:
|
||||
msg = "download stream cancelled"
|
||||
if self.data_downloading_deferred and not self.data_downloading_deferred.called:
|
||||
self.data_downloading_deferred.errback(DownloadCanceledError(msg))
|
||||
@@ -1,92 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer, task
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTHashAnnouncer:
|
||||
def __init__(self, conf: Config, dht_node, storage, concurrent_announcers=None):
|
||||
self.conf = conf
|
||||
self.dht_node = dht_node
|
||||
self.storage = storage
|
||||
self.clock = dht_node.clock
|
||||
self.peer_port = dht_node.peerPort
|
||||
self.hash_queue = []
|
||||
if concurrent_announcers is None:
|
||||
self.concurrent_announcers = conf.concurrent_announcers
|
||||
else:
|
||||
self.concurrent_announcers = concurrent_announcers
|
||||
self._manage_lc = None
|
||||
if self.concurrent_announcers:
|
||||
self._manage_lc = task.LoopingCall(self.manage)
|
||||
self._manage_lc.clock = self.clock
|
||||
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.concurrent_announcers or 1)
|
||||
|
||||
def start(self):
|
||||
if self._manage_lc:
|
||||
self._manage_lc.start(30)
|
||||
|
||||
def stop(self):
|
||||
if self._manage_lc and self._manage_lc.running:
|
||||
self._manage_lc.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def do_store(self, blob_hash):
|
||||
storing_node_ids = yield self.dht_node.announceHaveBlob(binascii.unhexlify(blob_hash))
|
||||
now = self.clock.seconds()
|
||||
if storing_node_ids:
|
||||
result = (now, storing_node_ids)
|
||||
yield f2d(self.storage.update_last_announced_blob(blob_hash, now))
|
||||
log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids))
|
||||
else:
|
||||
result = (None, [])
|
||||
self.hash_queue.remove(blob_hash)
|
||||
defer.returnValue(result)
|
||||
|
||||
def hash_queue_size(self):
|
||||
return len(self.hash_queue)
|
||||
|
||||
def _show_announce_progress(self, size, start):
|
||||
queue_size = len(self.hash_queue)
|
||||
average_blobs_per_second = float(size - queue_size) / (self.clock.seconds() - start)
|
||||
log.info("Announced %i/%i blobs, %f blobs per second", size - queue_size, size, average_blobs_per_second)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def immediate_announce(self, blob_hashes):
|
||||
self.hash_queue.extend(b for b in blob_hashes if b not in self.hash_queue)
|
||||
log.info("Announcing %i blobs", len(self.hash_queue))
|
||||
start = self.clock.seconds()
|
||||
progress_lc = task.LoopingCall(self._show_announce_progress, len(self.hash_queue), start)
|
||||
progress_lc.clock = self.clock
|
||||
progress_lc.start(60, now=False)
|
||||
results = yield utils.DeferredDict(
|
||||
{blob_hash: self.sem.run(self.do_store, blob_hash) for blob_hash in blob_hashes}
|
||||
)
|
||||
now = self.clock.seconds()
|
||||
|
||||
progress_lc.stop()
|
||||
|
||||
announced_to = [blob_hash for blob_hash in results if results[blob_hash][0]]
|
||||
if len(announced_to) != len(results):
|
||||
log.debug("Failed to announce %i blobs", len(results) - len(announced_to))
|
||||
if announced_to:
|
||||
log.info('Took %s seconds to announce %i of %i attempted hashes (%f hashes per second)',
|
||||
now - start, len(announced_to), len(blob_hashes),
|
||||
int(float(len(blob_hashes)) / float(now - start)))
|
||||
defer.returnValue(results)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def manage(self):
|
||||
if not self.dht_node.contacts:
|
||||
log.info("Not ready to start announcing hashes")
|
||||
return
|
||||
need_reannouncement = yield f2d(self.storage.get_blobs_to_announce())
|
||||
if need_reannouncement:
|
||||
yield self.immediate_announce(need_reannouncement)
|
||||
else:
|
||||
log.debug("Nothing to announce")
|
||||
@@ -1,70 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyPeerFinder:
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
return defer.succeed([])
|
||||
|
||||
|
||||
class DHTPeerFinder(DummyPeerFinder):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
#implements(IPeerFinder)
|
||||
|
||||
def __init__(self, component_manager):
|
||||
"""
|
||||
component_manager - an instance of ComponentManager
|
||||
"""
|
||||
self.component_manager = component_manager
|
||||
self.peer_manager = component_manager.peer_manager
|
||||
self.peers = {}
|
||||
self._ongoing_searchs = {}
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
"""
|
||||
Find peers for blob in the DHT
|
||||
blob_hash (str): blob hash to look for
|
||||
timeout (int): seconds to timeout after
|
||||
filter_self (bool): if True, and if a peer for a blob is itself, filter it
|
||||
from the result
|
||||
|
||||
Returns:
|
||||
list of peers for the blob
|
||||
"""
|
||||
if "dht" in self.component_manager.skip_components:
|
||||
return defer.succeed([])
|
||||
if not self.component_manager.all_components_running("dht"):
|
||||
return defer.succeed([])
|
||||
dht_node = self.component_manager.get_component("dht")
|
||||
|
||||
self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)})
|
||||
if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called:
|
||||
self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout)
|
||||
|
||||
def _filter_self(blob_hash):
|
||||
my_host, my_port = dht_node.externalIP, dht_node.peerPort
|
||||
return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}
|
||||
|
||||
peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash])
|
||||
return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _execute_peer_search(self, dht_node, blob_hash, timeout):
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
|
||||
timeout = timeout or self.component_manager.conf.peer_search_timeout
|
||||
if timeout:
|
||||
finished_deferred.addTimeout(timeout, dht_node.clock)
|
||||
try:
|
||||
peer_list = yield finished_deferred
|
||||
self.peers[blob_hash].update({(host, port) for _, host, port in peer_list})
|
||||
except defer.TimeoutError:
|
||||
log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout)
|
||||
finally:
|
||||
del self._ongoing_searchs[blob_hash]
|
||||
@@ -1,14 +0,0 @@
|
||||
from lbrynet.p2p.Peer import Peer
|
||||
|
||||
|
||||
class PeerManager:
|
||||
def __init__(self):
|
||||
self.peers = []
|
||||
|
||||
def get_peer(self, host, port):
|
||||
for peer in self.peers:
|
||||
if peer.host == host and peer.port == port:
|
||||
return peer
|
||||
peer = Peer(host, port)
|
||||
self.peers.append(peer)
|
||||
return peer
|
||||
@@ -1,77 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
|
||||
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def d2f(d):
|
||||
return d.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
class Publisher:
|
||||
def __init__(self, account, blob_manager, payment_rate_manager, storage,
|
||||
lbry_file_manager, wallet, certificate):
|
||||
self.account = account
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.storage = storage
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.wallet = wallet
|
||||
self.certificate = certificate
|
||||
self.lbry_file = None
|
||||
|
||||
async def create_and_publish_stream(self, name, bid, claim_dict, file_path, holding_address=None):
|
||||
"""Create lbry file and make claim"""
|
||||
log.info('Starting publish for %s', name)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File {file_path} not found")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
|
||||
file_name = os.path.basename(file_path)
|
||||
with open(file_path, 'rb') as read_handle:
|
||||
self.lbry_file = await d2f(create_lbry_file(
|
||||
self.blob_manager, self.storage, self.payment_rate_manager, self.lbry_file_manager, file_name,
|
||||
read_handle
|
||||
))
|
||||
|
||||
if 'source' not in claim_dict['stream']:
|
||||
claim_dict['stream']['source'] = {}
|
||||
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
tx = await self.wallet.claim_name(
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
|
||||
# check if we have a file already for this claim (if this is a publish update with a new stream)
|
||||
old_stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(
|
||||
tx.outputs[0].claim_id, self.lbry_file.stream_hash
|
||||
)
|
||||
if old_stream_hashes:
|
||||
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
|
||||
list(self.lbry_file_manager.lbry_files)):
|
||||
await d2f(self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False))
|
||||
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
|
||||
|
||||
await self.storage.save_content_claim(
|
||||
self.lbry_file.stream_hash, tx.outputs[0].id
|
||||
)
|
||||
return tx
|
||||
|
||||
async def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
|
||||
"""Make a claim without creating a lbry file"""
|
||||
tx = await self.wallet.claim_name(
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
|
||||
return tx
|
||||
+114
-145
@@ -1,13 +1,15 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
|
||||
import typing
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
@@ -19,103 +21,137 @@ HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
NEW_DOWNLOAD_STAT = 'Download'
|
||||
UPNP_SETUP = "UPnP Setup"
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
|
||||
TIME_TO_FIRST_BYTES = "Time To First Bytes"
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
def _event_properties(installation_id: str, session_id: str,
|
||||
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
properties = {
|
||||
'lbry_id': installation_id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
|
||||
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
|
||||
total_duration: typing.Optional[float], download_id: str, name: str,
|
||||
outpoint: str, active_peer_count: int, tried_peers_count: int,
|
||||
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_download_duration: typing.Optional[float] = None,
|
||||
error: typing.Optional[str] = None) -> typing.Dict:
|
||||
return {
|
||||
"external_ip": external_ip,
|
||||
"download_id": download_id,
|
||||
"total_duration": round(total_duration, 4),
|
||||
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
|
||||
"error": error,
|
||||
'name': name,
|
||||
"outpoint": outpoint,
|
||||
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
"tried_peers_count": tried_peers_count,
|
||||
|
||||
"sd_blob_hash": sd_hash,
|
||||
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
|
||||
|
||||
"head_blob_hash": head_blob_hash,
|
||||
"head_blob_length": head_blob_length,
|
||||
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4)
|
||||
}
|
||||
|
||||
|
||||
def _make_context(platform):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
|
||||
class AnalyticsManager:
|
||||
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.conf = conf
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = self._make_context(system_info.get_platform(), 'torba')
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: asyncio.Task = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.task = asyncio.create_task(self.run())
|
||||
log.info("Start")
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
async def _post(self, data: typing.Dict):
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + endpoint,
|
||||
'url': self.url + '/track',
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
async with utils.aiohttp_request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
|
||||
|
||||
async def track(self, event):
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post('/track', event)
|
||||
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
|
||||
'action': action,
|
||||
'error': e,
|
||||
}),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
await self._post(event)
|
||||
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
@@ -134,20 +170,33 @@ class Manager:
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
|
||||
total_duration: typing.Optional[float], download_id: str,
|
||||
name: str, outpoint: str, found_peers_count: int,
|
||||
tried_peers_count: int, added_fixed_peers: bool,
|
||||
fixed_peers_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_duration: typing.Optional[int] = None,
|
||||
error: typing.Optional[str] = None):
|
||||
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
|
||||
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
found_peers_count, tried_peers_count, added_fixed_peers, fixed_peers_delay, sd_hash,
|
||||
sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error
|
||||
)))
|
||||
|
||||
async def send_download_finished(self, download_id, name, sd_hash):
|
||||
await self.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
self._event(
|
||||
DOWNLOAD_FINISHED, {
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
@@ -160,91 +209,11 @@ class Manager:
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
async def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self._tracked_data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'properties': _event_properties(self.installation_id, self.session_id, properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash.decode(),
|
||||
'error': error_name(error),
|
||||
'reason': str(error),
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
@@ -0,0 +1,6 @@
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.cli import execute_command
|
||||
|
||||
|
||||
def daemon_rpc(conf: Config, method: str, **kwargs):
|
||||
return execute_command(conf, method, kwargs, callback=lambda data: data)
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
import aiohttp
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpc_body(method: str, rpc_id: any, **params) -> dict:
|
||||
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
clean = params.pop('clean', True)
|
||||
response = (await jsonrpc_batch(url, [rpc_body(method, 1, **params)]))[0]
|
||||
if clean:
|
||||
if 'error' in response:
|
||||
return response['error']
|
||||
return response['result']
|
||||
else:
|
||||
return response
|
||||
|
||||
|
||||
async def jsonrpc_batch(url: str, calls: list, batch_size: int = 50, clean: bool = False) -> list:
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
complete = []
|
||||
batch_size = max(batch_size, 50)
|
||||
for i in range(0, len(calls), batch_size):
|
||||
async with aiohttp.request('POST', url, headers=headers, json=calls[i:i+batch_size]) as response:
|
||||
complete += await response.json()
|
||||
if clean:
|
||||
complete = [body['result'] if 'result' in body else None for body in complete]
|
||||
return complete
|
||||
@@ -1,10 +1,11 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbrynet.utils import aiohttp_request
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,8 +53,8 @@ class MarketFeed:
|
||||
def is_online(self):
|
||||
return self._online
|
||||
|
||||
async def _make_request(self):
|
||||
async with aiohttp.request('get', self.url, params=self.params) as response:
|
||||
async def _make_request(self) -> str:
|
||||
async with aiohttp_request('get', self.url, params=self.params) as response:
|
||||
return (await response.read()).decode()
|
||||
|
||||
def _handle_response(self, response):
|
||||
@@ -219,19 +220,19 @@ class ExchangeRateManager:
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
log.debug("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
if from_currency == to_currency:
|
||||
return amount
|
||||
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return amount * market.rate.spot
|
||||
return amount * Decimal(market.rate.spot)
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot))
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
|
||||
@@ -4,25 +4,118 @@ from binascii import hexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
from ecdsa import BadSignatureError
|
||||
from lbrynet.extras.wallet import MainNetLedger
|
||||
from lbrynet.extras.wallet.transaction import Transaction, Output
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.wallet.ledger import MainNetLedger, Account
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_txo_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'nout': "position in the transaction",
|
||||
'height': "block where transaction was recorded",
|
||||
'amount': "value of the txo as a decimal",
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'payment'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim' or 'support', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
}
|
||||
|
||||
|
||||
def encode_tx_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'height': "block where transaction was recorded",
|
||||
'inputs': [encode_txo_doc()],
|
||||
'outputs': [encode_txo_doc()],
|
||||
'total_input': "sum of inputs as a decimal",
|
||||
'total_output': "sum of outputs, sans fee, as a decimal",
|
||||
'total_fee': "fee amount",
|
||||
'hex': "entire transaction encoded in hex",
|
||||
}
|
||||
|
||||
|
||||
def encode_account_doc():
|
||||
return {
|
||||
'id': 'account_id',
|
||||
'is_default': 'this account is used by default',
|
||||
'ledger': 'name of crypto currency and network',
|
||||
'name': 'optional account name',
|
||||
'seed': 'human friendly words from which account can be recreated',
|
||||
'encrypted': 'if account is encrypted',
|
||||
'private_key': 'extended private key',
|
||||
'public_key': 'extended public key',
|
||||
'address_generator': 'settings for generating addresses',
|
||||
'modified_on': 'date of last modification to account settings'
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'streaming_url': '(str) url to stream the file using range requests',
|
||||
'completed': '(bool) true if download is completed',
|
||||
'file_name': '(str) name of file',
|
||||
'download_directory': '(str) download directory',
|
||||
'points_paid': '(float) credit paid to download file',
|
||||
'stopped': '(bool) true if download is stopped',
|
||||
'stream_hash': '(str) stream hash of file',
|
||||
'stream_name': '(str) stream name',
|
||||
'suggested_file_name': '(str) suggested file name',
|
||||
'sd_hash': '(str) sd hash of file',
|
||||
'download_path': '(str) download path of file',
|
||||
'mime_type': '(str) mime type of file',
|
||||
'key': '(str) key attached to file',
|
||||
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
|
||||
'total_bytes': '(int) file upper bound size in bytes',
|
||||
'written_bytes': '(int) written size in bytes',
|
||||
'blobs_completed': '(int) number of fully downloaded blobs',
|
||||
'blobs_in_stream': '(int) total blobs on stream',
|
||||
'blobs_remaining': '(int) total blobs remaining to download',
|
||||
'status': '(str) downloader status',
|
||||
'claim_id': '(str) None if claim is not found else the claim id',
|
||||
'txid': '(str) None if claim is not found else the transaction id',
|
||||
'nout': '(int) None if claim is not found else the transaction output index',
|
||||
'outpoint': '(str) None if claim is not found else the tx and output',
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
}
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
|
||||
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -43,62 +136,88 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'hex': hexlify(tx.raw).decode(),
|
||||
}
|
||||
|
||||
def encode_output(self, txo):
|
||||
def encode_output(self, txo, check_signature=True):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = self.ledger.headers.height
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'height': tx_height,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'create'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'payment'
|
||||
|
||||
if txo.script.is_claim_involved:
|
||||
output.update({
|
||||
'name': txo.claim_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
})
|
||||
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
claim = txo.claim
|
||||
output['value'] = claim.claim_dict
|
||||
if claim.has_signature:
|
||||
output['valid_signature'] = None
|
||||
if txo.channel is not None:
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
output['value'] = txo.claim
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = {
|
||||
'name': txo.channel.claim_name,
|
||||
'claim_id': txo.channel.claim_id,
|
||||
'value': txo.channel.claim
|
||||
}
|
||||
if check_signature and txo.claim.is_signed:
|
||||
output['is_channel_signature_valid'] = False
|
||||
try:
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
pass
|
||||
except ValueError:
|
||||
log.exception(
|
||||
'txo.id: %s, txo.channel.id:%s, output: %s',
|
||||
txo.id, txo.channel.id, output
|
||||
)
|
||||
output['valid_signature'] = False
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'basic'
|
||||
|
||||
return output
|
||||
|
||||
def encode_input(self, txi):
|
||||
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
|
||||
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
|
||||
def encode_account(self, account):
|
||||
result = account.to_dict()
|
||||
result['id'] = account.id
|
||||
result.pop('certificates', None)
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
file = managed_stream.as_dict()
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
file.update({
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
})
|
||||
return file
|
||||
|
||||
@staticmethod
|
||||
def encode_claim(claim):
|
||||
return getattr(claim, claim.claim_type).to_dict()
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
@@ -51,14 +50,14 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
|
||||
async def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
|
||||
def emit(self, record):
|
||||
asyncio.ensure_future(self._emit(record))
|
||||
|
||||
|
||||
def get_loggly_handler(loggly_token=LOGGLY_TOKEN):
|
||||
handler = HTTPSLogglyHandler(loggly_token)
|
||||
def get_loggly_handler():
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -22,6 +22,10 @@ def migrate_db(conf, start, end):
|
||||
from .migrate7to8 import do_migration
|
||||
elif current == 8:
|
||||
from .migrate8to9 import do_migration
|
||||
elif current == 9:
|
||||
from .migrate9to10 import do_migration
|
||||
elif current == 10:
|
||||
from .migrate10to11 import do_migration
|
||||
else:
|
||||
raise Exception("DB migration of version {} to {} is not available".format(current,
|
||||
current+1))
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import binascii
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'content_fee' in current_columns or 'saved_file' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.execute(
|
||||
"pragma foreign_keys=off;"
|
||||
)
|
||||
|
||||
cursor.execute("""
|
||||
create table if not exists new_file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
""")
|
||||
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
|
||||
saved_file = 0
|
||||
if download_dir != '{stream}' and file_name != '{stream}':
|
||||
try:
|
||||
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
saved_file = 1
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
except Exception:
|
||||
download_dir, file_name = None, None
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
cursor.execute(
|
||||
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
|
||||
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
|
||||
)
|
||||
cursor.execute("drop table file")
|
||||
cursor.execute("alter table new_file rename to file")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import json
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.claim import Claim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -260,7 +260,7 @@ def do_migration(conf):
|
||||
(
|
||||
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
|
||||
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
|
||||
smart_decode(claim_arg_tup[8]).certificate_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
)
|
||||
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
|
||||
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import sqlite3
|
||||
import logging
|
||||
import os
|
||||
|
||||
from lbrynet.p2p.Error import InvalidStreamDescriptorError
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, format_sd_info, format_blobs, validate_descriptor
|
||||
from lbrynet.blob.CryptBlob import CryptBlobInfo
|
||||
import asyncio
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -22,18 +21,13 @@ def do_migration(conf):
|
||||
"left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
|
||||
blobs_by_stream = {}
|
||||
for stream_hash, position, iv, blob_hash, blob_length in blobs:
|
||||
blobs_by_stream.setdefault(stream_hash, []).append(CryptBlobInfo(blob_hash, position, blob_length or 0, iv))
|
||||
blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash))
|
||||
|
||||
for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
|
||||
sd_info = format_sd_info(
|
||||
EncryptedFileStreamType, stream_name, stream_key,
|
||||
suggested_filename, stream_hash, format_blobs(blobs_by_stream[stream_hash])
|
||||
)
|
||||
try:
|
||||
validate_descriptor(sd_info)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
log.warning("Stream for descriptor %s is invalid (%s), cleaning it up",
|
||||
sd_hash, err.message)
|
||||
sd = StreamDescriptor(asyncio.get_event_loop(), blob_dir, stream_name, stream_key, suggested_filename,
|
||||
blobs_by_stream[stream_hash], stream_hash, sd_hash)
|
||||
if sd_hash != sd.calculate_sd_hash():
|
||||
log.warning("Stream for descriptor %s is invalid, cleaning it up", sd_hash)
|
||||
blob_hashes = [blob.blob_hash for blob in blobs_by_stream[stream_hash]]
|
||||
delete_stream(cursor, stream_hash, sd_hash, blob_hashes, blob_dir)
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
query = "select stream_hash, sd_hash from main.stream"
|
||||
for stream_hash, sd_hash in cursor.execute(query).fetchall():
|
||||
head_blob_hash = cursor.execute(
|
||||
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
|
||||
(stream_hash,)
|
||||
).fetchone()
|
||||
if not head_blob_hash:
|
||||
continue
|
||||
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -1,158 +0,0 @@
|
||||
import os
|
||||
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
'.a': 'application/octet-stream',
|
||||
'.ai': 'application/postscript',
|
||||
'.aif': 'audio/x-aiff',
|
||||
'.aifc': 'audio/x-aiff',
|
||||
'.aiff': 'audio/x-aiff',
|
||||
'.au': 'audio/basic',
|
||||
'.avi': 'video/x-msvideo',
|
||||
'.bat': 'text/plain',
|
||||
'.bcpio': 'application/x-bcpio',
|
||||
'.bin': 'application/octet-stream',
|
||||
'.bmp': 'image/bmp',
|
||||
'.c': 'text/plain',
|
||||
'.cdf': 'application/x-netcdf',
|
||||
'.cpio': 'application/x-cpio',
|
||||
'.csh': 'application/x-csh',
|
||||
'.css': 'text/css',
|
||||
'.csv': 'text/csv',
|
||||
'.dll': 'application/octet-stream',
|
||||
'.doc': 'application/msword',
|
||||
'.dot': 'application/msword',
|
||||
'.dvi': 'application/x-dvi',
|
||||
'.eml': 'message/rfc822',
|
||||
'.eps': 'application/postscript',
|
||||
'.epub': 'application/epub+zip',
|
||||
'.etx': 'text/x-setext',
|
||||
'.exe': 'application/octet-stream',
|
||||
'.gif': 'image/gif',
|
||||
'.gtar': 'application/x-gtar',
|
||||
'.h': 'text/plain',
|
||||
'.hdf': 'application/x-hdf',
|
||||
'.htm': 'text/html',
|
||||
'.html': 'text/html',
|
||||
'.ico': 'image/vnd.microsoft.icon',
|
||||
'.ief': 'image/ief',
|
||||
'.iges': 'model/iges',
|
||||
'.jpe': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.js': 'application/javascript',
|
||||
'.json': 'application/json',
|
||||
'.ksh': 'text/plain',
|
||||
'.latex': 'application/x-latex',
|
||||
'.m1v': 'video/mpeg',
|
||||
'.m3u': 'application/vnd.apple.mpegurl',
|
||||
'.m3u8': 'application/vnd.apple.mpegurl',
|
||||
'.man': 'application/x-troff-man',
|
||||
'.markdown': 'text/markdown',
|
||||
'.md': 'text/markdown',
|
||||
'.me': 'application/x-troff-me',
|
||||
'.mht': 'message/rfc822',
|
||||
'.mhtml': 'message/rfc822',
|
||||
'.mif': 'application/x-mif',
|
||||
'.mov': 'video/quicktime',
|
||||
'.movie': 'video/x-sgi-movie',
|
||||
'.mp2': 'audio/mpeg',
|
||||
'.mp3': 'audio/mpeg',
|
||||
'.mp4': 'video/mp4',
|
||||
'.mpa': 'video/mpeg',
|
||||
'.mpe': 'video/mpeg',
|
||||
'.mpeg': 'video/mpeg',
|
||||
'.mpg': 'video/mpeg',
|
||||
'.ms': 'application/x-troff-ms',
|
||||
'.nc': 'application/x-netcdf',
|
||||
'.nws': 'message/rfc822',
|
||||
'.o': 'application/octet-stream',
|
||||
'.obj': 'application/octet-stream',
|
||||
'.oda': 'application/oda',
|
||||
'.p12': 'application/x-pkcs12',
|
||||
'.p7c': 'application/pkcs7-mime',
|
||||
'.pbm': 'image/x-portable-bitmap',
|
||||
'.pdf': 'application/pdf',
|
||||
'.pfx': 'application/x-pkcs12',
|
||||
'.pgm': 'image/x-portable-graymap',
|
||||
'.pl': 'text/plain',
|
||||
'.png': 'image/png',
|
||||
'.pnm': 'image/x-portable-anymap',
|
||||
'.pot': 'application/vnd.ms-powerpoint',
|
||||
'.ppa': 'application/vnd.ms-powerpoint',
|
||||
'.ppm': 'image/x-portable-pixmap',
|
||||
'.pps': 'application/vnd.ms-powerpoint',
|
||||
'.ppt': 'application/vnd.ms-powerpoint',
|
||||
'.ps': 'application/postscript',
|
||||
'.pwz': 'application/vnd.ms-powerpoint',
|
||||
'.py': 'text/x-python',
|
||||
'.pyc': 'application/x-python-code',
|
||||
'.pyo': 'application/x-python-code',
|
||||
'.qt': 'video/quicktime',
|
||||
'.ra': 'audio/x-pn-realaudio',
|
||||
'.ram': 'application/x-pn-realaudio',
|
||||
'.ras': 'image/x-cmu-raster',
|
||||
'.rdf': 'application/xml',
|
||||
'.rgb': 'image/x-rgb',
|
||||
'.roff': 'application/x-troff',
|
||||
'.rtx': 'text/richtext',
|
||||
'.sgm': 'text/x-sgml',
|
||||
'.sgml': 'text/x-sgml',
|
||||
'.sh': 'application/x-sh',
|
||||
'.shar': 'application/x-shar',
|
||||
'.snd': 'audio/basic',
|
||||
'.so': 'application/octet-stream',
|
||||
'.src': 'application/x-wais-source',
|
||||
'.stl': 'model/stl',
|
||||
'.sv4cpio': 'application/x-sv4cpio',
|
||||
'.sv4crc': 'application/x-sv4crc',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.swf': 'application/x-shockwave-flash',
|
||||
'.t': 'application/x-troff',
|
||||
'.tar': 'application/x-tar',
|
||||
'.tcl': 'application/x-tcl',
|
||||
'.tex': 'application/x-tex',
|
||||
'.texi': 'application/x-texinfo',
|
||||
'.texinfo': 'application/x-texinfo',
|
||||
'.tif': 'image/tiff',
|
||||
'.tiff': 'image/tiff',
|
||||
'.tr': 'application/x-troff',
|
||||
'.tsv': 'text/tab-separated-values',
|
||||
'.txt': 'text/plain',
|
||||
'.ustar': 'application/x-ustar',
|
||||
'.vcf': 'text/x-vcard',
|
||||
'.wav': 'audio/x-wav',
|
||||
'.webm': 'video/webm',
|
||||
'.wiz': 'application/msword',
|
||||
'.wsdl': 'application/xml',
|
||||
'.xbm': 'image/x-xbitmap',
|
||||
'.xlb': 'application/vnd.ms-excel',
|
||||
'.xls': 'application/vnd.ms-excel',
|
||||
'.xml': 'text/xml',
|
||||
'.xpdl': 'application/xml',
|
||||
'.xpm': 'image/x-xpixmap',
|
||||
'.xsl': 'application/xml',
|
||||
'.xwd': 'image/x-xwindowdump',
|
||||
'.zip': 'application/zip',
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.mid': 'audio/midi',
|
||||
'.midi': 'audio/midi',
|
||||
'.pct': 'image/pict',
|
||||
'.pic': 'image/pict',
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': 'video/x-ms-wmv'
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
|
||||
return types_map.get(ext, default)
|
||||
+458
-427
File diff suppressed because it is too large.
Load diff
@@ -1,20 +0,0 @@
|
||||
class LoopingCallManager:
|
||||
def __init__(self, calls=None):
|
||||
self.calls = calls or {}
|
||||
|
||||
def register_looping_call(self, name, call):
|
||||
assert name not in self.calls, f'{name} is already registered'
|
||||
self.calls[name] = call
|
||||
|
||||
def start(self, name, *args):
|
||||
lcall = self.calls[name]
|
||||
if not lcall.running:
|
||||
lcall.start(*args)
|
||||
|
||||
def stop(self, name):
|
||||
self.calls[name].stop()
|
||||
|
||||
def shutdown(self):
|
||||
for lcall in self.calls.values():
|
||||
if lcall.running:
|
||||
lcall.stop()
|
||||
@@ -1,65 +0,0 @@
|
||||
__doc__ = """
|
||||
Reflector is a protocol to re-host lbry blobs and streams
|
||||
Client queries and server responses follow, all dicts are encoded as json
|
||||
|
||||
############# Handshake request and response #############
|
||||
Upon connecting, the client sends a version handshake:
|
||||
{
|
||||
'version': int,
|
||||
}
|
||||
|
||||
The server replies with the same version
|
||||
{
|
||||
'version': int,
|
||||
}
|
||||
|
||||
############# Stream descriptor requests and responses #############
|
||||
(if sending blobs directly this is skipped)
|
||||
If the client is reflecting a whole stream, they send a stream descriptor request:
|
||||
{
|
||||
'sd_blob_hash': str,
|
||||
'sd_blob_size': int
|
||||
}
|
||||
|
||||
The server indicates if it's aware of this stream already by requesting (or not requesting)
|
||||
the stream descriptor blob. If the server has a validated copy of the sd blob, it will
|
||||
include the needed_blobs field (a list of blob hashes missing from reflector) in the response.
|
||||
If the server does not have the sd blob the needed_blobs field will not be included, as the
|
||||
server does not know what blobs it is missing - so the client should send all of the blobs
|
||||
in the stream.
|
||||
{
|
||||
'send_sd_blob': bool
|
||||
'needed_blobs': list, conditional
|
||||
}
|
||||
|
||||
The client may begin the file transfer of the sd blob if send_sd_blob was True.
|
||||
If the client sends the blob, after receiving it the server indicates if the
|
||||
transfer was successful:
|
||||
{
|
||||
'received_sd_blob': bool
|
||||
}
|
||||
If the transfer was not successful (False), the blob is added to the needed_blobs queue
|
||||
|
||||
############# Blob requests and responses #############
|
||||
A client with blobs to reflect (either populated by the client or by the stream descriptor
|
||||
response) queries if the server is ready to begin transferring a blob
|
||||
{
|
||||
'blob_hash': str,
|
||||
'blob_size': int
|
||||
}
|
||||
|
||||
The server replies, send_blob will be False if the server has a validated copy of the blob:
|
||||
{
|
||||
'send_blob': bool
|
||||
}
|
||||
|
||||
The client may begin the raw blob file transfer if the server replied True.
|
||||
If the client sends the blob, the server replies:
|
||||
{
|
||||
'received_blob': bool
|
||||
}
|
||||
If the transfer was not successful (False), the blob is re-added to the needed_blobs queue
|
||||
|
||||
Blob requests continue for each of the blobs the client has queued to send, when completed
|
||||
the client disconnects.
|
||||
"""
|
||||
@@ -1,206 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.internet import defer, error
|
||||
|
||||
from lbrynet.extras.reflector.common import IncompleteResponse, REFLECTOR_V2
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobReflectorClient(Protocol):
|
||||
# Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
self.blob_manager = self.factory.blob_manager
|
||||
self.response_buff = b''
|
||||
self.outgoing_buff = ''
|
||||
self.blob_hashes_to_send = self.factory.blobs
|
||||
self.next_blob_to_send = None
|
||||
self.blob_read_handle = None
|
||||
self.received_handshake_response = False
|
||||
self.protocol_version = self.factory.protocol_version
|
||||
self.file_sender = None
|
||||
self.producer = None
|
||||
self.streaming = False
|
||||
self.reflected_blobs = []
|
||||
d = self.send_handshake()
|
||||
d.addErrback(
|
||||
lambda err: log.warning("An error occurred immediately: %s", err.getTraceback()))
|
||||
|
||||
def dataReceived(self, data):
|
||||
log.debug('Received %s', data)
|
||||
self.response_buff += data
|
||||
try:
|
||||
msg = self.parse_response(self.response_buff)
|
||||
except IncompleteResponse:
|
||||
pass
|
||||
else:
|
||||
self.response_buff = b''
|
||||
d = self.handle_response(msg)
|
||||
d.addCallback(lambda _: self.send_next_request())
|
||||
d.addErrback(self.response_failure_handler)
|
||||
|
||||
def connectionLost(self, reason):
|
||||
if reason.check(error.ConnectionDone):
|
||||
if self.reflected_blobs:
|
||||
log.info('Finished sending data via reflector')
|
||||
self.factory.finished_deferred.callback(self.reflected_blobs)
|
||||
else:
|
||||
log.info('Reflector finished: %s', reason)
|
||||
self.factory.finished_deferred.callback(reason)
|
||||
|
||||
# IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
self.producer = producer
|
||||
self.streaming = streaming
|
||||
if self.streaming is False:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.producer = None
|
||||
|
||||
def write(self, data):
|
||||
self.transport.write(data)
|
||||
if self.producer is not None and self.streaming is False:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def send_handshake(self):
|
||||
log.debug('Sending handshake')
|
||||
self.write(json.dumps({'version': self.protocol_version}).encode())
|
||||
return defer.succeed(None)
|
||||
|
||||
def parse_response(self, buff):
|
||||
try:
|
||||
return json.loads(buff)
|
||||
except ValueError:
|
||||
raise IncompleteResponse()
|
||||
|
||||
def response_failure_handler(self, err):
|
||||
log.warning("An error occurred handling the response: %s", err.getTraceback())
|
||||
|
||||
def handle_response(self, response_dict):
|
||||
if self.received_handshake_response is False:
|
||||
return self.handle_handshake_response(response_dict)
|
||||
else:
|
||||
return self.handle_normal_response(response_dict)
|
||||
|
||||
def set_not_uploading(self):
|
||||
if self.next_blob_to_send is not None:
|
||||
self.read_handle.close()
|
||||
self.read_handle = None
|
||||
self.next_blob_to_send = None
|
||||
self.file_sender = None
|
||||
return defer.succeed(None)
|
||||
|
||||
def start_transfer(self):
|
||||
assert self.read_handle is not None, \
|
||||
"self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, self)
|
||||
d.addCallback(lambda _: self.read_handle.close())
|
||||
return d
|
||||
|
||||
def handle_handshake_response(self, response_dict):
|
||||
if 'version' not in response_dict:
|
||||
raise ValueError("Need protocol version number!")
|
||||
server_version = int(response_dict['version'])
|
||||
if self.protocol_version != server_version:
|
||||
raise ValueError(f"I can't handle protocol version {self.protocol_version}!")
|
||||
self.received_handshake_response = True
|
||||
return defer.succeed(True)
|
||||
|
||||
def handle_normal_response(self, response_dict):
|
||||
if self.file_sender is None: # Expecting Server Info Response
|
||||
if 'send_blob' not in response_dict:
|
||||
raise ValueError("I don't know whether to send the blob or not!")
|
||||
if response_dict['send_blob'] is True:
|
||||
self.file_sender = FileSender()
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
return self.set_not_uploading()
|
||||
else: # Expecting Server Blob Response
|
||||
if 'received_blob' not in response_dict:
|
||||
raise ValueError("I don't know if the blob made it to the intended destination!")
|
||||
else:
|
||||
if response_dict['received_blob']:
|
||||
self.reflected_blobs.append(self.next_blob_to_send.blob_hash)
|
||||
return self.set_not_uploading()
|
||||
|
||||
def open_blob_for_reading(self, blob):
|
||||
if blob.get_is_verified():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
log.debug('Getting ready to send %s', blob.blob_hash)
|
||||
self.next_blob_to_send = blob
|
||||
self.read_handle = read_handle
|
||||
return None
|
||||
raise ValueError(
|
||||
f"Couldn't open that blob for some reason. blob_hash: {blob.blob_hash}")
|
||||
|
||||
def send_blob_info(self):
|
||||
log.debug("Send blob info for %s", self.next_blob_to_send.blob_hash)
|
||||
assert self.next_blob_to_send is not None, "need to have a next blob to send at this point"
|
||||
log.debug('sending blob info')
|
||||
self.write(json.dumps({
|
||||
'blob_hash': self.next_blob_to_send.blob_hash,
|
||||
'blob_size': self.next_blob_to_send.length
|
||||
}).encode())
|
||||
|
||||
def disconnect(self, err):
|
||||
self.transport.loseConnection()
|
||||
|
||||
def send_next_request(self):
|
||||
if self.file_sender is not None:
|
||||
# send the blob
|
||||
log.debug('Sending the blob')
|
||||
return self.start_transfer()
|
||||
elif self.blob_hashes_to_send:
|
||||
# open the next blob to send
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
log.debug('No current blob, sending the next one: %s', blob_hash)
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
self.open_blob_for_reading(blob)
|
||||
# send the server the next blob hash + length
|
||||
return self.send_blob_info()
|
||||
else:
|
||||
# close connection
|
||||
log.debug('No more blob hashes, closing connection')
|
||||
self.transport.loseConnection()
|
||||
|
||||
|
||||
class BlobReflectorClientFactory(ClientFactory):
|
||||
protocol = BlobReflectorClient
|
||||
|
||||
def __init__(self, blob_manager, blobs):
|
||||
self.protocol_version = REFLECTOR_V2
|
||||
self.blob_manager = blob_manager
|
||||
self.blobs = blobs
|
||||
self.p = None
|
||||
self.finished_deferred = defer.Deferred()
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = self.protocol()
|
||||
p.factory = self
|
||||
self.p = p
|
||||
return p
|
||||
|
||||
def startFactory(self):
|
||||
log.debug('Starting reflector factory')
|
||||
ClientFactory.startFactory(self)
|
||||
|
||||
def startedConnecting(self, connector):
|
||||
log.debug('Started connecting')
|
||||
|
||||
def clientConnectionLost(self, connector, reason):
|
||||
"""If we get disconnected, reconnect to server."""
|
||||
log.debug("connection lost: %s", reason.getErrorMessage())
|
||||
|
||||
def clientConnectionFailed(self, connector, reason):
|
||||
log.debug("connection failed: %s", reason.getErrorMessage())
|
||||
@@ -1,346 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
from twisted.internet.error import ConnectionRefusedError
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.internet import defer, error
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.common import IncompleteResponse, ReflectorRequestError
|
||||
from lbrynet.extras.reflector.common import REFLECTOR_V1, REFLECTOR_V2
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileReflectorClient(Protocol):
|
||||
# Protocol stuff
|
||||
def connectionMade(self):
|
||||
log.debug("Connected to reflector")
|
||||
self.response_buff = b''
|
||||
self.outgoing_buff = b''
|
||||
self.blob_hashes_to_send = []
|
||||
self.failed_blob_hashes = []
|
||||
self.next_blob_to_send = None
|
||||
self.read_handle = None
|
||||
self.sent_stream_info = False
|
||||
self.received_descriptor_response = False
|
||||
self.received_server_version = False
|
||||
self.server_version = None
|
||||
self.stream_descriptor = None
|
||||
self.descriptor_needed = None
|
||||
self.needed_blobs = []
|
||||
self.reflected_blobs = []
|
||||
self.file_sender = None
|
||||
self.producer = None
|
||||
self.streaming = False
|
||||
|
||||
self.blob_manager = self.factory.blob_manager
|
||||
self.protocol_version = self.factory.protocol_version
|
||||
self.stream_hash = self.factory.stream_hash
|
||||
self.sd_hash = self.factory.sd_hash
|
||||
|
||||
d = self.load_descriptor()
|
||||
d.addCallback(lambda _: self.send_handshake())
|
||||
d.addErrback(lambda err: log.warning("An error occurred immediately: %s", err.getTraceback()))
|
||||
|
||||
def dataReceived(self, data):
|
||||
self.response_buff += data
|
||||
try:
|
||||
msg = self.parse_response(self.response_buff)
|
||||
except IncompleteResponse:
|
||||
pass
|
||||
else:
|
||||
self.response_buff = b''
|
||||
d = self.handle_response(msg)
|
||||
d.addCallback(lambda _: self.send_next_request())
|
||||
d.addErrback(self.response_failure_handler)
|
||||
|
||||
def store_result(self, result):
|
||||
if not self.needed_blobs or len(self.reflected_blobs) == len(self.needed_blobs):
|
||||
reflected = True
|
||||
else:
|
||||
reflected = False
|
||||
|
||||
d = f2d(self.blob_manager.storage.update_reflected_stream(
|
||||
self.sd_hash, self.transport.getPeer().host, reflected
|
||||
))
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
|
||||
def connectionLost(self, reason):
|
||||
# make sure blob file readers get closed
|
||||
self.set_not_uploading()
|
||||
|
||||
if reason.check(error.ConnectionDone):
|
||||
if not self.needed_blobs:
|
||||
log.info("Reflector has all blobs for %s", self.stream_descriptor)
|
||||
elif not self.reflected_blobs:
|
||||
log.info("No more completed blobs for %s to reflect, %i are still needed",
|
||||
self.stream_descriptor, len(self.needed_blobs))
|
||||
else:
|
||||
log.info('Finished sending reflector %i blobs for %s',
|
||||
len(self.reflected_blobs), self.stream_descriptor)
|
||||
result = self.reflected_blobs
|
||||
elif reason.check(error.ConnectionLost):
|
||||
log.warning("Stopped reflecting %s after sending %i blobs",
|
||||
self.stream_descriptor, len(self.reflected_blobs))
|
||||
result = self.reflected_blobs
|
||||
else:
|
||||
log.info('Reflector finished for %s: %s', self.stream_descriptor,
|
||||
reason)
|
||||
result = reason
|
||||
self.factory.finished_deferred.addCallback(self.store_result)
|
||||
self.factory.finished_deferred.callback(result)
|
||||
|
||||
# IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
self.producer = producer
|
||||
self.streaming = streaming
|
||||
if self.streaming is False:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.producer = None
|
||||
|
||||
def write(self, data):
|
||||
self.transport.write(data)
|
||||
if self.producer is not None and self.streaming is False:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def get_validated_blobs(self, blobs_in_stream):
|
||||
def get_blobs(blobs):
|
||||
for crypt_blob in blobs:
|
||||
if crypt_blob.blob_hash and crypt_blob.length:
|
||||
yield self.blob_manager.get_blob(crypt_blob.blob_hash, crypt_blob.length)
|
||||
return [blob for blob in get_blobs(blobs_in_stream) if blob.get_is_verified()]
|
||||
|
||||
def set_blobs_to_send(self, blobs_to_send):
|
||||
for blob in blobs_to_send:
|
||||
if blob.blob_hash not in self.blob_hashes_to_send:
|
||||
self.blob_hashes_to_send.append(blob.blob_hash)
|
||||
|
||||
def get_blobs_to_send(self):
|
||||
def _show_missing_blobs(filtered):
|
||||
if filtered:
|
||||
needs_desc = "" if not self.descriptor_needed else "descriptor and "
|
||||
log.info("Reflector needs %s%i blobs for stream",
|
||||
needs_desc,
|
||||
len(filtered))
|
||||
return filtered
|
||||
|
||||
d = f2d(self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash))
|
||||
d.addCallback(self.get_validated_blobs)
|
||||
if not self.descriptor_needed:
|
||||
d.addCallback(lambda filtered:
|
||||
[blob for blob in filtered if blob.blob_hash in self.needed_blobs])
|
||||
d.addCallback(_show_missing_blobs)
|
||||
d.addCallback(self.set_blobs_to_send)
|
||||
d.addCallback(lambda _: None if self.descriptor_needed else self.set_not_uploading())
|
||||
return d
|
||||
|
||||
def send_request(self, request_dict):
|
||||
self.write(json.dumps(request_dict).encode())
|
||||
|
||||
def send_handshake(self):
|
||||
self.send_request({'version': self.protocol_version})
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def load_descriptor(self):
|
||||
if self.sd_hash:
|
||||
self.stream_descriptor = yield self.factory.blob_manager.get_blob(self.sd_hash)
|
||||
else:
|
||||
raise ValueError("no sd hash for stream %s" % self.stream_hash)
|
||||
|
||||
def parse_response(self, buff):
|
||||
try:
|
||||
return json.loads(buff)
|
||||
except ValueError:
|
||||
raise IncompleteResponse()
|
||||
|
||||
def response_failure_handler(self, err):
|
||||
log.warning("An error occurred handling the response: %s", err.getTraceback())
|
||||
|
||||
def handle_response(self, response_dict):
|
||||
if not self.received_server_version:
|
||||
return self.handle_handshake_response(response_dict)
|
||||
elif not self.received_descriptor_response and self.server_version == REFLECTOR_V2:
|
||||
return self.handle_descriptor_response(response_dict)
|
||||
else:
|
||||
return self.handle_normal_response(response_dict)
|
||||
|
||||
def set_not_uploading(self):
|
||||
if self.next_blob_to_send is not None:
|
||||
log.debug("Close %s", self.next_blob_to_send)
|
||||
self.read_handle.close()
|
||||
self.read_handle = None
|
||||
self.next_blob_to_send = None
|
||||
if self.file_sender is not None:
|
||||
self.file_sender.stopProducing()
|
||||
self.file_sender = None
|
||||
return defer.succeed(None)
|
||||
|
||||
def start_transfer(self):
|
||||
assert self.read_handle is not None, \
|
||||
"self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, self)
|
||||
d.addCallback(lambda _: self.read_handle.close())
|
||||
return d
|
||||
|
||||
def handle_handshake_response(self, response_dict):
|
||||
if 'version' not in response_dict:
|
||||
raise ValueError("Need protocol version number!")
|
||||
self.server_version = int(response_dict['version'])
|
||||
if self.server_version not in [REFLECTOR_V1, REFLECTOR_V2]:
|
||||
raise ValueError(f"I can't handle protocol version {self.server_version}!")
|
||||
self.received_server_version = True
|
||||
return defer.succeed(True)
|
||||
|
||||
def handle_descriptor_response(self, response_dict):
|
||||
if self.file_sender is None: # Expecting Server Info Response
|
||||
if 'send_sd_blob' not in response_dict:
|
||||
raise ReflectorRequestError("I don't know whether to send the sd blob or not!")
|
||||
if response_dict['send_sd_blob'] is True:
|
||||
self.file_sender = FileSender()
|
||||
else:
|
||||
self.received_descriptor_response = True
|
||||
self.descriptor_needed = response_dict['send_sd_blob']
|
||||
self.needed_blobs = response_dict.get('needed_blobs', [])
|
||||
return self.get_blobs_to_send()
|
||||
else: # Expecting Server Blob Response
|
||||
if 'received_sd_blob' not in response_dict:
|
||||
raise ValueError("I don't know if the sd blob made it to the intended destination!")
|
||||
else:
|
||||
self.received_descriptor_response = True
|
||||
disconnect = False
|
||||
if response_dict['received_sd_blob']:
|
||||
self.reflected_blobs.append(self.next_blob_to_send.blob_hash)
|
||||
log.info("Sent reflector descriptor %s", self.next_blob_to_send)
|
||||
else:
|
||||
log.warning("Reflector failed to receive descriptor %s",
|
||||
self.next_blob_to_send)
|
||||
disconnect = True
|
||||
d = self.set_not_uploading()
|
||||
if disconnect:
|
||||
d.addCallback(lambda _: self.transport.loseConnection())
|
||||
return d
|
||||
|
||||
def handle_normal_response(self, response_dict):
|
||||
if self.file_sender is None: # Expecting Server Info Response
|
||||
if 'send_blob' not in response_dict:
|
||||
raise ValueError("I don't know whether to send the blob or not!")
|
||||
if response_dict['send_blob'] is True:
|
||||
self.file_sender = FileSender()
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
log.info("Reflector already has %s", self.next_blob_to_send)
|
||||
return self.set_not_uploading()
|
||||
else: # Expecting Server Blob Response
|
||||
if 'received_blob' not in response_dict:
|
||||
raise ValueError("I don't know if the blob made it to the intended destination!")
|
||||
else:
|
||||
if response_dict['received_blob']:
|
||||
self.reflected_blobs.append(self.next_blob_to_send.blob_hash)
|
||||
log.debug("Sent reflector blob %s", self.next_blob_to_send)
|
||||
else:
|
||||
log.warning("Reflector failed to receive blob %s", self.next_blob_to_send)
|
||||
return self.set_not_uploading()
|
||||
|
||||
def open_blob_for_reading(self, blob):
|
||||
if blob.get_is_verified():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
log.debug('Getting ready to send %s', blob.blob_hash)
|
||||
self.next_blob_to_send = blob
|
||||
self.read_handle = read_handle
|
||||
return defer.succeed(None)
|
||||
return defer.fail(ValueError(
|
||||
f"Couldn't open that blob for some reason. blob_hash: {blob.blob_hash}"))
|
||||
|
||||
def send_blob_info(self):
|
||||
assert self.next_blob_to_send is not None, "need to have a next blob to send at this point"
|
||||
r = {
|
||||
'blob_hash': self.next_blob_to_send.blob_hash,
|
||||
'blob_size': self.next_blob_to_send.length
|
||||
}
|
||||
self.send_request(r)
|
||||
|
||||
def send_descriptor_info(self):
|
||||
assert self.stream_descriptor is not None, "need to have a sd blob to send at this point"
|
||||
r = {
|
||||
'sd_blob_hash': self.stream_descriptor.blob_hash,
|
||||
'sd_blob_size': self.stream_descriptor.length
|
||||
}
|
||||
self.sent_stream_info = True
|
||||
self.send_request(r)
|
||||
|
||||
def skip_missing_blob(self, err, blob_hash):
|
||||
err.trap(ValueError)
|
||||
if blob_hash not in self.failed_blob_hashes:
|
||||
log.warning("Failed to reflect blob %s, reason: %s",
|
||||
str(blob_hash)[:16], err.getTraceback())
|
||||
self.blob_hashes_to_send.append(blob_hash)
|
||||
self.failed_blob_hashes.append(blob_hash)
|
||||
else:
|
||||
log.warning("Failed second try reflecting blob %s, giving up, reason: %s",
|
||||
str(blob_hash)[:16], err.getTraceback())
|
||||
|
||||
def send_next_request(self):
|
||||
if self.file_sender is not None:
|
||||
# send the blob
|
||||
return self.start_transfer()
|
||||
elif not self.sent_stream_info:
|
||||
# open the sd blob to send
|
||||
blob = self.stream_descriptor
|
||||
d = self.open_blob_for_reading(blob)
|
||||
d.addCallbacks(lambda _: self.send_descriptor_info(),
|
||||
lambda err: self.skip_missing_blob(err, blob.blob_hash))
|
||||
return d
|
||||
elif self.blob_hashes_to_send:
|
||||
# open the next blob to send
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
d = defer.succeed(self.blob_manager.get_blob(blob_hash))
|
||||
d.addCallback(self.open_blob_for_reading)
|
||||
d.addCallbacks(lambda _: self.send_blob_info(),
|
||||
lambda err: self.skip_missing_blob(err, blob.blob_hash))
|
||||
return d
|
||||
# close connection
|
||||
self.transport.loseConnection()
|
||||
|
||||
|
||||
class EncryptedFileReflectorClientFactory(ClientFactory):
|
||||
protocol = EncryptedFileReflectorClient
|
||||
protocol_version = REFLECTOR_V2
|
||||
|
||||
def __init__(self, blob_manager, stream_hash, sd_hash):
|
||||
self.blob_manager = blob_manager
|
||||
self.stream_hash = stream_hash
|
||||
self.sd_hash = sd_hash
|
||||
self.p = None
|
||||
self.finished_deferred = defer.Deferred()
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = self.protocol()
|
||||
p.factory = self
|
||||
self.p = p
|
||||
return p
|
||||
|
||||
def startFactory(self):
|
||||
log.debug('Starting reflector factory')
|
||||
ClientFactory.startFactory(self)
|
||||
|
||||
def startedConnecting(self, connector):
|
||||
log.debug('Connecting to reflector')
|
||||
|
||||
def clientConnectionLost(self, connector, reason):
|
||||
"""If we get disconnected, reconnect to server."""
|
||||
|
||||
def clientConnectionFailed(self, connector, reason):
|
||||
if reason.check(ConnectionRefusedError):
|
||||
log.warning("Could not connect to reflector server")
|
||||
else:
|
||||
log.error("Reflector connection failed: %s", reason)
|
||||
@@ -1,27 +0,0 @@
|
||||
REFLECTOR_V1 = 0
|
||||
REFLECTOR_V2 = 1
|
||||
|
||||
|
||||
class ReflectorClientVersionError(Exception):
|
||||
"""
|
||||
Raised by reflector server if client sends an incompatible or unknown version
|
||||
"""
|
||||
|
||||
|
||||
class ReflectorRequestError(Exception):
|
||||
"""
|
||||
Raised by reflector server if client sends a message without the required fields
|
||||
"""
|
||||
|
||||
|
||||
class ReflectorRequestDecodeError(Exception):
|
||||
"""
|
||||
Raised by reflector server if client sends an invalid json request
|
||||
"""
|
||||
|
||||
|
||||
class IncompleteResponse(Exception):
|
||||
"""
|
||||
Raised by reflector server when client sends a portion of a json request,
|
||||
used buffering the incoming request
|
||||
"""
|
||||
@@ -1,63 +0,0 @@
|
||||
from twisted.internet import reactor, defer
|
||||
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
|
||||
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
|
||||
|
||||
|
||||
def _is_ip(host):
|
||||
try:
|
||||
if len(host.split(".")) == 4 and all([0 <= int(x) <= 255 for x in host.split(".")]):
|
||||
return True
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def resolve(host):
|
||||
if _is_ip(host):
|
||||
ip = host
|
||||
else:
|
||||
ip = yield reactor.resolve(host)
|
||||
defer.returnValue(ip)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server):
|
||||
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
|
||||
factory = EncryptedFileReflectorClientFactory(blob_manager, stream_hash, sd_hash)
|
||||
ip = yield resolve(reflector_address)
|
||||
yield reactor.connectTCP(ip, reflector_port, factory)
|
||||
result = yield factory.finished_deferred
|
||||
defer.returnValue(result)
|
||||
|
||||
|
||||
def _reflect_file(lbry_file, reflector_server):
|
||||
return _reflect_stream(lbry_file.blob_manager, lbry_file.stream_hash, lbry_file.sd_hash, reflector_server)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
|
||||
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
|
||||
factory = BlobReflectorClientFactory(blob_manager, blob_hashes)
|
||||
ip = yield resolve(reflector_address)
|
||||
yield reactor.connectTCP(ip, reflector_port, factory)
|
||||
result = yield factory.finished_deferred
|
||||
return result
|
||||
|
||||
|
||||
def reflect_file(lbry_file, reflector_server):
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
return _reflect_file(lbry_file, reflector_server)
|
||||
|
||||
|
||||
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server):
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
return _reflect_blobs(blob_manager, blob_hashes, reflector_server)
|
||||
@@ -1,325 +0,0 @@
|
||||
import logging
|
||||
import json
|
||||
from twisted.python import failure
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ServerFactory
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidBlobHashError
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.extras.reflector.common import REFLECTOR_V1, REFLECTOR_V2
|
||||
from lbrynet.extras.reflector.common import ReflectorRequestError, ReflectorClientVersionError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAXIMUM_QUERY_SIZE = 200
|
||||
SEND_SD_BLOB = 'send_sd_blob'
|
||||
SEND_BLOB = 'send_blob'
|
||||
RECEIVED_SD_BLOB = 'received_sd_blob'
|
||||
RECEIVED_BLOB = 'received_blob'
|
||||
NEEDED_BLOBS = 'needed_blobs'
|
||||
VERSION = 'version'
|
||||
BLOB_SIZE = 'blob_size'
|
||||
BLOB_HASH = 'blob_hash'
|
||||
SD_BLOB_SIZE = 'sd_blob_size'
|
||||
SD_BLOB_HASH = 'sd_blob_hash'
|
||||
|
||||
|
||||
class ReflectorServer(Protocol):
|
||||
def connectionMade(self):
|
||||
peer_info = self.transport.getPeer()
|
||||
log.debug('Connection made to %s', peer_info)
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.blob_manager = self.factory.blob_manager
|
||||
self.storage = self.factory.blob_manager.storage
|
||||
self.lbry_file_manager = self.factory.lbry_file_manager
|
||||
self.protocol_version = self.factory.protocol_version
|
||||
self.received_handshake = False
|
||||
self.peer_version = None
|
||||
self.receiving_blob = False
|
||||
self.incoming_blob = None
|
||||
self.blob_finished_d = None
|
||||
self.request_buff = b""
|
||||
|
||||
self.blob_writer = None
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
log.info("Reflector upload from %s finished" % self.peer.host)
|
||||
|
||||
def handle_error(self, err):
|
||||
log.error(err.getTraceback())
|
||||
self.transport.loseConnection()
|
||||
|
||||
def send_response(self, response_dict):
|
||||
self.transport.write(json.dumps(response_dict).encode())
|
||||
|
||||
############################
|
||||
# Incoming blob file stuff #
|
||||
############################
|
||||
|
||||
def clean_up_failed_upload(self, err, blob):
|
||||
log.warning("Failed to receive %s", blob)
|
||||
if err.check(DownloadCanceledError):
|
||||
self.blob_manager.delete_blobs([blob.blob_hash])
|
||||
else:
|
||||
log.exception(err)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _on_completed_blob(self, blob, response_key):
|
||||
yield self.blob_manager.blob_completed(blob, should_announce=False)
|
||||
|
||||
if response_key == RECEIVED_SD_BLOB:
|
||||
sd_info = yield BlobStreamDescriptorReader(blob).get_info()
|
||||
yield save_sd_info(self.blob_manager, blob.blob_hash, sd_info)
|
||||
yield self.blob_manager.set_should_announce(blob.blob_hash, True)
|
||||
else:
|
||||
stream_hash = yield self.storage.get_stream_of_blob(blob.blob_hash)
|
||||
if stream_hash is not None:
|
||||
blob_num = yield self.storage.get_blob_num_by_hash(stream_hash,
|
||||
blob.blob_hash)
|
||||
if blob_num == 0:
|
||||
yield self.blob_manager.set_should_announce(blob.blob_hash, True)
|
||||
|
||||
yield self.close_blob()
|
||||
log.info("Received %s", blob)
|
||||
yield self.send_response({response_key: True})
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _on_failed_blob(self, err, response_key):
|
||||
yield self.clean_up_failed_upload(err, self.incoming_blob)
|
||||
yield self.send_response({response_key: False})
|
||||
|
||||
def handle_incoming_blob(self, response_key):
|
||||
"""
|
||||
Open blob for writing and send a response indicating if the transfer was
|
||||
successful when finished.
|
||||
|
||||
response_key will either be received_blob or received_sd_blob
|
||||
"""
|
||||
|
||||
blob = self.incoming_blob
|
||||
self.blob_writer, self.blob_finished_d = blob.open_for_writing(self.peer)
|
||||
self.blob_finished_d.addCallback(self._on_completed_blob, response_key)
|
||||
self.blob_finished_d.addErrback(self._on_failed_blob, response_key)
|
||||
|
||||
def close_blob(self):
|
||||
self.blob_writer.close()
|
||||
self.blob_writer = None
|
||||
self.blob_finished_d = None
|
||||
self.incoming_blob = None
|
||||
self.receiving_blob = False
|
||||
|
||||
####################
|
||||
# Request handling #
|
||||
####################
|
||||
|
||||
def dataReceived(self, data):
|
||||
if self.receiving_blob:
|
||||
self.blob_writer.write(data)
|
||||
else:
|
||||
log.debug('Not yet receiving blob, data needs further processing')
|
||||
self.request_buff += data
|
||||
msg, extra_data = self._get_valid_response(self.request_buff)
|
||||
if msg is not None:
|
||||
self.request_buff = b''
|
||||
d = self.handle_request(msg)
|
||||
d.addErrback(self.handle_error)
|
||||
if self.receiving_blob and extra_data:
|
||||
log.debug('Writing extra data to blob')
|
||||
self.blob_writer.write(extra_data)
|
||||
|
||||
def _get_valid_response(self, response_msg):
|
||||
extra_data = None
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while not self.receiving_blob:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren != -1:
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
except ValueError:
|
||||
if curr_pos > MAXIMUM_QUERY_SIZE:
|
||||
raise ValueError("Error decoding response: %s" % str(response_msg))
|
||||
else:
|
||||
pass
|
||||
else:
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
else:
|
||||
break
|
||||
return response, extra_data
|
||||
|
||||
def need_handshake(self):
|
||||
return self.received_handshake is False
|
||||
|
||||
def is_descriptor_request(self, request_dict):
|
||||
if SD_BLOB_HASH not in request_dict or SD_BLOB_SIZE not in request_dict:
|
||||
return False
|
||||
if not is_valid_blobhash(request_dict[SD_BLOB_HASH]):
|
||||
raise InvalidBlobHashError(request_dict[SD_BLOB_HASH])
|
||||
return True
|
||||
|
||||
def is_blob_request(self, request_dict):
|
||||
if BLOB_HASH not in request_dict or BLOB_SIZE not in request_dict:
|
||||
return False
|
||||
if not is_valid_blobhash(request_dict[BLOB_HASH]):
|
||||
raise InvalidBlobHashError(request_dict[BLOB_HASH])
|
||||
return True
|
||||
|
||||
def handle_request(self, request_dict):
|
||||
if self.need_handshake():
|
||||
return self.handle_handshake(request_dict)
|
||||
if self.is_descriptor_request(request_dict):
|
||||
return self.handle_descriptor_request(request_dict)
|
||||
if self.is_blob_request(request_dict):
|
||||
return self.handle_blob_request(request_dict)
|
||||
raise ReflectorRequestError("Invalid request")
|
||||
|
||||
def handle_handshake(self, request_dict):
|
||||
"""
|
||||
Upon connecting, the client sends a version handshake:
|
||||
{
|
||||
'version': int,
|
||||
}
|
||||
|
||||
The server replies with the same version if it is supported
|
||||
{
|
||||
'version': int,
|
||||
}
|
||||
"""
|
||||
|
||||
if VERSION not in request_dict:
|
||||
raise ReflectorRequestError("Client should send version")
|
||||
|
||||
if int(request_dict[VERSION]) not in [REFLECTOR_V1, REFLECTOR_V2]:
|
||||
raise ReflectorClientVersionError("Unknown version: %i" % int(request_dict[VERSION]))
|
||||
|
||||
self.peer_version = int(request_dict[VERSION])
|
||||
log.debug('Handling handshake for client version %i', self.peer_version)
|
||||
self.received_handshake = True
|
||||
return self.send_handshake_response()
|
||||
|
||||
def send_handshake_response(self):
|
||||
d = defer.succeed({VERSION: self.peer_version})
|
||||
d.addCallback(self.send_response)
|
||||
return d
|
||||
|
||||
def handle_descriptor_request(self, request_dict):
|
||||
"""
|
||||
If the client is reflecting a whole stream, they send a stream descriptor request:
|
||||
{
|
||||
'sd_blob_hash': str,
|
||||
'sd_blob_size': int
|
||||
}
|
||||
|
||||
The server indicates if it's aware of this stream already by requesting (or not requesting)
|
||||
the stream descriptor blob. If the server has a validated copy of the sd blob, it will
|
||||
include the needed_blobs field (a list of blob hashes missing from reflector) in the
|
||||
response. If the server does not have the sd blob the needed_blobs field will not be
|
||||
included, as the server does not know what blobs it is missing - so the client should send
|
||||
all of the blobs in the stream.
|
||||
{
|
||||
'send_sd_blob': bool
|
||||
'needed_blobs': list, conditional
|
||||
}
|
||||
|
||||
|
||||
The client may begin the file transfer of the sd blob if send_sd_blob was True.
|
||||
If the client sends the blob, after receiving it the server indicates if the
|
||||
transfer was successful:
|
||||
{
|
||||
'received_sd_blob': bool
|
||||
}
|
||||
"""
|
||||
|
||||
sd_blob_hash = request_dict[SD_BLOB_HASH]
|
||||
sd_blob_size = request_dict[SD_BLOB_SIZE]
|
||||
|
||||
if self.blob_writer is None:
|
||||
d = self.blob_manager.get_blob(sd_blob_hash, sd_blob_size)
|
||||
d.addCallback(self.get_descriptor_response)
|
||||
d.addCallback(self.send_response)
|
||||
else:
|
||||
self.receiving_blob = True
|
||||
d = self.blob_finished_d
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_descriptor_response(self, sd_blob):
|
||||
if sd_blob.get_is_verified():
|
||||
sd_info = yield BlobStreamDescriptorReader(sd_blob).get_info()
|
||||
yield save_sd_info(self.blob_manager, sd_blob.blob_hash, sd_info)
|
||||
yield self.storage.verify_will_announce_head_and_sd_blobs(sd_info['stream_hash'])
|
||||
response = yield self.request_needed_blobs({SEND_SD_BLOB: False}, sd_info['stream_hash'])
|
||||
else:
|
||||
self.incoming_blob = sd_blob
|
||||
self.receiving_blob = True
|
||||
self.handle_incoming_blob(RECEIVED_SD_BLOB)
|
||||
response = {SEND_SD_BLOB: True}
|
||||
defer.returnValue(response)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def request_needed_blobs(self, response, stream_hash):
|
||||
needed_blobs = yield self.storage.get_pending_blobs_for_stream(stream_hash)
|
||||
response.update({NEEDED_BLOBS: needed_blobs})
|
||||
defer.returnValue(response)
|
||||
|
||||
def handle_blob_request(self, request_dict):
|
||||
"""
|
||||
A client queries if the server will accept a blob
|
||||
{
|
||||
'blob_hash': str,
|
||||
'blob_size': int
|
||||
}
|
||||
|
||||
The server replies, send_blob will be False if the server has a validated copy of the blob:
|
||||
{
|
||||
'send_blob': bool
|
||||
}
|
||||
|
||||
The client may begin the raw blob file transfer if the server replied True.
|
||||
If the client sends the blob, the server replies:
|
||||
{
|
||||
'received_blob': bool
|
||||
}
|
||||
"""
|
||||
|
||||
blob_hash = request_dict[BLOB_HASH]
|
||||
blob_size = request_dict[BLOB_SIZE]
|
||||
|
||||
if self.blob_writer is None:
|
||||
log.debug('Received info for blob: %s', blob_hash[:16])
|
||||
d = self.blob_manager.get_blob(blob_hash, blob_size)
|
||||
d.addCallback(self.get_blob_response)
|
||||
d.addCallback(self.send_response)
|
||||
else:
|
||||
log.debug('blob is already open')
|
||||
self.receiving_blob = True
|
||||
d = self.blob_finished_d
|
||||
return d
|
||||
|
||||
def get_blob_response(self, blob):
|
||||
if blob.get_is_verified():
|
||||
return defer.succeed({SEND_BLOB: False})
|
||||
else:
|
||||
self.incoming_blob = blob
|
||||
self.receiving_blob = True
|
||||
self.handle_incoming_blob(RECEIVED_BLOB)
|
||||
d = defer.succeed({SEND_BLOB: True})
|
||||
return d
|
||||
|
||||
|
||||
class ReflectorServerFactory(ServerFactory):
|
||||
protocol = ReflectorServer
|
||||
|
||||
def __init__(self, peer_manager, blob_manager, lbry_file_manager):
|
||||
self.peer_manager = peer_manager
|
||||
self.blob_manager = blob_manager
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.protocol_version = REFLECTOR_V2
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
log.debug('Creating a protocol for %s', addr)
|
||||
return ServerFactory.buildProtocol(self, addr)
|
||||
@@ -2,29 +2,27 @@ import platform
|
||||
import os
|
||||
import logging.handlers
|
||||
|
||||
from lbrynet.schema import __version__ as schema_version
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_platform() -> dict:
|
||||
os_system = platform.system()
|
||||
if os.environ and 'ANDROID_ARGUMENT' in os.environ:
|
||||
os_system = 'android'
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"os_system": os_system,
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"lbryschema_version": schema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
try:
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
|
||||
return p
|
||||
@@ -1,11 +0,0 @@
|
||||
__node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.1/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
|
||||
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
from lbrynet.extras.wallet.manager import LbryWalletManager
|
||||
from lbrynet.extras.wallet.network import Network
|
||||
@@ -1,491 +0,0 @@
|
||||
import logging
|
||||
|
||||
from ecdsa import BadSignatureError
|
||||
from binascii import unhexlify, hexlify
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.p2p.Error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
|
||||
from lbrynet.schema.address import is_address
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Resolver:
|
||||
|
||||
def __init__(self, claim_trie_root, height, transaction_class, hash160_to_address, network):
|
||||
self.claim_trie_root = claim_trie_root
|
||||
self.height = height
|
||||
self.transaction_class = transaction_class
|
||||
self.hash160_to_address = hash160_to_address
|
||||
self.network = network
|
||||
|
||||
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size):
|
||||
results = {}
|
||||
for uri in requested_uris:
|
||||
resolution = (resolutions or {}).get(uri, {})
|
||||
if resolution:
|
||||
try:
|
||||
results[uri] = _handle_claim_result(
|
||||
await self._handle_resolve_uri_response(uri, resolution, page, page_size)
|
||||
)
|
||||
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
|
||||
results[uri] = {'error': str(err)}
|
||||
else:
|
||||
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
|
||||
return results
|
||||
|
||||
async def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False):
|
||||
result = {}
|
||||
claim_trie_root = self.claim_trie_root
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
certificate = None
|
||||
# parse an included certificate
|
||||
if 'certificate' in resolution:
|
||||
certificate_response = resolution['certificate']['result']
|
||||
certificate_resolution_type = resolution['certificate']['resolution_type']
|
||||
if certificate_resolution_type == "winning" and certificate_response:
|
||||
if 'height' in certificate_response:
|
||||
height = certificate_response['height']
|
||||
depth = self.height - height
|
||||
certificate_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
certificate_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_result,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "claim_id":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "sequence":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", certificate_resolution_type)
|
||||
|
||||
if 'certificate' in result:
|
||||
certificate = result['certificate']
|
||||
if 'unverified_claims_in_channel' in resolution:
|
||||
max_results = len(resolution['unverified_claims_in_channel'])
|
||||
result['claims_in_channel'] = max_results
|
||||
else:
|
||||
result['claims_in_channel'] = 0
|
||||
else:
|
||||
result['error'] = "claim not found"
|
||||
result['success'] = False
|
||||
result['uri'] = str(parsed_uri)
|
||||
|
||||
else:
|
||||
certificate = None
|
||||
|
||||
# if this was a resolution for a name, parse the result
|
||||
if 'claim' in resolution:
|
||||
claim_response = resolution['claim']['result']
|
||||
claim_resolution_type = resolution['claim']['resolution_type']
|
||||
if claim_resolution_type == "winning" and claim_response:
|
||||
if 'height' in claim_response:
|
||||
height = claim_response['height']
|
||||
depth = self.height - height
|
||||
claim_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
claim_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_result,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "claim_id":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "sequence":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", claim_resolution_type)
|
||||
|
||||
# if this was a resolution for a name in a channel make sure there is only one valid
|
||||
# match
|
||||
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
|
||||
unverified_claims_for_name = resolution['unverified_claims_for_name']
|
||||
|
||||
channel_info = await self.get_channel_claims_page(unverified_claims_for_name,
|
||||
result['certificate'], page=1)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if not claims_in_channel:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
elif len(claims_in_channel) > 1:
|
||||
log.warning("Multiple signed claims for the same name.")
|
||||
winner = pick_winner_from_channel_path_collision(claims_in_channel)
|
||||
if winner:
|
||||
result['claim'] = winner
|
||||
else:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
else:
|
||||
result['claim'] = claims_in_channel[0]
|
||||
|
||||
# parse and validate claims in a channel iteratively into pages of results
|
||||
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
|
||||
ids_to_check = resolution['unverified_claims_in_channel']
|
||||
channel_info = await self.get_channel_claims_page(ids_to_check, result['certificate'],
|
||||
page=page, page_size=page_size)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if claims_in_channel:
|
||||
result['claims_in_channel'] = claims_in_channel
|
||||
elif 'error' not in result:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
|
||||
# invalid signatures can only return outside a channel
|
||||
if result.get('claim', {}).get('has_signature', False):
|
||||
if parsed_uri.path and not result['claim']['signature_is_valid']:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
return result
|
||||
|
||||
async def get_certificate_and_validate_result(self, claim_result):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
certificate = None
|
||||
certificate_id = smart_decode(claim_result['value']).certificate_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
|
||||
claim_result['decoded_claim'] = False
|
||||
decoded = None
|
||||
|
||||
if not raw:
|
||||
claim_value = claim_result['value']
|
||||
try:
|
||||
decoded = smart_decode(claim_value)
|
||||
claim_result['value'] = decoded.claim_dict
|
||||
claim_result['decoded_claim'] = True
|
||||
except DecodeError:
|
||||
pass
|
||||
|
||||
if decoded:
|
||||
claim_result['has_signature'] = False
|
||||
if decoded.has_signature:
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.certificate_id)
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
decoded, certificate, claim_result['address'], claim_result['name'])
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
|
||||
if 'height' in claim_result and claim_result['height'] is None:
|
||||
claim_result['height'] = -1
|
||||
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result, decoded.certificate_id)
|
||||
|
||||
return claim_result
|
||||
|
||||
@staticmethod
|
||||
def prepare_claim_queries(start_position, query_size, channel_claim_infos):
|
||||
queries = [tuple()]
|
||||
names = {}
|
||||
# a table of index counts for the sorted claim ids, including ignored claims
|
||||
absolute_position_index = {}
|
||||
|
||||
block_sorted_infos = sorted(channel_claim_infos.items(), key=lambda x: int(x[1][1]))
|
||||
per_block_infos = {}
|
||||
for claim_id, (name, height) in block_sorted_infos:
|
||||
claims = per_block_infos.get(height, [])
|
||||
claims.append((claim_id, name))
|
||||
per_block_infos[height] = sorted(claims, key=lambda x: int(x[0], 16))
|
||||
|
||||
abs_position = 0
|
||||
|
||||
for height in sorted(per_block_infos.keys(), reverse=True):
|
||||
for claim_id, name in per_block_infos[height]:
|
||||
names[claim_id] = name
|
||||
absolute_position_index[claim_id] = abs_position
|
||||
if abs_position >= start_position:
|
||||
if len(queries[-1]) >= query_size:
|
||||
queries.append(tuple())
|
||||
queries[-1] += (claim_id,)
|
||||
abs_position += 1
|
||||
return queries, names, absolute_position_index
|
||||
|
||||
async def iter_channel_claims_pages(self, queries, claim_positions, claim_names, certificate,
|
||||
page_size=10):
|
||||
# lbryum server returns a dict of {claim_id: (name, claim_height)}
|
||||
# first, sort the claims by block height (and by claim id int value within a block).
|
||||
|
||||
# map the sorted claims into getclaimsbyids queries of query_size claim ids each
|
||||
|
||||
# send the batched queries to lbryum server and iteratively validate and parse
|
||||
# the results, yield a page of results at a time.
|
||||
|
||||
# these results can include those where `signature_is_valid` is False. if they are skipped,
|
||||
# page indexing becomes tricky, as the number of results isn't known until after having
|
||||
# processed them.
|
||||
# TODO: fix ^ in lbrynet.schema
|
||||
|
||||
async def iter_validate_channel_claims():
|
||||
formatted_claims = []
|
||||
for claim_ids in queries:
|
||||
batch_result = await self.network.get_claims_by_ids(*claim_ids)
|
||||
for claim_id in claim_ids:
|
||||
claim = batch_result[claim_id]
|
||||
if claim['name'] == claim_names[claim_id]:
|
||||
formatted_claim = await self.parse_and_validate_claim_result(claim, certificate)
|
||||
formatted_claim['absolute_channel_position'] = claim_positions[
|
||||
claim['claim_id']]
|
||||
formatted_claims.append(formatted_claim)
|
||||
else:
|
||||
log.warning("ignoring claim with name mismatch %s %s", claim['name'],
|
||||
claim['claim_id'])
|
||||
return formatted_claims
|
||||
|
||||
results = []
|
||||
|
||||
for claim in await iter_validate_channel_claims():
|
||||
results.append(claim)
|
||||
|
||||
# if there is a full page of results, yield it
|
||||
if len(results) and len(results) % page_size == 0:
|
||||
return results[-page_size:]
|
||||
|
||||
return results
|
||||
|
||||
async def get_channel_claims_page(self, channel_claim_infos, certificate, page, page_size=10):
|
||||
page = page or 0
|
||||
page_size = max(page_size, 1)
|
||||
if page_size > 500:
|
||||
raise Exception("page size above maximum allowed")
|
||||
start_position = (page - 1) * page_size
|
||||
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
|
||||
channel_claim_infos)
|
||||
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
|
||||
certificate, page_size=page_size)
|
||||
upper_bound = len(claim_positions)
|
||||
if not page:
|
||||
return None, upper_bound
|
||||
if start_position > upper_bound:
|
||||
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
|
||||
return page_generator, upper_bound
|
||||
|
||||
|
||||
# Format amount to be decimal encoded string
|
||||
# Format value to be hex encoded string
|
||||
# TODO: refactor. Came from lbryum, there could be another part of torba doing it
|
||||
def format_amount_value(obj):
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in ('amount', 'effective_amount'):
|
||||
if not isinstance(obj[k], float):
|
||||
obj[k] = dewies_to_lbc(obj[k])
|
||||
elif k == 'supports' and isinstance(v, list):
|
||||
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
|
||||
for (txid, nout, amount) in v]
|
||||
elif isinstance(v, (list, dict)):
|
||||
obj[k] = format_amount_value(v)
|
||||
elif isinstance(obj, list):
|
||||
obj = [format_amount_value(o) for o in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
"""
|
||||
Verify proof for name claim
|
||||
"""
|
||||
|
||||
def _build_response(name, value, claim_id, txid, n, amount, effective_amount,
|
||||
claim_sequence, claim_address, supports):
|
||||
r = {
|
||||
'name': name,
|
||||
'value': hexlify(value),
|
||||
'claim_id': claim_id,
|
||||
'txid': txid,
|
||||
'nout': n,
|
||||
'amount': amount,
|
||||
'effective_amount': effective_amount,
|
||||
'height': height,
|
||||
'depth': depth,
|
||||
'claim_sequence': claim_sequence,
|
||||
'address': claim_address,
|
||||
'supports': supports
|
||||
}
|
||||
return r
|
||||
|
||||
def _parse_proof_result(name, result):
|
||||
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
|
||||
supports = result['supports']
|
||||
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
|
||||
if 'transaction' in result:
|
||||
tx = transaction_class(raw=unhexlify(result['transaction']))
|
||||
nOut = result['proof']['nOut']
|
||||
if result['proof']['txhash'] == tx.id:
|
||||
if 0 <= nOut < len(tx.outputs):
|
||||
claim_output = tx.outputs[nOut]
|
||||
effective_amount = claim_output.amount + support_amount
|
||||
claim_address = hash160_to_address(claim_output.script.values['pubkey_hash'])
|
||||
claim_id = result['claim_id']
|
||||
claim_sequence = result['claim_sequence']
|
||||
claim_script = claim_output.script
|
||||
decoded_name = claim_script.values['claim_name'].decode()
|
||||
decoded_value = claim_script.values['claim']
|
||||
if decoded_name == name:
|
||||
return _build_response(name, decoded_value, claim_id,
|
||||
tx.id, nOut, claim_output.amount,
|
||||
effective_amount, claim_sequence,
|
||||
claim_address, supports)
|
||||
return {'error': 'name in proof did not match requested name'}
|
||||
outputs = len(tx['outputs'])
|
||||
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
|
||||
return {'error': "computed txid did not match given transaction: %s vs %s" %
|
||||
(tx.id, result['proof']['txhash'])
|
||||
}
|
||||
return {'error': "didn't receive a transaction with the proof"}
|
||||
return {'error': 'name is not claimed'}
|
||||
|
||||
if 'proof' in result:
|
||||
try:
|
||||
verify_proof(result['proof'], claim_trie_root, name)
|
||||
except InvalidProofError:
|
||||
return {'error': "Proof was invalid"}
|
||||
return _parse_proof_result(name, result)
|
||||
else:
|
||||
return {'error': "proof not in result"}
|
||||
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
claim_address, name, decoded_certificate=None):
|
||||
if not certificate_claim:
|
||||
return False, None
|
||||
if 'value' not in certificate_claim:
|
||||
log.warning('Got an invalid claim while parsing certificates, please report: %s', certificate_claim)
|
||||
return False, None
|
||||
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
|
||||
if not isinstance(certificate, ClaimDict):
|
||||
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
|
||||
if _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
return True, certificate_claim['name']
|
||||
return False, None
|
||||
|
||||
|
||||
def _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
if not claim.has_signature:
|
||||
raise Exception("Claim is not signed")
|
||||
if not is_address(claim_address):
|
||||
raise Exception("Not given a valid claim address")
|
||||
try:
|
||||
if claim.validate_signature(claim_address, certificate.protobuf, name):
|
||||
return True
|
||||
except BadSignatureError:
|
||||
# print_msg("Signature for %s is invalid" % claim_id)
|
||||
return False
|
||||
except Exception as err:
|
||||
log.error("Signature for %s is invalid, reason: %s - %s", claim_address,
|
||||
str(type(err)), err)
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
|
||||
def _decode_claim_result(claim):
|
||||
if 'has_signature' in claim and claim['has_signature']:
|
||||
if not claim['signature_is_valid']:
|
||||
log.warning("lbry://%s#%s has an invalid signature",
|
||||
claim['name'], claim['claim_id'])
|
||||
if 'value' not in claim:
|
||||
log.warning('Got an invalid claim while parsing, please report: %s', claim)
|
||||
claim['hex'] = None
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to parse: missing value"
|
||||
return claim
|
||||
try:
|
||||
decoded = smart_decode(claim['value'])
|
||||
claim_dict = decoded.claim_dict
|
||||
claim['value'] = claim_dict
|
||||
claim['hex'] = hexlify(decoded.serialized)
|
||||
except DecodeError:
|
||||
claim['hex'] = claim['value']
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to decode value"
|
||||
return claim
|
||||
|
||||
|
||||
def _handle_claim_result(results):
|
||||
if not results:
|
||||
#TODO: cannot determine what name we searched for here
|
||||
# we should fix lbryum commands that return None
|
||||
raise UnknownNameError("")
|
||||
|
||||
if 'error' in results:
|
||||
if results['error'] in ['name is not claimed', 'claim not found']:
|
||||
if 'claim_id' in results:
|
||||
raise UnknownClaimID(results['claim_id'])
|
||||
elif 'name' in results:
|
||||
raise UnknownNameError(results['name'])
|
||||
elif 'uri' in results:
|
||||
raise UnknownURI(results['uri'])
|
||||
elif 'outpoint' in results:
|
||||
raise UnknownOutpoint(results['outpoint'])
|
||||
raise Exception(results['error'])
|
||||
|
||||
# case where return value is {'certificate':{'txid', 'value',...},...}
|
||||
if 'certificate' in results:
|
||||
results['certificate'] = _decode_claim_result(results['certificate'])
|
||||
|
||||
# case where return value is {'claim':{'txid','value',...},...}
|
||||
if 'claim' in results:
|
||||
results['claim'] = _decode_claim_result(results['claim'])
|
||||
|
||||
# case where return value is {'txid','value',...}
|
||||
# returned by queries that are not name resolve related
|
||||
# (getclaimbyoutpoint, getclaimbyid, getclaimsfromtx)
|
||||
elif 'value' in results:
|
||||
results = _decode_claim_result(results)
|
||||
|
||||
# case where there is no 'certificate', 'value', or 'claim' key
|
||||
elif 'certificate' not in results:
|
||||
msg = f'result in unexpected format:{results}'
|
||||
assert False, msg
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def pick_winner_from_channel_path_collision(claims_in_channel):
|
||||
# we should be doing this by effective amount so we pick the controlling claim, however changing the resolved
|
||||
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
|
||||
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
|
||||
# potentially triggering that.
|
||||
winner = None
|
||||
for claim in claims_in_channel:
|
||||
if not claim['signature_is_valid']:
|
||||
continue
|
||||
if winner is None:
|
||||
winner = claim
|
||||
elif claim['claim_sequence'] < winner['claim_sequence']:
|
||||
winner = claim
|
||||
return winner
|
||||
@@ -1,47 +0,0 @@
|
||||
from collections import namedtuple
|
||||
import msgpack
|
||||
from torba.server.util import cachedproperty
|
||||
# Classes representing data and their serializers, if any.
|
||||
|
||||
|
||||
class ClaimInfo(namedtuple("NameClaim", "name value txid nout amount address height cert_id")):
|
||||
'''Claim information as its stored on database'''
|
||||
|
||||
@classmethod
|
||||
def from_serialized(cls, serialized):
|
||||
return cls(*msgpack.loads(serialized))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
return msgpack.dumps(self)
|
||||
|
||||
|
||||
class NameClaim(namedtuple("NameClaim", "name value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimUpdate(namedtuple("ClaimUpdate", "name claim_id value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimSupport(namedtuple("ClaimSupport", "name claim_id")):
|
||||
pass
|
||||
|
||||
|
||||
class LBRYTx(namedtuple("Tx", "version inputs outputs locktime")):
|
||||
'''Transaction that can contain claim, update or support in its outputs.'''
|
||||
|
||||
@cachedproperty
|
||||
def is_coinbase(self):
|
||||
return self.inputs[0].is_coinbase
|
||||
|
||||
@cachedproperty
|
||||
def has_claims(self):
|
||||
for output in self.outputs:
|
||||
if output.claim:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class TxClaimOutput(namedtuple("TxClaimOutput", "value pk_script claim")):
|
||||
pass
|
||||
Loaded 100 of 309 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user