Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ba8bd627a8 | ||
|
|
e6528963ad | ||
|
|
6aa709e34e | ||
|
|
e73f121c08 | ||
|
|
3b9d6c4f8e | ||
|
|
69b60c19c3 | ||
|
|
04278cb1ab | ||
|
|
e8dec5b440 | ||
|
|
75f82b107e | ||
|
|
0e90c4b611 | ||
|
|
fb66b68bc8 | ||
|
|
c601677386 | ||
|
|
1fbd1cd04e | ||
|
|
4a71b91fbf | ||
|
|
b8b0d8b1ac | ||
|
|
8ef73d3985 | ||
|
|
05e556a5e3 | ||
|
|
d1da42bea8 | ||
|
|
90685e8357 | ||
|
|
a471d5d167 | ||
|
|
16122299e4 | ||
|
|
9f18da4bae | ||
|
|
1abce52867 | ||
|
|
072e2b65d3 | ||
|
|
f6d11e5467 | ||
|
|
458bddcb65 | ||
|
|
5fad43ff69 | ||
|
|
6992d0d947 | ||
|
|
d1b0180437 | ||
|
|
ba59e126cd | ||
|
|
4e3467bb7e | ||
|
|
4d4c705eb9 | ||
|
|
ce7b3184c4 | ||
|
|
b71faddaac | ||
|
|
1328cb4cce | ||
|
|
946488bce9 | ||
|
|
7713881e71 | ||
|
|
26fcc48c5a | ||
|
|
bf12fb68a2 | ||
|
|
3a4c9ff4f9 | ||
|
|
6180a2917d | ||
|
|
014cbb4623 | ||
|
|
ee98251d79 | ||
|
|
e3d08ec773 | ||
|
|
f03e68e63e | ||
|
|
72090b45dd | ||
|
|
93f9622798 | ||
|
|
16ff4b4e2c | ||
|
|
c90cca4b20 | ||
|
|
98d83ea74f | ||
|
|
a234f92d2c | ||
|
|
8ba3575901 | ||
|
|
a857c83c59 | ||
|
|
a6128ac8a2 | ||
|
|
2693301ad0 | ||
|
|
f7c7cb9535 | ||
|
|
b36a5f9b6b | ||
|
|
4826eb9969 | ||
|
|
c34b150794 | ||
|
|
c4c7670a7c | ||
|
|
0918bac86e | ||
|
|
a60a921706 | ||
|
|
b3ee29b66d | ||
|
|
3a1a9b988c | ||
|
|
12f2bebc46 | ||
|
|
31a3812c8a | ||
|
|
70ec08e30b | ||
|
|
23e8f0d849 | ||
|
|
1cdfcd5b4e | ||
|
|
dc2786fbc0 | ||
|
|
7e39b0c4fe | ||
|
|
4af830c405 | ||
|
|
fe844581fd | ||
|
|
443ce6352b | ||
|
|
158c354a27 | ||
|
|
92d1230a90 | ||
|
|
d64916a061 | ||
|
|
20a0568253 | ||
|
|
5a1f42ee54 | ||
|
|
ab95f14e21 | ||
|
|
bae2676b69 | ||
|
|
ea6869a63e | ||
|
|
953327c150 | ||
|
|
e01ad6bc85 | ||
|
|
47a33009ff | ||
|
|
bc844013a1 | ||
|
|
2fd2319cdb | ||
|
|
a50657c472 | ||
|
|
ef5f6e06ba | ||
|
|
977cf9f506 | ||
|
|
ffae10d11f | ||
|
|
221107e76e | ||
|
|
e74c78b607 | ||
|
|
434c7b57b1 | ||
|
|
9dc4608a62 | ||
|
|
fc69366092 | ||
|
|
9f0b1f3e25 | ||
|
|
3f03a845ec | ||
|
|
bc28976e12 | ||
|
|
8f826098f9 | ||
|
|
0a765e6207 | ||
|
|
f83aa61923 | ||
|
|
f9ff3e5d14 | ||
|
|
b3fde9d78d | ||
|
|
0c999083cc | ||
|
|
561215db8a | ||
|
|
8dbb7f2c78 | ||
|
|
6f7f3bb316 | ||
|
|
8d507c4b73 | ||
|
|
2beb0eda94 | ||
|
|
b7e07b72ca | ||
|
|
fde8718c8c | ||
|
|
b2d1bddcfe | ||
|
|
dd1d1be994 | ||
|
|
a93957178a | ||
|
|
5026d0b2f3 | ||
|
|
cc8b47d882 | ||
|
|
db0b5961b3 | ||
|
|
cb6f7ada42 | ||
|
|
59464099b4 | ||
|
|
aed00b53fd | ||
|
|
cdb0f97cb4 | ||
|
|
33bcc8256c | ||
|
|
0a8d44f12f | ||
|
|
1b74594f5b | ||
|
|
b0718d295e | ||
|
|
9bb91b126d | ||
|
|
6dbb00d8d5 | ||
|
|
3a27a4509a | ||
|
|
1d438f5d23 | ||
|
|
3840f19869 | ||
|
|
ec0483b975 | ||
|
|
2fba1f417f | ||
|
|
5b419b55c5 | ||
|
|
b31379e51c | ||
|
|
8394dcb663 | ||
|
|
457a56ce04 | ||
|
|
7c8016d807 | ||
|
|
d704336f20 | ||
|
|
1b4230b06e | ||
|
|
88095af1cd | ||
|
|
e9efd9f68a | ||
|
|
7b32e2ff7c | ||
|
|
572777305e | ||
|
|
7e27a6e37d | ||
|
|
b98621d7ea | ||
|
|
bc5824b8bc | ||
|
|
a6b643bb34 | ||
|
|
488b976734 | ||
|
|
253912b52e | ||
|
|
0bf8416d9f | ||
|
|
2e921437d8 | ||
|
|
b6f9b60683 | ||
|
|
57430e9f5f | ||
|
|
1e13fbee8d | ||
|
|
3d8b9e2b66 | ||
|
|
64898d43f1 | ||
|
|
b2117fddab | ||
|
|
5658623f29 | ||
|
|
934682bd0f | ||
|
|
daecc582f3 | ||
|
|
3cb4012dbf | ||
|
|
f7bd44a4f5 | ||
|
|
52317b05bb | ||
|
|
83c0baa3cb | ||
|
|
340c5addab | ||
|
|
e3fd94dbf0 | ||
|
|
923fcfb9e9 | ||
|
|
0ae5f1a4c6 | ||
|
|
90e06abba2 | ||
|
|
6ecfc94ff1 | ||
|
|
60c4a727d3 | ||
|
|
9fbb9f2e98 | ||
|
|
f1feec66c0 | ||
|
|
73b9b983b1 | ||
|
|
588779e3a4 | ||
|
|
b7ae446059 | ||
|
|
db898307b5 | ||
|
|
7e3512cbf2 | ||
|
|
cdd65c6783 | ||
|
|
320ad717cd | ||
|
|
264e6744e1 | ||
|
|
64e2b79c07 | ||
|
|
9255bb5abe | ||
|
|
d7525dacc3 | ||
|
|
5cb07100f8 | ||
|
|
171c080d33 | ||
|
|
cfc3ec2a05 | ||
|
|
1392b8f83b | ||
|
|
8b265801df | ||
|
|
6c4fb8dab1 | ||
|
|
c1a9cd93ee | ||
|
|
0d4e83e167 | ||
|
|
1292eb345d | ||
|
|
f4c6dc90c9 | ||
|
|
8c8f5d8f58 | ||
|
|
b6d986cb40 | ||
|
|
6cc134c7c8 | ||
|
|
eee964cfa0 | ||
|
|
a03c94bfad | ||
|
|
f1f0917063 | ||
|
|
fe2476b5aa | ||
|
|
9c5fa8bf76 | ||
|
|
cd43a71172 | ||
|
|
e2c6de76f4 | ||
|
|
05aeadc9fe | ||
|
|
7153417372 | ||
|
|
fceb2ae0a2 | ||
|
|
0cef2fdd82 | ||
|
|
9c91b135bd | ||
|
|
e3b127192c | ||
|
|
ee1738da6e | ||
|
|
8ca2b75c80 | ||
|
|
dc81b6def1 | ||
|
|
22f5aaee70 | ||
|
|
4bd6569c9c | ||
|
|
dbf39baeb1 | ||
|
|
3adf3c26f2 | ||
|
|
fa0d13e630 | ||
|
|
227a82d6d3 | ||
|
|
65cfc1f63c | ||
|
|
e46f0e2444 | ||
|
|
00bc7995c4 | ||
|
|
2dcebe48b7 | ||
|
|
d12b6cddb3 | ||
|
|
699c057f2d | ||
|
|
3ce9f31f4b | ||
|
|
485d5d55e8 | ||
|
|
6ee9f2513b | ||
|
|
81a6e14a5c | ||
|
|
496ffe0096 | ||
|
|
216c760179 | ||
|
|
01eca3004f | ||
|
|
2f47a3758e | ||
|
|
73bcc1796f | ||
|
|
e03e41ad31 | ||
|
|
6cb30f8e05 | ||
|
|
d9b1be6214 | ||
|
|
5f2f06c269 | ||
|
|
e1dcdcf4db | ||
|
|
416472c5d8 | ||
|
|
196f3ba92d | ||
|
|
6feae93c58 | ||
|
|
203d30d4f7 | ||
|
|
d0a7ca841b | ||
|
|
2e6c848c39 | ||
|
|
18a0204105 | ||
|
|
01baa193d5 | ||
|
|
ba57af5502 | ||
|
|
8139f93ff1 | ||
|
|
4596d5c26f | ||
|
|
810a63782d | ||
|
|
ea83827d57 | ||
|
|
e19f8f3d95 | ||
|
|
7af501e0bc | ||
|
|
a692b6ff7d | ||
|
|
3364ac0976 | ||
|
|
cf1157f02a | ||
|
|
0ed56bbe1a | ||
|
|
213bdac55f | ||
|
|
6d8ab97ca9 | ||
|
|
fb34476cd7 | ||
|
|
07d353a947 | ||
|
|
3c5fb6490d | ||
|
|
a5d21081b9 | ||
|
|
ed98a3cc21 | ||
|
|
45c2cd2986 | ||
|
|
75f21950da | ||
|
|
3f704be85c | ||
|
|
67a15ea788 | ||
|
|
55bcd2dfee | ||
|
|
d9757fdae8 | ||
|
|
a4ae8d7c53 | ||
|
|
78078b4c9f | ||
|
|
1331e2bbe6 | ||
|
|
4c4d652f1b | ||
|
|
73bd76b631 | ||
|
|
59a0f9f602 | ||
|
|
1b1847bc31 | ||
|
|
21a569edcc | ||
|
|
2ddf1a08f6 | ||
|
|
8754c6fabe | ||
|
|
f049109a38 | ||
|
|
8db7c15aa1 | ||
|
|
90e0694d2e | ||
|
|
4748cbf654 | ||
|
|
74eb991983 | ||
|
|
e11bae2dda | ||
|
|
8462f56b13 | ||
|
|
0bec3c7294 | ||
|
|
2bb4052193 | ||
|
|
101a3f2b22 | ||
|
|
04a2bde8dd | ||
|
|
a1368b05da | ||
|
|
99ea7a26b0 | ||
|
|
8b3a457925 | ||
|
|
e02e2a249a | ||
|
|
3019a1c222 | ||
|
|
1f3a19c100 | ||
|
|
5eac4247e1 | ||
|
|
1ca82b9d52 | ||
|
|
21702ec43c | ||
|
|
d2083bc17e | ||
|
|
52edf880e0 | ||
|
|
1f45af991f | ||
|
|
8afdd0b2e9 | ||
|
|
99efc70ba3 | ||
|
|
64e306801d | ||
|
|
330db61b36 | ||
|
|
357763ff61 | ||
|
|
8a6e6b4d77 | ||
|
|
64913c4288 | ||
|
|
f01ad0e4bd | ||
|
|
d9fcc458dc | ||
|
|
24a11d3f21 | ||
|
|
234eaf8890 | ||
|
|
9e2e53147e | ||
|
|
fc41af5889 | ||
|
|
72ce0cd0b0 | ||
|
|
e4497c1327 | ||
|
|
a1bf832ea6 | ||
|
|
d4e76500fb | ||
|
|
f898bb3d0e | ||
|
|
ad13282494 | ||
|
|
73519f8c6a | ||
|
|
b2c95f35c7 | ||
|
|
0bfccb86ff | ||
|
|
5118240897 | ||
|
|
427fc179ac | ||
|
|
55bcc520e4 | ||
|
|
bc8251696e | ||
|
|
677ba25ccd | ||
|
|
95436b233c | ||
|
|
82871b8f6b | ||
|
|
9525b87224 | ||
|
|
f06098978c | ||
|
|
141491f2da | ||
|
|
38006f7d29 | ||
|
|
3664c25d98 | ||
|
|
1747c9d866 | ||
|
|
1066fbaa7c | ||
|
|
7f7961f4cc | ||
|
|
9d9916548b | ||
|
|
ada5b55f22 | ||
|
|
a2f28e7788 | ||
|
|
c6092a4c79 | ||
|
|
315a557019 | ||
|
|
20221e1186 | ||
|
|
50003d2600 | ||
|
|
ea6b2b98fb | ||
|
|
4890fdfb50 | ||
|
|
2f95c3a9d1 | ||
|
|
6039159ecc | ||
|
|
8e4d8345bd | ||
|
|
22df26eeb6 | ||
|
|
7b9ff3e8b5 | ||
|
|
1a5b2c08ee | ||
|
|
b6babb1c9d | ||
|
|
a88485c5a0 | ||
|
|
0cefa07e97 | ||
|
|
ca9d48d574 | ||
|
|
e10e1de92f | ||
|
|
bdf6887791 | ||
|
|
295f2c1115 | ||
|
|
78fcf6a44a | ||
|
|
7dcebf99e0 | ||
|
|
463cbde4b1 | ||
|
|
1adce8b227 | ||
|
|
673d259d7a | ||
|
|
d0b739a0b9 | ||
|
|
b4aa51444c | ||
|
|
9178ca701c | ||
|
|
7eb9f344f4 | ||
|
|
9f8d3e69ae | ||
|
|
e1130147b5 | ||
|
|
0fb6f50aba | ||
|
|
91fc772e3c | ||
|
|
30b3d31af0 | ||
|
|
1a9f81ea14 | ||
|
|
5813df74a5 | ||
|
|
dc6d803fed | ||
|
|
f9cba69d03 | ||
|
|
17f9d7cfb2 | ||
|
|
42c43e86ef | ||
|
|
8cd230eb6e | ||
|
|
dcc71713c3 | ||
|
|
3d72b2adf3 | ||
|
|
655ece18de | ||
|
|
cda14f7c69 | ||
|
|
3d36e36173 | ||
|
|
2dd0221711 | ||
|
|
1fec9f52ef | ||
|
|
ca9ca09a00 | ||
|
|
b16c36b55f | ||
|
|
1c2849235c | ||
|
|
202a957a1a | ||
|
|
66f33202c7 | ||
|
|
f310bb6825 | ||
|
|
1fb7ffd759 | ||
|
|
3fcca4bc47 | ||
|
|
12c73cf50c | ||
|
|
02d398613d | ||
|
|
65e13898c2 | ||
|
|
559afa465a | ||
|
|
c753a5825a | ||
|
|
4185a3b9e3 | ||
|
|
2f4e6911d6 | ||
|
|
bcf7218b66 | ||
|
|
e21ade7360 | ||
|
|
53d4c149bc | ||
|
|
0adcaa6a5d | ||
|
|
ce4d734250 | ||
|
|
04e0933d36 | ||
|
|
bcf52c3f24 | ||
|
|
a702afd010 | ||
|
|
ad492a09f7 | ||
|
|
790991ecad | ||
|
|
f5789775a2 | ||
|
|
5cc0d365c2 | ||
|
|
5430a0f289 | ||
|
|
cc7656faa4 | ||
|
|
03415415ed | ||
|
|
1e47ff5a83 | ||
|
|
88bad1f868 | ||
|
|
91405ef232 | ||
|
|
570c1163c1 | ||
|
|
17827036ce | ||
|
|
338e6c2116 | ||
|
|
cdf6155102 | ||
|
|
e47ef741f7 | ||
|
|
2878716381 | ||
|
|
6d9947779e | ||
|
|
c7e5bdd3ae | ||
|
|
46dcf8a899 | ||
|
|
c050231ac5 | ||
|
|
ee5be5adc8 | ||
|
|
d99ce5f3d6 | ||
|
|
e7634b74c8 | ||
|
|
26af01a5cc | ||
|
|
36989f1974 | ||
|
|
1d40ec34a2 | ||
|
|
f0f30fd75c | ||
|
|
be2ef747dc | ||
|
|
268a01dc23 | ||
|
|
8399f2ab33 | ||
|
|
7698ab7053 | ||
|
|
eff66bdf80 | ||
|
|
70e68c6361 | ||
|
|
2a53afbfd8 | ||
|
|
a42bf67800 | ||
|
|
b783b4daed | ||
|
|
c38b463bf6 | ||
|
|
8696cd3c27 | ||
|
|
0abbc20899 | ||
|
|
2fca29144e | ||
|
|
1675cc4580 | ||
|
|
93d134f0ce | ||
|
|
1c6ce49b7b | ||
|
|
a7f34e2913 | ||
|
|
2a285d139a | ||
|
|
5e371e8e6a | ||
|
|
fdc98fc746 | ||
|
|
0d8e55bc95 | ||
|
|
69851a6588 | ||
|
|
b101fafd39 | ||
|
|
d2aaf7bd09 | ||
|
|
5a1f551ce5 | ||
|
|
584433e16c | ||
|
|
0c9c6c2705 | ||
|
|
afae649a9e | ||
|
|
2238e1f802 | ||
|
|
518c447fef | ||
|
|
cb9b348828 | ||
|
|
eab95a6246 | ||
|
|
593d0046bd | ||
|
|
c5d1478c02 | ||
|
|
1c57293e9c | ||
|
|
14a067a4c4 | ||
|
|
707ab831fd | ||
|
|
ceb25f917a | ||
|
|
002faf7a58 | ||
|
|
d1bbb7af74 | ||
|
|
ea603dce13 | ||
|
|
f2a768c9c4 | ||
|
|
12384a89a1 | ||
|
|
c21956b4f3 | ||
|
|
4a8776be10 | ||
|
|
d6d69a46d4 | ||
|
|
4384a5f022 | ||
|
|
aa83bbd637 | ||
|
|
daae0b2836 | ||
|
|
5afd446bb3 | ||
|
|
2b5e3204c0 | ||
|
|
84c91c480f | ||
|
|
cbc98e72a7 | ||
|
|
d1f00255c1 | ||
|
|
434d98c6a6 | ||
|
|
e51c98ca8c | ||
|
|
f719aa1db0 | ||
|
|
3686b1d970 | ||
|
|
e6fd8cc0f2 | ||
|
|
8739dc5d2c | ||
|
|
d6c04d3586 | ||
|
|
7236203f1e | ||
|
|
8c6c442fdd | ||
|
|
49c659b832 | ||
|
|
2590523fa1 | ||
|
|
08b4fd6942 | ||
|
|
6a8963d807 | ||
|
|
083bf4f61d | ||
|
|
ab3b0134b5 | ||
|
|
3b88d2465d | ||
|
|
0841c90e6c | ||
|
|
3f6e928cc2 | ||
|
|
a32a0c6401 | ||
|
|
95cb29d3c8 | ||
|
|
e85f8b245b | ||
|
|
04836ea0d9 | ||
|
|
451823f33e | ||
|
|
8c3552dd34 | ||
|
|
a204f0d3e6 | ||
|
|
892758be65 | ||
|
|
3594c8976d | ||
|
|
23ede44bff | ||
|
|
0006a68bf2 | ||
|
|
0349e68201 | ||
|
|
be248dc448 | ||
|
|
d0d5d0340d | ||
|
|
f79a49bbf4 | ||
|
|
6a5d88a0d5 | ||
|
|
4165881d5f | ||
|
|
8dc4e3be43 | ||
|
|
0badea874d | ||
|
|
0dd6193eaa | ||
|
|
d10778430f | ||
|
|
da8e09846d | ||
|
|
310fe4a42c | ||
|
|
a7ef8889dd | ||
|
|
f41229cb5b | ||
|
|
8ab4e3ca49 | ||
|
|
ff8d37443e | ||
|
|
fcd46629c4 | ||
|
|
9ad9eb083b | ||
|
|
4669507880 | ||
|
|
9348f4f366 | ||
|
|
47bb634035 | ||
|
|
10b34d6b33 | ||
|
|
f061ca2b15 | ||
|
|
bc24dbea29 | ||
|
|
2d4bf73632 | ||
|
|
c21d8cc8dc | ||
|
|
cc6a91a477 | ||
|
|
21a2725215 | ||
|
|
411b8c74cc | ||
|
|
91bf312e4d | ||
|
|
277b8b122c | ||
|
|
59c49d2d31 | ||
|
|
16024c95aa | ||
|
|
923ed80fcf | ||
|
|
2ee2916f78 | ||
|
|
79bf1b427f | ||
|
|
c21c329aa0 | ||
|
|
6b3af4bc21 | ||
|
|
ffbdf03847 | ||
|
|
8d08bfb3f9 | ||
|
|
1af6326b97 | ||
|
|
31630a84be | ||
|
|
0601bf3197 | ||
|
|
826d2866fa | ||
|
|
ba00498d83 | ||
|
|
4da40e8d52 | ||
|
|
d54d989c40 | ||
|
|
18ecbba87c | ||
|
|
9b417486a9 | ||
|
|
6d208ac496 | ||
|
|
8990c145fb | ||
|
|
e5b1c3c829 | ||
|
|
6c7128c0f6 | ||
|
|
f31f1fa40b | ||
|
|
2ceba79b9f | ||
|
|
1c8a59c1a9 | ||
|
|
298bf91293 | ||
|
|
b25d592d99 | ||
|
|
875edb5f76 | ||
|
|
84d97ab323 | ||
|
|
aeaffd620e | ||
|
|
3f7091cbf2 | ||
|
|
a95f791215 | ||
|
|
af2aeaa66b | ||
|
|
855fd8bf9a | ||
|
|
a937aff80f | ||
|
|
5597d45aed | ||
|
|
473d2eabfa | ||
|
|
78c8c8e64d | ||
|
|
7335d012ef | ||
|
|
99be38604a | ||
|
|
d35d340613 | ||
|
|
f05ca137be | ||
|
|
e718caca77 | ||
|
|
693a3346d2 | ||
|
|
1f75b6a49e | ||
|
|
4ece422f48 | ||
|
|
607677f98f | ||
|
|
e1e7be63b8 | ||
|
|
e1314a9d1e | ||
|
|
1ee682f06f | ||
|
|
ba80c0e594 | ||
|
|
fbdbcc8070 | ||
|
|
24a872885a | ||
|
|
005a131440 | ||
|
|
0b1cb16353 | ||
|
|
1a3eaf7e6a | ||
|
|
971252d5d1 | ||
|
|
c5ba9b263e | ||
|
|
c01716a6c0 | ||
|
|
0fbf131df3 | ||
|
|
4bd0806e58 | ||
|
|
4cbb71705e | ||
|
|
8ab4dd6b2c | ||
|
|
318c369752 | ||
|
|
cea3b7630c | ||
|
|
19211d4417 | ||
|
|
c312d1b3a6 | ||
|
|
e4ea1ccbfb | ||
|
|
5b35c4e8f0 | ||
|
|
9967857a57 | ||
|
|
78c560a3be | ||
|
|
07a12b66e8 | ||
|
|
a28c9d09c8 | ||
|
|
b2ab13187f | ||
|
|
5ec696fbfd | ||
|
|
44bd0ae847 | ||
|
|
5809ba093d | ||
|
|
17aee92dc9 | ||
|
|
6f1d2e7690 | ||
|
|
75f57fc517 | ||
|
|
f589410496 | ||
|
|
aecc7c6643 | ||
|
|
c5c1b62939 | ||
|
|
adc92a2b52 | ||
|
|
9636ca22e7 | ||
|
|
0442770c9d | ||
|
|
3b6a879fc9 | ||
|
|
a1b4c9acd6 | ||
|
|
0f90dee224 | ||
|
|
159e84468c | ||
|
|
59e4ac30c2 | ||
|
|
b5ce948bc1 | ||
|
|
076af7ef43 | ||
|
|
43bef9447c | ||
|
|
b1c5fe0b4d | ||
|
|
39d7f2e46e | ||
|
|
315661208d | ||
|
|
e175e43082 | ||
|
|
cb9b320ed7 | ||
|
|
1d906fb0c7 | ||
|
|
c544e26206 | ||
|
|
1eaac35a3e | ||
|
|
cc7962fc2e | ||
|
|
3638984d67 | ||
|
|
62e77c69f5 | ||
|
|
774d813576 | ||
|
|
688caf4453 | ||
|
|
eeb5b3c139 | ||
|
|
b5e6dd1060 | ||
|
|
272d818050 | ||
|
|
0fc1bd7a8c | ||
|
|
2b234e0713 | ||
|
|
9e8cb17ecb | ||
|
|
e0230864ce | ||
|
|
9e89b3e40e | ||
|
|
036663ae62 | ||
|
|
b62321689d | ||
|
|
bdd271e78f | ||
|
|
4a56d38782 | ||
|
|
70a7ca95fe | ||
|
|
f85e61d8ed | ||
|
|
ac8c641d64 | ||
|
|
d81502e191 | ||
|
|
8c35d355e5 | ||
|
|
72b004664f | ||
|
|
49a7408968 | ||
|
|
17738ad248 | ||
|
|
425c952293 | ||
|
|
03d2d0e237 | ||
|
|
edfc2c6124 | ||
|
|
545451a829 | ||
|
|
12ff7015cd | ||
|
|
f2f212edba | ||
|
|
69446491b8 | ||
|
|
d5beaa0937 | ||
|
|
cc970c499b | ||
|
|
0087b16912 | ||
|
|
5520d518b5 | ||
|
|
b3e9240aa8 | ||
|
|
137a270488 | ||
|
|
e36610d63c | ||
|
|
c4273fbef0 | ||
|
|
a4e9f3100f | ||
|
|
2668ceac0e | ||
|
|
fb2fb5c38c | ||
|
|
56175df121 | ||
|
|
7d3daa9fe6 | ||
|
|
1d2de27536 | ||
|
|
aab42e56bd | ||
|
|
1368a6e074 | ||
|
|
80990b959c | ||
|
|
053b6d1c80 | ||
|
|
692d8e8e1a | ||
|
|
5e71dcbaf0 | ||
|
|
83958604d5 | ||
|
|
dcd8a6bb0e | ||
|
|
7161a0edb9 | ||
|
|
0fd160e6e6 | ||
|
|
ca8b2dd83e | ||
|
|
79d1da5ff8 | ||
|
|
d73252b11a | ||
|
|
1f5c56cac3 | ||
|
|
18db2aa970 | ||
|
|
a7a53f9381 | ||
|
|
f4b2a05fff | ||
|
|
cdd2b6bfc1 | ||
|
|
26bd27cd32 | ||
|
|
613feb0808 | ||
|
|
d4849443e5 | ||
|
|
db70f7e98a | ||
|
|
37ddaeca12 | ||
|
|
2c546b2cde | ||
|
|
c48d195951 | ||
|
|
149fa625e0 | ||
|
|
16cda60f31 | ||
|
|
0d54f6215f | ||
|
|
a9774a293f | ||
|
|
ee8cf08c0e | ||
|
|
ea481e6c5a | ||
|
|
36df9e8283 | ||
|
|
0467b65e69 | ||
|
|
68f1dda7b0 | ||
|
|
5b7f7c87bf | ||
|
|
153022a1a7 | ||
|
|
1098ca0494 | ||
|
|
d4a7dbf805 | ||
|
|
56ee815eec | ||
|
|
fe43764bcb | ||
|
|
93be8b9e31 | ||
|
|
2448d51ed5 | ||
|
|
2804fa4448 | ||
|
|
90a119bb2c | ||
|
|
69ae60b4bb | ||
|
|
dca4af942f | ||
|
|
2b7e135839 | ||
|
|
d565aa0b3f | ||
|
|
8b53140581 | ||
|
|
917bae568d | ||
|
|
efb6af4d51 | ||
|
|
2361d8bdc1 | ||
|
|
5fe8022b15 | ||
|
|
7931397595 | ||
|
|
0501f52805 | ||
|
|
108d2a033b | ||
|
|
17a9cfd65c | ||
|
|
7069ffa37d | ||
|
|
8eb740cf67 | ||
|
|
7803eb2e5a | ||
|
|
b73e50e64b | ||
|
|
c73148a1fb | ||
|
|
65a02190dc | ||
|
|
74b49fe69f | ||
|
|
26e32472d5 | ||
|
|
20d8b2963c | ||
|
|
d21313e590 | ||
|
|
81e9c9bf74 | ||
|
|
382226ff9a | ||
|
|
9511040d5a | ||
|
|
049936f4a6 | ||
|
|
15161dd429 | ||
|
|
efc5982860 | ||
|
|
acb95fdcdf | ||
|
|
f1b19f10cc | ||
|
|
6eaf1ac2a0 | ||
|
|
01ee1e1468 | ||
|
|
83dc295b7b | ||
|
|
4a28c620a4 | ||
|
|
e50ac69358 | ||
|
|
aef561d78b | ||
|
|
4ef4a4816f | ||
|
|
8533bfa7c5 | ||
|
|
6389e961b9 | ||
|
|
6359fc608e | ||
|
|
314400a1bd | ||
|
|
a3de065c93 | ||
|
|
99207b7221 | ||
|
|
e3c3fafa1e | ||
|
|
5b3103e41b | ||
|
|
9742da348b | ||
|
|
94a31922f7 | ||
|
|
1937eb17c4 | ||
|
|
f291ea515a | ||
|
|
56522f39c0 | ||
|
|
ff4aba9423 | ||
|
|
9ab256df30 | ||
|
|
88b8ca8a49 | ||
|
|
09f637496c | ||
|
|
826d269e0e | ||
|
|
5163baf9c1 | ||
|
|
ab27203100 | ||
|
|
88c2051605 | ||
|
|
f510c2a433 | ||
|
|
66982c86a9 | ||
|
|
ec140d5d8a | ||
|
|
ad96b006f9 | ||
|
|
32507d74f7 | ||
|
|
2a00d00e44 | ||
|
|
40c5f6e3ab | ||
|
|
ae67918c8a | ||
|
|
96e73f3476 | ||
|
|
c1b05402ef | ||
|
|
1dd1bad9c2 | ||
|
|
9b873d2f48 | ||
|
|
5984ae7ce9 | ||
|
|
903cd86cdd | ||
|
|
5d9a187659 | ||
|
|
a9c94ca22d | ||
|
|
a285db1b08 | ||
|
|
13bf8125e3 | ||
|
|
b06dcf0a0d | ||
|
|
a800f6ddf0 | ||
|
|
a89306b6bf | ||
|
|
c3120e93cf | ||
|
|
defe9506bb | ||
|
|
55d3bb0ec3 | ||
|
|
005a8b3008 | ||
|
|
a741fdfc44 | ||
|
|
944b94aae8 | ||
|
|
37eaf634c4 | ||
|
|
edcb06a415 | ||
|
|
62b50dc0ae | ||
|
|
e7c57dcabc | ||
|
|
094d9c6497 | ||
|
|
3dc5a9de7b | ||
|
|
4e29041291 | ||
|
|
6b6a29fdb7 | ||
|
|
2d61ba629f | ||
|
|
5a2075019c | ||
|
|
944200ca8c | ||
|
|
40d8e96811 | ||
|
|
7e8ca842a2 | ||
|
|
b808d08eb3 | ||
|
|
eb11da9b19 | ||
|
|
68b31a09b4 | ||
|
|
cab8416596 | ||
|
|
94f4e6714a | ||
|
|
2ddd6b051d | ||
|
|
c506fc5c75 | ||
|
|
d306ece0d4 | ||
|
|
d29f103910 | ||
|
|
2386a3a79b | ||
|
|
ec51333b56 | ||
|
|
75829812f3 | ||
|
|
4d720370b1 | ||
|
|
7d7dd665aa | ||
|
|
b6bee47481 | ||
|
|
0ec8739d5a | ||
|
|
7d09d1402f | ||
|
|
5492ab9081 | ||
|
|
7f6bbc0648 | ||
|
|
b1d4072c0b | ||
|
|
562d23df11 | ||
|
|
2c13c04f86 | ||
|
|
7365bfad9f | ||
|
|
1af1cf212c | ||
|
|
5c7d279104 | ||
|
|
cfe8e17223 | ||
|
|
91d4bf3cd9 | ||
|
|
df9642c325 | ||
|
|
af98b0eb4d | ||
|
|
c589e89bf0 |
No files matched your search
@@ -1,48 +0,0 @@
|
||||
version: 1.0.{build}
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
|
||||
environment:
|
||||
GH_TOKEN:
|
||||
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
|
||||
key_pass:
|
||||
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
|
||||
pfx_key:
|
||||
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
|
||||
AWS_ACCESS_KEY_ID:
|
||||
secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
|
||||
|
||||
#notifications:
|
||||
# - provider: Slack
|
||||
# incoming_webhook:
|
||||
# secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
|
||||
|
||||
|
||||
clone_folder: c:\projects\lbry
|
||||
|
||||
test_script:
|
||||
- cd C:\projects\lbry\
|
||||
- pip install cython
|
||||
- pip install mock pylint unqlite Faker
|
||||
- pip install .
|
||||
- pylint lbrynet
|
||||
# disable tests for now so that appveyor can build the app
|
||||
- set PYTHONPATH=.
|
||||
- trial lbrynet.tests
|
||||
# TODO: integration tests do not work
|
||||
#- python -m unittest discover tests/integration
|
||||
#- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
|
||||
|
||||
build_script:
|
||||
- cd C:\projects\lbry\build\
|
||||
- ps: .\build.ps1
|
||||
|
||||
|
||||
artifacts:
|
||||
- path: build\dist\*.zip
|
||||
name: lbrynet-daemon
|
||||
@@ -4,7 +4,7 @@ Thanks for reporting an issue to LBRY and helping us improve!
|
||||
To make it possible for us to help you, please fill out below information carefully.
|
||||
|
||||
Before reporting any issues, please make sure that you're using the latest version.
|
||||
- App: https://github.com/lbryio/lbry-app/releases
|
||||
- App: https://github.com/lbryio/lbry-desktop/releases
|
||||
- Daemon: https://github.com/lbryio/lbry/releases
|
||||
|
||||
We are also available on Discord at https://chat.lbry.io
|
||||
@@ -51,3 +51,24 @@ Tell us what happens instead
|
||||
|
||||
## Screenshots
|
||||
<!-- If a screenshot would help explain the bug, please include one or two here -->
|
||||
|
||||
|
||||
|
||||
## Internal Use
|
||||
|
||||
### Acceptance Criteria
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
### Definition of Done
|
||||
- [ ] Tested against acceptance criteria
|
||||
- [ ] Tested against the assumptions of user story
|
||||
- [ ] The project builds without errors
|
||||
- [ ] Unit tests are written and passing
|
||||
- [ ] Tests on devices/browsers listed in the issue have passed
|
||||
- [ ] QA performed & issues resolved
|
||||
- [ ] Refactoring completed
|
||||
- [ ] Any configuration or build changes documented
|
||||
- [ ] Documentation updated
|
||||
- [ ] Peer Code Review performed
|
||||
@@ -0,0 +1,38 @@
|
||||
## PR Checklist
|
||||
Please check all that apply to this PR using "x":
|
||||
|
||||
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
|
||||
- [ ] I have checked that this PR does not introduce a breaking change
|
||||
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
|
||||
|
||||
|
||||
## PR Type
|
||||
What kind of change does this PR introduce?
|
||||
|
||||
Why is this change necessary?
|
||||
|
||||
<!-- Please check all that apply to this PR using "x". -->
|
||||
|
||||
- [ ] Bugfix
|
||||
- [ ] Feature
|
||||
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
|
||||
- [ ] Code style update (formatting)
|
||||
- [ ] Refactoring (no functional changes)
|
||||
- [ ] Documentation changes
|
||||
- [ ] Other - Please describe:
|
||||
|
||||
## Fixes
|
||||
|
||||
Issue Number: N/A
|
||||
|
||||
|
||||
## What is the current behavior?
|
||||
|
||||
|
||||
## What is the new behavior?
|
||||
|
||||
|
||||
## Other information
|
||||
|
||||
|
||||
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
|
||||
@@ -1,25 +1,9 @@
|
||||
*.pyc
|
||||
*.egg
|
||||
*.so
|
||||
*.xml
|
||||
*.iml
|
||||
*.log
|
||||
*.pem
|
||||
*.decTest
|
||||
*.prof
|
||||
.#*
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
|
||||
/build/build
|
||||
/build/dist
|
||||
/bulid/requirements_base.txt
|
||||
|
||||
/lbrynet.egg-info
|
||||
/docs_build
|
||||
/lbry-venv
|
||||
|
||||
.idea/
|
||||
.coverage
|
||||
.DS_Store
|
||||
|
||||
# temporary files from the twisted.trial test runner
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
ignore=CVS,schema
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
@@ -62,7 +62,7 @@ confidence=
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# disable everything first and then re-enable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
@@ -121,7 +121,11 @@ disable=
|
||||
unidiomatic-typecheck,
|
||||
global-at-module-level,
|
||||
inconsistent-return-statements,
|
||||
keyword-arg-before-vararg
|
||||
keyword-arg-before-vararg,
|
||||
assignment-from-no-return,
|
||||
useless-return,
|
||||
assignment-from-none,
|
||||
stop-iteration-return
|
||||
|
||||
|
||||
[REPORTS]
|
||||
@@ -386,7 +390,7 @@ int-import-graph=
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
max-args=10
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
@@ -405,7 +409,7 @@ max-branches=12
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
max-parents=8
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
@@ -1,46 +1,128 @@
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
dist: xenial
|
||||
language: python
|
||||
python: 2.7
|
||||
python: "3.7"
|
||||
|
||||
branches:
|
||||
except:
|
||||
- gh-pages
|
||||
jobs:
|
||||
include:
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4
|
||||
# newer astroid fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
- &tests
|
||||
stage: test
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .[test]
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -v tests.unit.wallet
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.components tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.test_cli tests.unit.test_customLogger tests.unit.schema
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
|
||||
script: tox
|
||||
after_success:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
services:
|
||||
- docker
|
||||
install:
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-windows.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-windows.zip
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- &build
|
||||
name: "Linux"
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- python scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
|
||||
- ./dist/lbrynet --version
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- <<: *build
|
||||
name: "Mac"
|
||||
os: osx
|
||||
osx_image: xcode8.3
|
||||
language: generic
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
addons:
|
||||
#srcclr:
|
||||
# debug: true
|
||||
apt:
|
||||
packages:
|
||||
- libgmp3-dev
|
||||
- build-essential
|
||||
- git
|
||||
- libssl-dev
|
||||
- libffi-dev
|
||||
|
||||
before_install:
|
||||
- virtualenv venv
|
||||
- source venv/bin/activate
|
||||
|
||||
install:
|
||||
- pip install -U pip==9.0.3
|
||||
- pip install -r requirements.txt
|
||||
- pip install -r requirements_testing.txt
|
||||
- pip install .
|
||||
|
||||
script:
|
||||
- pip install mock pylint
|
||||
- pylint lbrynet
|
||||
- PYTHONPATH=. trial lbrynet.tests
|
||||
- rvm install ruby-2.3.1
|
||||
- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
- $TRAVIS_BUILD_DIR/.tox
|
||||
@@ -4,33 +4,198 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/) with
|
||||
regard to the json-rpc api. As we're currently pre-1.0 release, we
|
||||
can and probably will change functionality and break backwards compatability
|
||||
can and probably will change functionality and break backwards compatibility
|
||||
at anytime.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.30.1] - 2018-12-09
|
||||
|
||||
### API
|
||||
* add `aioupnp_version` to `upnp` field in the response to `status`
|
||||
* fixed uncaught claim signature errors
|
||||
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
|
||||
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
|
||||
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
|
||||
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
|
||||
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
|
||||
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
|
||||
|
||||
### Schema
|
||||
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
|
||||
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
|
||||
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
|
||||
|
||||
### Wallet
|
||||
* fixed resolving names with invalid channel signatures
|
||||
* fixed wallet sync issues
|
||||
* fixed wallet locking/encryption issues
|
||||
|
||||
### DHT
|
||||
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
|
||||
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
|
||||
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
|
||||
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
|
||||
|
||||
### Database
|
||||
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
|
||||
|
||||
### Misc
|
||||
* switch to Python 3.7
|
||||
* brought back icon for windows binary
|
||||
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
|
||||
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
|
||||
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
|
||||
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
|
||||
|
||||
## [0.30.0] - 2018-10-22
|
||||
This release is the most ambitious upgrade to lbrynet yet; including a brand
|
||||
new wallet implementation, switch to Python 3 of the entire code base and
|
||||
many changes to the API such as pagination and a more consistent output from
|
||||
most commands.
|
||||
|
||||
|
||||
### Security
|
||||
*
|
||||
*
|
||||
* upgraded `cryptography` package.
|
||||
* switch to `coincurve` for private / public key generation (addresses)
|
||||
* support for connecting to daemon via SSL
|
||||
* improved workflow for encrypt/decrypt of wallet
|
||||
|
||||
### API
|
||||
* unified all command line executables into a single `lbrynet` executable.
|
||||
* added `use_https` configuration setting (off by default), if true the api
|
||||
server will generate a self signed ssl certificate when started and only
|
||||
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
|
||||
* deprecated `daemon_stop` command, use `stop` instead.
|
||||
* deprecated `wallet_balance` command, use `account_balance` instead.
|
||||
* deprecated `wallet_unlock` command, use `account_unlock` instead.
|
||||
* deprecated `wallet_decrypt` command, use `account_decrypt` instead.
|
||||
* deprecated `wallet_encrypt` command, use `account_encrypt` instead.
|
||||
* deprecated `wallet_prefill_addresses` command, use `account_fund` instead.
|
||||
* deprecated `wallet_list` command, use `address_list` instead.
|
||||
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
|
||||
* deprecated `wallet_public_key` command.
|
||||
* deprecated `wallet_new_address` command.
|
||||
* deprecated `wallet_unused_address` command, use `address_unused` instead.
|
||||
* deprecated `claim_renew` command.
|
||||
* added `account_list` command to list accounts including their balance.
|
||||
* added `account_add` command to add a previously created account from seed or private key.
|
||||
* added `account_create` command to generate a new account.
|
||||
* added `account_remove` command to remove an account from wallet.
|
||||
* added `account_set` command to change a setting on an account.
|
||||
* added `account_balance` command to get just the account balance.
|
||||
* added `account_unlock` command to unlock an encrypted account.
|
||||
* added `account_lock` command to lock an encrypted account.
|
||||
* added `account_encrypt` command to encrypt an account.
|
||||
* added `account_decrypt` command to decrypt an account.
|
||||
* added `account_fund` command to move funds between or within an account in various ways.
|
||||
* added `account_max_address_gap` command to find large gaps of unused addresses.
|
||||
* added `address_list` command to list addresses.
|
||||
* added `address_is_mine` command to check if an address is one of your addresses.
|
||||
* added `address_unused` command to get existing or generate a new unused address.
|
||||
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
|
||||
`transaction_list` and `utxo_list`.
|
||||
* added `upnp` field to `status` response
|
||||
* removed `send_amount_to_address` command previously marked as deprecated
|
||||
* removed `channel_list_mine` command previously marked as deprecated
|
||||
* removed `get_availability` command previously marked as deprecated
|
||||
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
|
||||
|
||||
### Wallet
|
||||
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
|
||||
* changed wallet file format to support multiple accounts in one wallet.
|
||||
* moved transaction data from wallet file into an sqlite database.
|
||||
* changed channel certificates to be keyed by txid:nout instead of claim_id which
|
||||
makes it possible to recover old certificates.
|
||||
|
||||
### File Manager
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### DHT
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### P2P
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### Reflector
|
||||
* No major changes to reflector.
|
||||
|
||||
### Database
|
||||
* No changes to existing storage module.
|
||||
|
||||
### Known Issues
|
||||
* Channel certificate import/export is currently not working.
|
||||
|
||||
## [0.21.2] - 2018-08-23
|
||||
### Fixed
|
||||
* daemon cli spelling fixes
|
||||
*
|
||||
* issue in dht ping queue where enqueued pings that aren't yet due wouldn't be rescheduled
|
||||
* blob mirror downloader not finishing streams that were partially uploaded at the time of the download attempt (https://github.com/lbryio/lbry/issues/1376)
|
||||
|
||||
### Deprecated
|
||||
*
|
||||
*
|
||||
|
||||
### Changed
|
||||
*
|
||||
*
|
||||
## [0.21.1] - 2018-08-13
|
||||
### Fixed
|
||||
* `download_progress` field in `blockchain_headers` section of `status` not initializing correctly when resuming a download (https://github.com/lbryio/lbry/issues/1355)
|
||||
* `wallet_send` not accepting decimal amounts (https://github.com/lbryio/lbry/issues/1356 commit https://github.com/lbryio/lbry/commit/1098ca0494ece420c70fc57f69d6d388715a99b8)
|
||||
|
||||
### Added
|
||||
*
|
||||
*
|
||||
* `is_locked` to `wallet` in `status` response (https://github.com/lbryio/lbry/issues/1354, commit https://github.com/lbryio/lbry/commit/153022a1a7122ab6d31d3db433dccbe469bbcb3c)
|
||||
|
||||
### Changed
|
||||
* Bumped `lbryum` requirement to 3.2.4 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#324---2018-08-13)
|
||||
|
||||
## [0.21.0] - 2018-08-09
|
||||
### Fixed
|
||||
* check `claim_address` and `change_address` earlier on publishing, to avoid hard to understand errors later in the process (pr https://github.com/lbryio/lbry/pull/1347)
|
||||
* loggly error reporting not following `share_usage_data` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* improper error handling when data is not valid JSON (pr https://github.com/lbryio/lbry/pull/1326)
|
||||
* blob mirroring being set in unrelated tests, making them fail (pr https://github.com/lbryio/lbry/pull/1348)
|
||||
* http blob mirroring edge cases (pr https://github.com/lbryio/lbry/pull/1315)
|
||||
* external ports in log messages not showing the correct external port from the upnp redirects (https://github.com/lbryio/lbry/issues/1338) (pr https://github.com/lbryio/lbry/pull/1349)
|
||||
* miniupnpc fallback issues in txupnp (https://github.com/lbryio/lbry/issues/1341) (pr https://github.com/lbryio/lbry/pull/1349)
|
||||
* upnp error when disabled on router and a non-gateway is found, such as chromecast (https://github.com/lbryio/lbry/issues/1352) (https://github.com/lbryio/lbry/commit/dca4af942fbe95547794213775f0a62cd04a393f)
|
||||
|
||||
### Deprecated
|
||||
* automatic claim renew, this is no longer needed
|
||||
|
||||
### Changed
|
||||
* api server class to use components, and for all JSONRPC API commands to be callable so long as the required components are available. (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* return error messages when required conditions on components are not met for API calls (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* `status` to no longer return a base58 encoded `lbry_id`, instead return this as the hex encoded `node_id` in a new `dht` field. (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* `startup_status` field in the response to `status` to be a dict of component names to status booleans (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* renamed the `blockchain_status` field in the response to `status` to `wallet` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* moved and renamed `wallet_is_encrypted` to `is_encrypted` in the `wallet` field in the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* moved wallet, upnp and dht startup code from `Session` to `Components` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* attempt blob downloads from http mirror sources (by default) concurrently to p2p sources (pr https://github.com/lbryio/lbry/pull/1233)
|
||||
* replace miniupnpc with [txupnp](https://github.com/lbryio/txupnp). Since txupnp is still under development, it will internally fall back to miniupnpc. (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* simplified test_misc.py in the functional tests (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* update `cryptography` requirement to 2.3 (pr https://github.com/lbryio/lbry/pull/1333)
|
||||
|
||||
### Added
|
||||
* `skipped_components` list to the response from `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* component statuses (`blockchain_headers`, `dht`, `wallet`, `blob_manager` `hash_announcer`, and `file_manager`) to the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* `skipped_components` config setting, accepts a list of names of components to not run (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* `ComponentManager` for managing the life-cycles of dependencies (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* `requires` decorator to register the components required by a `jsonrpc_` command, to facilitate commands registering asynchronously (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* unit tests for `ComponentManager` (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
* script to generate docs/api.json file (https://github.com/lbryio/lbry.tech/issues/42)
|
||||
* additional information to the balance error message when editing a claim (pr https://github.com/lbryio/lbry/pull/1309)
|
||||
* `address` and `port` arguments to `peer_ping` (https://github.com/lbryio/lbry/issues/1313) (pr https://github.com/lbryio/lbry/pull/1299)
|
||||
* ability to download from HTTP mirrors by setting `download_mirrors` (prs https://github.com/lbryio/lbry/pull/1233 and https://github.com/lbryio/lbry/pull/1315)
|
||||
* ability to filter peers from an iterative find value operation (finding peers for a blob). This is used to filter peers we've already found for a blob when accumulating the list of peers. (pr https://github.com/lbryio/lbry/pull/1287)
|
||||
|
||||
### Removed
|
||||
*
|
||||
*
|
||||
* `session_status` argument and response field from `status` (pr https://github.com/lbryio/lbry/pull/1328)
|
||||
* most of the internal attributes from `Daemon` (pr https://github.com/lbryio/lbry/pull/1294)
|
||||
|
||||
|
||||
## [0.20.4] - 2018-07-18
|
||||
### Fixed
|
||||
* spelling errors in messages printed by `lbrynet-cli`
|
||||
* high CPU usage when a stream is incomplete and the peers we're requesting from have no more blobs to send us (https://github.com/lbryio/lbry/pull/1301)
|
||||
|
||||
### Changed
|
||||
* keep track of failures for DHT peers for up to ten minutes instead of indefinitely (https://github.com/lbryio/lbry/pull/1300)
|
||||
* skip ignored peers from iterative lookups instead of blocking the peer who returned them to us too (https://github.com/lbryio/lbry/pull/1300)
|
||||
* if a node becomes ignored during an iterative find cycle remove it from the shortlist so that we can't return it as a result nor try to probe it anyway (https://github.com/lbryio/lbry/pull/1303)
|
||||
|
||||
|
||||
## [0.20.3] - 2018-07-03
|
||||
@@ -321,7 +486,7 @@ at anytime.
|
||||
* Fixed https://github.com/lbryio/lbry/issues/923
|
||||
* Fixed concurrent reflects opening too many files
|
||||
* Fixed cases when reflecting would fail on error conditions
|
||||
* Fixed deadlocks from occuring during blob writes
|
||||
* Fixed deadlocks from occurring during blob writes
|
||||
* Fixed and updated`lbrynet.tests.dht`
|
||||
* Fixed redundant dht id
|
||||
* Fixed dht `ping` method
|
||||
@@ -708,7 +873,7 @@ at anytime.
|
||||
* Removed check_pending logic from Daemon
|
||||
* Switched to txrequests so requests can use twisted event loop
|
||||
* Renamed API command file_seed to file_set_status
|
||||
* Dont add expected payment to wallet when payment rate is 0
|
||||
* Don't add expected payment to wallet when payment rate is 0
|
||||
### Fixed
|
||||
* Fix restart procedure in DaemonControl
|
||||
* Create download directory if it doesn't exist
|
||||
@@ -739,7 +904,7 @@ at anytime.
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
* Made dictionary key names in API commmand outputs to be more consistent
|
||||
* Made dictionary key names in API command outputs to be more consistent
|
||||
### Added
|
||||
* Add file filters: `claim_id`, `outpoint`, and `rowid`
|
||||
* Make loggly logs less verbose
|
||||
@@ -791,7 +956,7 @@ at anytime.
|
||||
* Include download url in version check
|
||||
|
||||
### Fixed
|
||||
* add misssing traceback to logging
|
||||
* add missing traceback to logging
|
||||
|
||||
## [0.8.3] - 2017-02-15
|
||||
### Fixed
|
||||
|
||||
@@ -1,70 +1,48 @@
|
||||
# Installing LBRY
|
||||
|
||||
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
|
||||
[](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, you'll need to install some additional libraries depending on your operating system.
|
||||
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
|
||||
### Python
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
LBRY requires Python 2.7. Python 3 is not supported.
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### OS X Prerequisites
|
||||
### macOS
|
||||
|
||||
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install mpfr libmpc protobuf
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
brew install python3 protobuf
|
||||
```
|
||||
|
||||
### Linux Prerequisites
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 16.04), install the following:
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux flavor, install the equivalent of the above packages for your system.
|
||||
|
||||
### Windows Prerequisites
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
|
||||
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- requests
|
||||
|
||||
Install the above binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -73,35 +51,27 @@ To install:
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
python --version # Python 3 is not supported. Make sure you're on Python 2.7
|
||||
|
||||
pip install -U -r requirements.txt
|
||||
pip install -U -r requirements_testing.txt
|
||||
pip install --editable .
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
## Run the tests
|
||||
To run the unit tests from the repo directory:
|
||||
```
|
||||
trial --reactor=asyncio tests.unit
|
||||
```
|
||||
|
||||
## Making a Change
|
||||
## Usage
|
||||
|
||||
If you are not already inside the virtual environment, run:
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
|
||||
```
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
This will install `lbrynet-daemon` in such a way that the changes you make to the code will be used as soon as you restart the daemon.
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,36 +1,35 @@
|
||||
# LBRY
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
[](https://coveralls.io/github/lbryio/lbry)
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules. LBRY also provides a data network that consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
This project aims to provide a daemon that can interact with the network via a json-rpc interface over HTTP.
|
||||
|
||||
The project is written in python2.7 and extensively uses Twisted framework.
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
| | Windows | macOS | Linux |
|
||||
| --------------------- | -------------------------------------------- | -------------------------------------------- | -------------------------------------------- |
|
||||
| Latest Stable Release | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.0/lbrynet-daemon-v0.20.0-windows.zip) | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.0/lbrynet-daemon-v0.20.0-macos.zip) | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.0/lbrynet-daemon-v0.20.0-linux.zip) |
|
||||
| Latest Pre-release | [Download](https://github.com/lbryio/lbry/releases/download/untagged-99114fa31abbfe3a5ef4/lbrynet-daemon-v0.20.1rc3-windows.zip) | [Download](https://github.com/lbryio/lbry/releases/download/untagged-99114fa31abbfe3a5ef4/lbrynet-daemon-v0.20.1rc3-macos.zip) | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.1rc4/lbrynet-daemon-v0.20.1rc4-linux.zip) |
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows.
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
|
||||
## Usage
|
||||
|
||||
Run `lbrynet-daemon` to launch the daemon.
|
||||
Run `lbrynet start` to launch the API server.
|
||||
|
||||
By default, `lbrynet-daemon` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
|
||||
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
|
||||
|
||||
The full API is documented [here](https://lbryio.github.io/lbry/cli).
|
||||
With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
|
||||
## Running from source
|
||||
|
||||
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -42,12 +41,16 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. Our PGP key is here if you need it.
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is @jackrobison(jack@lbry.io)
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
The documentation for the api can be found [here](https://lbry.io/api).
|
||||
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
|
||||
@@ -1,33 +0,0 @@
|
||||
$env:Path += ";C:\MinGW\bin\"
|
||||
|
||||
$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
|
||||
gcc --version
|
||||
mingw32-make --version
|
||||
|
||||
# build/install miniupnpc manually
|
||||
tar zxf miniupnpc-1.9.tar.gz
|
||||
cd miniupnpc-1.9
|
||||
mingw32-make -f Makefile.mingw
|
||||
python setupmingw32.py build --compiler=mingw32
|
||||
python setupmingw32.py install
|
||||
cd ..\
|
||||
Remove-Item -Recurse -Force miniupnpc-1.9
|
||||
|
||||
# copy requirements from lbry, but remove miniupnpc (installed manually)
|
||||
Get-Content ..\requirements.txt | Select-String -Pattern 'miniupnpc' -NotMatch | Out-File requirements_base.txt
|
||||
|
||||
python set_build.py
|
||||
|
||||
pip install -r requirements.txt
|
||||
pip install ..\.
|
||||
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
pyinstaller -y console.onefile.spec
|
||||
|
||||
nuget install secure-file -ExcludeVersion
|
||||
secure-file\tools\secure-file -decrypt .\lbry2.pfx.enc -secret "$env:pfx_key"
|
||||
signtool.exe sign /f .\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
|
||||
|
||||
python zip_daemon.py
|
||||
python upload_assets.py
|
||||
@@ -1,59 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
|
||||
cd "$ROOT"
|
||||
BUILD_DIR="$ROOT/build"
|
||||
|
||||
FULL_BUILD="${FULL_BUILD:-false}"
|
||||
if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then
|
||||
FULL_BUILD="true"
|
||||
fi
|
||||
|
||||
[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build"
|
||||
[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# install dependencies
|
||||
$BUILD_DIR/prebuild.sh
|
||||
|
||||
VENV="$BUILD_DIR/venv"
|
||||
if [ -d "$VENV" ]; then
|
||||
rm -rf "$VENV"
|
||||
fi
|
||||
virtualenv "$VENV"
|
||||
set +u
|
||||
source "$VENV/bin/activate"
|
||||
set -u
|
||||
|
||||
# must set build before installing lbrynet. otherwise it has no effect
|
||||
python "$BUILD_DIR/set_build.py"
|
||||
fi
|
||||
|
||||
cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt"
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pip install -r requirements.txt
|
||||
)
|
||||
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
pyinstaller -y console.onefile.spec
|
||||
)
|
||||
|
||||
python "$BUILD_DIR/zip_daemon.py"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# electron-build has a publish feature, but I had a hard time getting
|
||||
# it to reliably work and it also seemed difficult to configure. Not proud of
|
||||
# this, but it seemed better to write my own.
|
||||
python "$BUILD_DIR/upload_assets.py"
|
||||
|
||||
deactivate
|
||||
fi
|
||||
|
||||
echo 'Build complete.'
|
||||
@@ -1,42 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-cli', pathex=[cwd])
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-cli',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
import lbryum
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
datas = [
|
||||
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
|
||||
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
|
||||
]
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-console', pathex=[cwd], datas=datas)
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-console',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
import lbryum
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
datas = [
|
||||
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
|
||||
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
|
||||
]
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-daemon', pathex=[cwd], datas=datas)
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-daemon',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,47 +0,0 @@
|
||||
# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point
|
||||
def Entrypoint(dist, group, name,
|
||||
scripts=None, pathex=None, binaries=None, datas=None,
|
||||
hiddenimports=None, hookspath=None, excludes=None, runtime_hooks=None,
|
||||
cipher=None, win_no_prefer_redirects=False, win_private_assemblies=False):
|
||||
import pkg_resources
|
||||
|
||||
# get toplevel packages of distribution from metadata
|
||||
def get_toplevel(dist):
|
||||
distribution = pkg_resources.get_distribution(dist)
|
||||
if distribution.has_metadata('top_level.txt'):
|
||||
return list(distribution.get_metadata('top_level.txt').split())
|
||||
else:
|
||||
return []
|
||||
|
||||
hiddenimports = hiddenimports or []
|
||||
packages = []
|
||||
for distribution in hiddenimports:
|
||||
packages += get_toplevel(distribution)
|
||||
|
||||
scripts = scripts or []
|
||||
pathex = pathex or []
|
||||
# get the entry point
|
||||
ep = pkg_resources.get_entry_info(dist, group, name)
|
||||
# insert path of the egg at the verify front of the search path
|
||||
pathex = [ep.dist.location] + pathex
|
||||
# script name must not be a valid module name to avoid name clashes on import
|
||||
script_path = os.path.join(workpath, name + '-script.py')
|
||||
print "creating script for entry point", dist, group, name
|
||||
with open(script_path, 'w') as fh:
|
||||
fh.write("import {0}\n".format(ep.module_name))
|
||||
fh.write("{0}.{1}()\n".format(ep.module_name, '.'.join(ep.attrs)))
|
||||
for package in packages:
|
||||
fh.write("import {0}\n".format(package))
|
||||
|
||||
return Analysis([script_path] + scripts,
|
||||
pathex=pathex,
|
||||
binaries=binaries,
|
||||
datas=datas,
|
||||
hiddenimports=hiddenimports,
|
||||
hookspath=hookspath,
|
||||
excludes=excludes,
|
||||
runtime_hooks=runtime_hooks,
|
||||
cipher=cipher,
|
||||
win_no_prefer_redirects=win_no_prefer_redirects,
|
||||
win_private_assemblies=win_private_assemblies
|
||||
)
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
|
||||
LINUX=false
|
||||
OSX=false
|
||||
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
OSX=true
|
||||
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
|
||||
LINUX=true
|
||||
else
|
||||
echo "Platform detection failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
SUDO=''
|
||||
if $LINUX && (( $EUID != 0 )); then
|
||||
SUDO='sudo'
|
||||
fi
|
||||
|
||||
cmd_exists() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
return $?
|
||||
}
|
||||
|
||||
set +eu
|
||||
GITUSERNAME=$(git config --global --get user.name)
|
||||
if [ -z "$GITUSERNAME" ]; then
|
||||
git config --global user.name "$(whoami)"
|
||||
fi
|
||||
GITEMAIL=$(git config --global --get user.email)
|
||||
if [ -z "$GITEMAIL" ]; then
|
||||
git config --global user.email "$(whoami)@lbry.io"
|
||||
fi
|
||||
set -eu
|
||||
|
||||
|
||||
if $LINUX; then
|
||||
INSTALL="$SUDO apt-get install --no-install-recommends -y"
|
||||
$INSTALL build-essential libssl-dev libffi-dev python2.7-dev wget
|
||||
elif $OSX && ! cmd_exists brew ; then
|
||||
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
|
||||
fi
|
||||
|
||||
|
||||
if ! cmd_exists python; then
|
||||
if $LINUX; then
|
||||
$INSTALL python2.7
|
||||
elif $OSX; then
|
||||
brew install python
|
||||
fi
|
||||
fi
|
||||
|
||||
PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
|
||||
if [ "$PYTHON_VERSION" != "2.7" ]; then
|
||||
echo "Python 2.7 required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! cmd_exists pip; then
|
||||
if $LINUX; then
|
||||
$INSTALL python-pip
|
||||
$SUDO pip install --upgrade pip
|
||||
else
|
||||
echo "Pip required"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
|
||||
#$INSTALL python-setuptools
|
||||
$SUDO pip install setuptools
|
||||
fi
|
||||
|
||||
if ! cmd_exists virtualenv; then
|
||||
$SUDO pip install virtualenv
|
||||
fi
|
||||
@@ -1,11 +0,0 @@
|
||||
# install daemon requirements (created by build script. see build.sh, build.ps1)
|
||||
-r requirements_base.txt
|
||||
|
||||
# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
|
||||
file:../.
|
||||
|
||||
# install other build requirements
|
||||
PyInstaller==3.2.1
|
||||
requests[security]==2.13.0
|
||||
uritemplate==3.0.0
|
||||
boto3==1.4.4
|
||||
@@ -1,143 +0,0 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import github
|
||||
import uritemplate
|
||||
import boto3
|
||||
|
||||
|
||||
def main():
|
||||
upload_to_github_if_tagged('lbryio/lbry')
|
||||
upload_to_s3('daemon')
|
||||
|
||||
|
||||
def get_asset_filename():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
return glob.glob(this_dir + '/dist/*.zip')[0]
|
||||
|
||||
|
||||
def upload_to_s3(folder):
|
||||
tag = subprocess.check_output(['git', 'describe', '--always', '--abbrev=8', 'HEAD']).strip()
|
||||
commit_date = subprocess.check_output([
|
||||
'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
bucket = 'releases.lbry.io'
|
||||
key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
|
||||
|
||||
print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
|
||||
|
||||
if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
|
||||
print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
|
||||
return 1
|
||||
|
||||
s3 = boto3.resource(
|
||||
's3',
|
||||
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
|
||||
config=boto3.session.Config(signature_version='s3v4')
|
||||
)
|
||||
s3.meta.client.upload_file(asset_path, bucket, key)
|
||||
|
||||
|
||||
def upload_to_github_if_tagged(repo_name):
|
||||
try:
|
||||
current_tag = subprocess.check_output(
|
||||
['git', 'describe', '--exact-match', 'HEAD']).strip()
|
||||
except subprocess.CalledProcessError:
|
||||
print 'Not uploading to GitHub as we are not currently on a tag'
|
||||
return 1
|
||||
|
||||
print "Current tag: " + current_tag
|
||||
|
||||
if 'GH_TOKEN' not in os.environ:
|
||||
print 'Must set GH_TOKEN in order to publish assets to a release'
|
||||
return 1
|
||||
|
||||
gh_token = os.environ['GH_TOKEN']
|
||||
auth = github.Github(gh_token)
|
||||
repo = auth.get_repo(repo_name)
|
||||
|
||||
if not check_repo_has_tag(repo, current_tag):
|
||||
print 'Tag {} is not in repo {}'.format(current_tag, repo)
|
||||
# TODO: maybe this should be an error
|
||||
return 1
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
print "Uploading " + asset_path + " to Github tag " + current_tag
|
||||
release = get_github_release(repo, current_tag)
|
||||
upload_asset_to_github(release, asset_path, gh_token)
|
||||
|
||||
|
||||
def check_repo_has_tag(repo, target_tag):
|
||||
tags = repo.get_tags().get_page(0)
|
||||
for tag in tags:
|
||||
if tag.name == target_tag:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_github_release(repo, current_tag):
|
||||
for release in repo.get_releases():
|
||||
if release.tag_name == current_tag:
|
||||
return release
|
||||
raise Exception('No release for {} was found'.format(current_tag))
|
||||
|
||||
|
||||
def upload_asset_to_github(release, asset_to_upload, token):
|
||||
basename = os.path.basename(asset_to_upload)
|
||||
for asset in release.raw_data['assets']:
|
||||
if asset['name'] == basename:
|
||||
print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
|
||||
return
|
||||
|
||||
upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
|
||||
count = 0
|
||||
while count < 10:
|
||||
try:
|
||||
output = _curl_uploader(upload_uri, asset_to_upload, token)
|
||||
if 'errors' in output:
|
||||
raise Exception(output)
|
||||
else:
|
||||
print 'Successfully uploaded to {}'.format(output['browser_download_url'])
|
||||
except Exception:
|
||||
print 'Failed uploading on attempt {}'.format(count + 1)
|
||||
count += 1
|
||||
|
||||
|
||||
def _curl_uploader(upload_uri, asset_to_upload, token):
|
||||
# using requests.post fails miserably with SSL EPIPE errors. I spent
|
||||
# half a day trying to debug before deciding to switch to curl.
|
||||
#
|
||||
# TODO: actually set the content type
|
||||
print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
|
||||
cmd = [
|
||||
'curl',
|
||||
'-sS',
|
||||
'-X', 'POST',
|
||||
'-u', ':{}'.format(os.environ['GH_TOKEN']),
|
||||
'--header', 'Content-Type: application/octet-stream',
|
||||
'--data-binary', '@-',
|
||||
upload_uri
|
||||
]
|
||||
# '-d', '{"some_key": "some_value"}',
|
||||
print 'Calling curl:'
|
||||
print cmd
|
||||
print
|
||||
with open(asset_to_upload, 'rb') as fp:
|
||||
p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||
stdout, stderr = p.communicate()
|
||||
print 'curl return code:', p.returncode
|
||||
if stderr:
|
||||
print 'stderr output from curl:'
|
||||
print stderr
|
||||
print 'stdout from curl:'
|
||||
print stdout
|
||||
return json.loads(stdout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,29 +0,0 @@
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
|
||||
def main():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
tag = subprocess.check_output(['git', 'describe']).strip()
|
||||
zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
|
||||
full_filename = os.path.join(this_dir, 'dist', zipfilename)
|
||||
executables = ['lbrynet-daemon', 'lbrynet-cli', 'lbrynet-console']
|
||||
ext = '.exe' if platform.system() == 'Windows' else ''
|
||||
with zipfile.ZipFile(full_filename, 'w') as myzip:
|
||||
for executable in executables:
|
||||
myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext)
|
||||
|
||||
|
||||
def get_system_label():
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
return 'macos'
|
||||
else:
|
||||
return system.lower()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -100,7 +100,7 @@
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
|
||||
|
||||
</a>
|
||||
</div>
|
||||
@@ -196,7 +196,7 @@
|
||||
<label class="md-nav__title md-nav__title--site" for="drawer">
|
||||
<span class="md-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
|
||||
|
||||
</span>
|
||||
LBRY
|
||||
@@ -304,4 +304,4 @@
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -1579,12 +1579,12 @@ Usage:
|
||||
|
||||
|
||||
Options:
|
||||
--a_arg : a arg
|
||||
--b_arg : b arg
|
||||
--pos_arg=<pos_arg> : pos arg
|
||||
--pos_args=<pos_args> : pos args
|
||||
--pos_arg2=<pos_arg2> : pos arg 2
|
||||
--pos_arg3=<pos_arg3> : pos arg 3
|
||||
--a_arg : (bool) a arg
|
||||
--b_arg : (bool) b arg
|
||||
--pos_arg=<pos_arg> : (int) pos arg
|
||||
--pos_args=<pos_args> : (int) pos args
|
||||
--pos_arg2=<pos_arg2> : (int) pos arg 2
|
||||
--pos_arg3=<pos_arg3> : (int) pos arg 3
|
||||
|
||||
Returns:
|
||||
pos args
|
||||
@@ -1828,14 +1828,16 @@ Returns:
|
||||
</code></pre>
|
||||
|
||||
<h2 id="peer_ping">peer_ping</h2>
|
||||
<pre><code class="text">Find and ping a peer by node id
|
||||
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
|
||||
if not provided the peer is located first.
|
||||
|
||||
Usage:
|
||||
peer_ping (<node_id> | --node_id=<node_id>)
|
||||
peer_ping (<node_id> | --node_id=<node_id>) [<address> | --address=<address>] [<port> | --port=<port>]
|
||||
|
||||
|
||||
Options:
|
||||
None
|
||||
--address=<address> : (str) ip address of the peer
|
||||
--port=<port> : (int) udp port of the peer
|
||||
|
||||
Returns:
|
||||
(str) pong, or {'error': <error message>} if an error is encountered
|
||||
@@ -1901,7 +1903,7 @@ Options:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
--claim_address=<claim_address> : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
@@ -2110,41 +2112,62 @@ Returns:
|
||||
<pre><code class="text">Get daemon status
|
||||
|
||||
Usage:
|
||||
status [--session_status]
|
||||
status
|
||||
|
||||
|
||||
Options:
|
||||
--session_status : (bool) include session status in results
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) lbrynet-daemon status
|
||||
{
|
||||
'lbry_id': lbry peer id, base58,
|
||||
'installation_id': installation id, base58,
|
||||
'is_running': bool,
|
||||
'installation_id': (str) installation id - base58,
|
||||
'is_running': (bool),
|
||||
'is_first_run': bool,
|
||||
'startup_status': {
|
||||
'code': status code,
|
||||
'message': status message
|
||||
'skipped_components': (list) [names of skipped components (str)],
|
||||
'startup_status': { Does not include components which have been skipped
|
||||
'database': (bool),
|
||||
'wallet': (bool),
|
||||
'session': (bool),
|
||||
'dht': (bool),
|
||||
'hash_announcer': (bool),
|
||||
'stream_identifier': (bool),
|
||||
'file_manager': (bool),
|
||||
'blob_manager': (bool),
|
||||
'blockchain_headers': (bool),
|
||||
'peer_protocol_server': (bool),
|
||||
'reflector': (bool),
|
||||
'upnp': (bool),
|
||||
'exchange_rate_manager': (bool),
|
||||
},
|
||||
'connection_status': {
|
||||
'code': connection status code,
|
||||
'message': connection status message
|
||||
'code': (str) connection status code,
|
||||
'message': (str) connection status message
|
||||
},
|
||||
'blockchain_status': {
|
||||
'blocks': local blockchain height,
|
||||
'blocks_behind': remote_height - local_height,
|
||||
'best_blockhash': block hash of most recent block,
|
||||
'blockchain_headers': {
|
||||
'downloading_headers': (bool),
|
||||
'download_progress': (float) 0-100.0
|
||||
},
|
||||
'wallet_is_encrypted': bool,
|
||||
|
||||
If given the session status option:
|
||||
'session_status': {
|
||||
'managed_blobs': count of blobs in the blob manager,
|
||||
'managed_streams': count of streams in the file manager
|
||||
'announce_queue_size': number of blobs currently queued to be announced
|
||||
'should_announce_blobs': number of blobs that should be announced
|
||||
}
|
||||
'wallet': {
|
||||
'blocks': (int) local blockchain height,
|
||||
'blocks_behind': (int) remote_height - local_height,
|
||||
'best_blockhash': (str) block hash of most recent block,
|
||||
'is_encrypted': (bool),
|
||||
'is_locked': (bool),
|
||||
},
|
||||
'dht': {
|
||||
'node_id': (str) lbry dht node id - hex encoded,
|
||||
'peers_in_routing_table': (int) the number of peers in the routing table,
|
||||
},
|
||||
'blob_manager': {
|
||||
'finished_blobs': (int) number of finished blobs in the blob manager,
|
||||
},
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
},
|
||||
'file_manager': {
|
||||
'managed_files': (int) count of files in the file manager,
|
||||
}
|
||||
}
|
||||
</code></pre>
|
||||
|
||||
@@ -2452,9 +2475,9 @@ Usage:
|
||||
|
||||
|
||||
Options:
|
||||
--amount=<amount> : (float) amount of credit to send
|
||||
--address=<address> : (str) address to send credits to
|
||||
--claim_id=<claim_id> : (float) claim_id of the claim to send to tip to
|
||||
--amount=<amount> : (Decimal) amount of credit to send
|
||||
--address=<address> : (str) address to send credits to
|
||||
--claim_id=<claim_id> : (str) claim_id of the claim to send to tip to
|
||||
|
||||
Returns:
|
||||
If sending to an address:
|
||||
@@ -2563,4 +2586,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -1479,12 +1479,12 @@ Returns:
|
||||
<h2 id="cli_test_command">cli_test_command</h2>
|
||||
<pre><code class="text">This command is only for testing the CLI argument parsing
|
||||
Args:
|
||||
'a_arg' : a arg
|
||||
'b_arg' : b arg
|
||||
'pos_arg' (required) : pos arg
|
||||
'pos_args' : pos args
|
||||
'pos_arg2' : pos arg 2
|
||||
'pos_arg3' : pos arg 3
|
||||
'a_arg' : (bool) a arg
|
||||
'b_arg' : (bool) b arg
|
||||
'pos_arg' (required) : (int) pos arg
|
||||
'pos_args' : (int) pos args
|
||||
'pos_arg2' : (int) pos arg 2
|
||||
'pos_arg3' : (int) pos arg 3
|
||||
|
||||
Returns:
|
||||
pos args
|
||||
@@ -1682,10 +1682,12 @@ Returns:
|
||||
</code></pre>
|
||||
|
||||
<h2 id="peer_ping">peer_ping</h2>
|
||||
<pre><code class="text">Find and ping a peer by node id
|
||||
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
|
||||
if not provided the peer is located first.
|
||||
|
||||
Args:
|
||||
None
|
||||
'address' : (str) ip address of the peer
|
||||
'port' : (int) udp port of the peer
|
||||
|
||||
Returns:
|
||||
(str) pong, or {'error': <error message>} if an error is encountered
|
||||
@@ -1741,7 +1743,7 @@ Args:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
'claim_address' : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
@@ -1913,37 +1915,58 @@ Returns:
|
||||
<pre><code class="text">Get daemon status
|
||||
|
||||
Args:
|
||||
'session_status' : (bool) include session status in results
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) lbrynet-daemon status
|
||||
{
|
||||
'lbry_id': lbry peer id, base58,
|
||||
'installation_id': installation id, base58,
|
||||
'is_running': bool,
|
||||
'installation_id': (str) installation id - base58,
|
||||
'is_running': (bool),
|
||||
'is_first_run': bool,
|
||||
'startup_status': {
|
||||
'code': status code,
|
||||
'message': status message
|
||||
'skipped_components': (list) [names of skipped components (str)],
|
||||
'startup_status': { Does not include components which have been skipped
|
||||
'database': (bool),
|
||||
'wallet': (bool),
|
||||
'session': (bool),
|
||||
'dht': (bool),
|
||||
'hash_announcer': (bool),
|
||||
'stream_identifier': (bool),
|
||||
'file_manager': (bool),
|
||||
'blob_manager': (bool),
|
||||
'blockchain_headers': (bool),
|
||||
'peer_protocol_server': (bool),
|
||||
'reflector': (bool),
|
||||
'upnp': (bool),
|
||||
'exchange_rate_manager': (bool),
|
||||
},
|
||||
'connection_status': {
|
||||
'code': connection status code,
|
||||
'message': connection status message
|
||||
'code': (str) connection status code,
|
||||
'message': (str) connection status message
|
||||
},
|
||||
'blockchain_status': {
|
||||
'blocks': local blockchain height,
|
||||
'blocks_behind': remote_height - local_height,
|
||||
'best_blockhash': block hash of most recent block,
|
||||
'blockchain_headers': {
|
||||
'downloading_headers': (bool),
|
||||
'download_progress': (float) 0-100.0
|
||||
},
|
||||
'wallet_is_encrypted': bool,
|
||||
|
||||
If given the session status option:
|
||||
'session_status': {
|
||||
'managed_blobs': count of blobs in the blob manager,
|
||||
'managed_streams': count of streams in the file manager
|
||||
'announce_queue_size': number of blobs currently queued to be announced
|
||||
'should_announce_blobs': number of blobs that should be announced
|
||||
}
|
||||
'wallet': {
|
||||
'blocks': (int) local blockchain height,
|
||||
'blocks_behind': (int) remote_height - local_height,
|
||||
'best_blockhash': (str) block hash of most recent block,
|
||||
'is_encrypted': (bool),
|
||||
'is_locked': (bool),
|
||||
},
|
||||
'dht': {
|
||||
'node_id': (str) lbry dht node id - hex encoded,
|
||||
'peers_in_routing_table': (int) the number of peers in the routing table,
|
||||
},
|
||||
'blob_manager': {
|
||||
'finished_blobs': (int) number of finished blobs in the blob manager,
|
||||
},
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
},
|
||||
'file_manager': {
|
||||
'managed_files': (int) count of files in the file manager,
|
||||
}
|
||||
}
|
||||
</code></pre>
|
||||
|
||||
@@ -2186,9 +2209,9 @@ to the owner of a claim specified by uri. A tip is a claim support where the rec
|
||||
of the support is the claim address for the claim being supported.
|
||||
|
||||
Args:
|
||||
'amount' (required) : (float) amount of credit to send
|
||||
'address' (required) : (str) address to send credits to
|
||||
'claim_id' (required) : (float) claim_id of the claim to send to tip to
|
||||
'amount' (required) : (Decimal) amount of credit to send
|
||||
'address' (required) : (str) address to send credits to
|
||||
'claim_id' (required) : (str) claim_id of the claim to send to tip to
|
||||
|
||||
Returns:
|
||||
If sending to an address:
|
||||
@@ -2289,4 +2312,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -2,7 +2,7 @@
|
||||
"docs": [
|
||||
{
|
||||
"location": "/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"title": "API"
|
||||
},
|
||||
{
|
||||
@@ -112,7 +112,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#cli_test_command",
|
||||
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : a arg\n 'b_arg' : b arg\n 'pos_arg' (required) : pos arg\n 'pos_args' : pos args\n 'pos_arg2' : pos arg 2\n 'pos_arg3' : pos arg 3\n\nReturns:\n pos args",
|
||||
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : (bool) a arg\n 'b_arg' : (bool) b arg\n 'pos_arg' (required) : (int) pos arg\n 'pos_args' : (int) pos args\n 'pos_arg2' : (int) pos arg 2\n 'pos_arg3' : (int) pos arg 3\n\nReturns:\n pos args",
|
||||
"title": "cli_test_command"
|
||||
},
|
||||
{
|
||||
@@ -162,12 +162,12 @@
|
||||
},
|
||||
{
|
||||
"location": "/#peer_ping",
|
||||
"text": "Find and ping a peer by node id\n\nArgs:\n None\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nArgs:\n 'address' : (str) ip address of the peer\n 'port' : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"title": "peer_ping"
|
||||
},
|
||||
{
|
||||
"location": "/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
@@ -202,7 +202,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#status",
|
||||
"text": "Get daemon status\n\nArgs:\n 'session_status' : (bool) include session status in results\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n }",
|
||||
"text": "Get daemon status\n\nArgs:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
|
||||
"title": "status"
|
||||
},
|
||||
{
|
||||
@@ -277,7 +277,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#wallet_send",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (float) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (Decimal) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"title": "wallet_send"
|
||||
},
|
||||
{
|
||||
@@ -292,7 +292,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"title": "CLI"
|
||||
},
|
||||
{
|
||||
@@ -402,7 +402,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#cli_test_command",
|
||||
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : a arg\n --b_arg : b arg\n --pos_arg= pos_arg : pos arg\n --pos_args= pos_args : pos args\n --pos_arg2= pos_arg2 : pos arg 2\n --pos_arg3= pos_arg3 : pos arg 3\n\nReturns:\n pos args",
|
||||
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : (bool) a arg\n --b_arg : (bool) b arg\n --pos_arg= pos_arg : (int) pos arg\n --pos_args= pos_args : (int) pos args\n --pos_arg2= pos_arg2 : (int) pos arg 2\n --pos_arg3= pos_arg3 : (int) pos arg 3\n\nReturns:\n pos args",
|
||||
"title": "cli_test_command"
|
||||
},
|
||||
{
|
||||
@@ -452,12 +452,12 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#peer_ping",
|
||||
"text": "Find and ping a peer by node id\n\nUsage:\n peer_ping ( node_id | --node_id= node_id )\n\n\nOptions:\n None\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nUsage:\n peer_ping ( node_id | --node_id= node_id ) [ address | --address= address ] [ port | --port= port ]\n\n\nOptions:\n --address= address : (str) ip address of the peer\n --port= port : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
|
||||
"title": "peer_ping"
|
||||
},
|
||||
{
|
||||
"location": "/cli/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
@@ -492,7 +492,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#status",
|
||||
"text": "Get daemon status\n\nUsage:\n status [--session_status]\n\n\nOptions:\n --session_status : (bool) include session status in results\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n }",
|
||||
"text": "Get daemon status\n\nUsage:\n status\n\n\nOptions:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
|
||||
"title": "status"
|
||||
},
|
||||
{
|
||||
@@ -567,7 +567,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#wallet_send",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (float) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (Decimal) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
|
||||
"title": "wallet_send"
|
||||
},
|
||||
{
|
||||
@@ -581,4 +581,4 @@
|
||||
"title": "wallet_unused_address"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# This is an example daemon_settings.yml file.
|
||||
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
|
||||
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.io:50001
|
||||
- lbryumx2.lbry.io:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
use_auth_http: True
|
||||
use_https: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - dht
|
||||
|
Before Width: | Height: | Size: 7.4 KiB After Width: | Height: | Size: 7.4 KiB |
|
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 19 KiB |
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 2.6 KiB |
|
Before Width: | Height: | Size: 6.1 KiB After Width: | Height: | Size: 6.1 KiB |
|
Before Width: | Height: | Size: 97 KiB After Width: | Height: | Size: 97 KiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 361 KiB After Width: | Height: | Size: 361 KiB |
|
Before Width: | Height: | Size: 5.3 KiB After Width: | Height: | Size: 5.3 KiB |
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 31 KiB After Width: | Height: | Size: 31 KiB |
@@ -1,6 +1,10 @@
|
||||
import logging
|
||||
from lbrynet.custom_logger import install_logger
|
||||
|
||||
__version__ = "0.20.3"
|
||||
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.3"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
install_logger()
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -1 +0,0 @@
|
||||
import paths
|
||||
@@ -1,26 +0,0 @@
|
||||
# App root files dir
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
|
||||
def android_files_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base internal storage path
|
||||
def android_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base external (SD card, if present) storage path
|
||||
def android_external_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Internal device storage (private app folder)
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# External (app folder on SD card, if present) storage
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_external_storage_dir():
|
||||
return None
|
||||
@@ -7,7 +7,7 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.p2p.BlobInfo import BlobInfo
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -16,21 +16,21 @@ backend = default_backend()
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
super().__init__(blob_hash, blob_num, length)
|
||||
self.iv = iv
|
||||
|
||||
def get_dict(self):
|
||||
info = {
|
||||
"blob_num": self.blob_num,
|
||||
"length": self.length,
|
||||
"iv": self.iv
|
||||
"iv": self.iv.decode()
|
||||
}
|
||||
if self.blob_hash:
|
||||
info['blob_hash'] = self.blob_hash
|
||||
return info
|
||||
|
||||
|
||||
class StreamBlobDecryptor(object):
|
||||
class StreamBlobDecryptor:
|
||||
def __init__(self, blob, key, iv, length):
|
||||
"""
|
||||
This class decrypts blob
|
||||
@@ -53,10 +53,10 @@ class StreamBlobDecryptor(object):
|
||||
|
||||
def decrypt(self, write_func):
|
||||
"""
|
||||
Decrypt blob and write its content useing write_func
|
||||
Decrypt blob and write its content using write_func
|
||||
|
||||
write_func - function that takes decrypted string as
|
||||
arugment and writes it somewhere
|
||||
argument and writes it somewhere
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -68,14 +68,14 @@ class StreamBlobDecryptor(object):
|
||||
|
||||
def write_bytes():
|
||||
if self.len_read < self.length:
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size / 8))
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size // 8))
|
||||
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
|
||||
write_func(self.cipher.update(data_to_decrypt))
|
||||
|
||||
def finish_decrypt():
|
||||
bytes_left = len(self.buff) % (AES.block_size / 8)
|
||||
bytes_left = len(self.buff) % (AES.block_size // 8)
|
||||
if bytes_left != 0:
|
||||
log.warning(self.buff[-1 * (AES.block_size / 8):].encode('hex'))
|
||||
log.warning(binascii.hexlify(self.buff[-1 * (AES.block_size // 8):]).decode())
|
||||
raise Exception("blob %s has incorrect padding: %i bytes left" %
|
||||
(self.blob.blob_hash, bytes_left))
|
||||
data_to_decrypt, self.buff = self.buff, b''
|
||||
@@ -99,7 +99,7 @@ class StreamBlobDecryptor(object):
|
||||
return d
|
||||
|
||||
|
||||
class CryptStreamBlobMaker(object):
|
||||
class CryptStreamBlobMaker:
|
||||
def __init__(self, key, iv, blob_num, blob):
|
||||
"""
|
||||
This class encrypts data and writes it to a new blob
|
||||
@@ -146,7 +146,7 @@ class CryptStreamBlobMaker(object):
|
||||
def close(self):
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self.length += (AES.block_size / 8) - (self.length % (AES.block_size / 8))
|
||||
self.length += (AES.block_size // 8) - (self.length % (AES.block_size // 8))
|
||||
padded_data = self.padder.finalize()
|
||||
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
|
||||
self.blob.write(encrypted_data)
|
||||
@@ -5,15 +5,14 @@ import os
|
||||
import logging
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
from twisted.internet import defer
|
||||
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(object):
|
||||
class CryptStreamCreator:
|
||||
"""
|
||||
Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
@@ -22,7 +21,7 @@ class CryptStreamCreator(object):
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
|
||||
implements(interfaces.IConsumer)
|
||||
#implements(interfaces.IConsumer)
|
||||
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@@ -101,13 +100,13 @@ class CryptStreamCreator(object):
|
||||
@staticmethod
|
||||
def random_iv_generator():
|
||||
while 1:
|
||||
yield os.urandom(AES.block_size / 8)
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
def setup(self):
|
||||
"""Create the symmetric key if it wasn't provided"""
|
||||
|
||||
if self.key is None:
|
||||
self.key = os.urandom(AES.block_size / 8)
|
||||
self.key = os.urandom(AES.block_size // 8)
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
@@ -122,7 +121,7 @@ class CryptStreamCreator(object):
|
||||
|
||||
yield defer.DeferredList(self.finished_deferreds)
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
iv = next(self.iv_generator)
|
||||
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
|
||||
stream_terminator = yield final_blob.close()
|
||||
terminator_info = yield self._blob_finished(stream_terminator)
|
||||
@@ -133,7 +132,7 @@ class CryptStreamCreator(object):
|
||||
if self.current_blob is None:
|
||||
self.next_blob_creator = self.blob_manager.get_blob_creator()
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
iv = next(self.iv_generator)
|
||||
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
|
||||
done, num_bytes_written = self.current_blob.write(data)
|
||||
data = data[num_bytes_written:]
|
||||
@@ -2,16 +2,16 @@
|
||||
Utilities for turning plain files into LBRY Files.
|
||||
"""
|
||||
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.core.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,7 +23,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
|
||||
def __init__(self, blob_manager, lbry_file_manager, stream_name=None,
|
||||
key=None, iv_generator=None):
|
||||
CryptStreamCreator.__init__(self, blob_manager, stream_name, key, iv_generator)
|
||||
super().__init__(blob_manager, stream_name, key, iv_generator)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.stream_hash = None
|
||||
self.blob_infos = []
|
||||
@@ -37,14 +37,14 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
def _finished(self):
|
||||
# calculate the stream hash
|
||||
self.stream_hash = get_stream_hash(
|
||||
hexlify(self.name), hexlify(self.key), hexlify(self.name),
|
||||
hexlify(self.name.encode()).decode(), hexlify(self.key).decode(), hexlify(self.name.encode()).decode(),
|
||||
self.blob_infos
|
||||
)
|
||||
|
||||
# generate the sd info
|
||||
self.sd_info = format_sd_info(
|
||||
EncryptedFileStreamType, hexlify(self.name), hexlify(self.key),
|
||||
hexlify(self.name), self.stream_hash, self.blob_infos
|
||||
EncryptedFileStreamType, hexlify(self.name.encode()).decode(), hexlify(self.key).decode(),
|
||||
hexlify(self.name.encode()).decode(), self.stream_hash, self.blob_infos
|
||||
)
|
||||
|
||||
# sanity check
|
||||
@@ -59,7 +59,8 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
# we can simply read the file from the disk without needing to
|
||||
# involve reactor.
|
||||
@defer.inlineCallbacks
|
||||
def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None, iv_generator=None):
|
||||
def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_manager, file_name, file_handle,
|
||||
key=None, iv_generator=None):
|
||||
"""Turn a plain file into an LBRY File.
|
||||
|
||||
An LBRY File is a collection of encrypted blobs of data and the metadata that binds them
|
||||
@@ -98,7 +99,7 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
file_directory = os.path.dirname(file_handle.name)
|
||||
|
||||
lbry_file_creator = EncryptedFileStreamCreator(
|
||||
session.blob_manager, lbry_file_manager, base_file_name, key, iv_generator
|
||||
blob_manager, lbry_file_manager, base_file_name, key, iv_generator
|
||||
)
|
||||
|
||||
yield lbry_file_creator.setup()
|
||||
@@ -114,25 +115,17 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
|
||||
log.debug("making the sd blob")
|
||||
sd_info = lbry_file_creator.sd_info
|
||||
descriptor_writer = BlobStreamDescriptorWriter(session.blob_manager)
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield session.storage.store_stream(
|
||||
yield storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
)
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, binascii.hexlify(file_directory), session.payment_rate_manager,
|
||||
session.payment_rate_manager.min_blob_data_payment_rate
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
)
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
|
||||
def hexlify(str_or_unicode):
|
||||
if isinstance(str_or_unicode, unicode):
|
||||
strng = str_or_unicode.encode('utf-8')
|
||||
else:
|
||||
strng = str_or_unicode
|
||||
return binascii.hexlify(strng)
|
||||
@@ -2,18 +2,17 @@
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
import logging
|
||||
import binascii
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.utils import short_hash
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileDownloader
|
||||
from lbrynet.file_manager.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.core.StreamDescriptor import save_sd_info
|
||||
from lbrynet import conf
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -37,14 +36,14 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
|
||||
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
|
||||
suggested_file_name):
|
||||
EncryptedFileSaver.__init__(
|
||||
self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
suggested_file_name, download_mirrors=None):
|
||||
super().__init__(
|
||||
stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name
|
||||
)
|
||||
self.sd_hash = sd_hash
|
||||
self.rowid = rowid
|
||||
self.suggested_file_name = binascii.unhexlify(suggested_file_name)
|
||||
self.suggested_file_name = unhexlify(suggested_file_name).decode()
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self._saving_status = False
|
||||
self.claim_id = None
|
||||
@@ -55,6 +54,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_claim_id = None
|
||||
self.channel_name = None
|
||||
self.metadata = None
|
||||
self.mirror = None
|
||||
if download_mirrors or conf.settings['download_mirrors']:
|
||||
self.mirror = HTTPBlobDownloader(
|
||||
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
|
||||
)
|
||||
|
||||
def set_claim_info(self, claim_info):
|
||||
self.claim_id = claim_info['claim_id']
|
||||
@@ -89,16 +93,18 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
self.completed = True
|
||||
else:
|
||||
raise Exception("Unknown status for stream %s: %s" % (self.stream_hash, status))
|
||||
raise Exception(f"Unknown status for stream {self.stream_hash}: {status}")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None, change_status=True):
|
||||
log.debug('Stopping download for stream %s', short_hash(self.stream_hash))
|
||||
if self.mirror:
|
||||
self.mirror.stop()
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
yield EncryptedFileDownloader.stop(self, err=err)
|
||||
if change_status is True:
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
defer.returnValue(status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def status(self):
|
||||
@@ -123,6 +129,8 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
yield EncryptedFileSaver._start(self)
|
||||
status = yield self._save_status()
|
||||
log_status(self.sd_hash, status)
|
||||
if self.mirror:
|
||||
self.mirror.download_stream(self.stream_hash, self.sd_hash)
|
||||
defer.returnValue(status)
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
@@ -152,26 +160,29 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
class ManagedEncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, lbry_file_manager):
|
||||
def __init__(self, lbry_file_manager, blob_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def can_download(self, sd_validator):
|
||||
# TODO: add a sd_validator for non live streams, use it
|
||||
return True
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def make_downloader(self, metadata, data_rate, payment_rate_manager, download_directory, file_name=None):
|
||||
stream_hash = yield save_sd_info(self.lbry_file_manager.session.blob_manager,
|
||||
def make_downloader(self, metadata, data_rate, payment_rate_manager, download_directory, file_name=None,
|
||||
download_mirrors=None):
|
||||
stream_hash = yield save_sd_info(self.blob_manager,
|
||||
metadata.source_blob_hash,
|
||||
metadata.validator.raw_info)
|
||||
if file_name:
|
||||
file_name = binascii.hexlify(file_name)
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, binascii.hexlify(download_directory), payment_rate_manager,
|
||||
data_rate, file_name=file_name
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
)
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@@ -3,24 +3,23 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.reflector.reupload import reflect_file
|
||||
# from lbrynet.core.PaymentRateManager import NegotiatedPaymentRateManager
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
from lbrynet import conf
|
||||
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileManager(object):
|
||||
class EncryptedFileManager:
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and
|
||||
their LBRY File specific metadata.
|
||||
@@ -28,15 +27,17 @@ class EncryptedFileManager(object):
|
||||
# when reflecting files, reflect up to this many files at a time
|
||||
CONCURRENT_REFLECTS = 5
|
||||
|
||||
def __init__(self, session, sd_identifier):
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
|
||||
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
|
||||
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
|
||||
self.session = session
|
||||
self.storage = session.storage
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.storage = storage
|
||||
# TODO: why is sd_identifier part of the file manager?
|
||||
self.sd_identifier = sd_identifier
|
||||
assert sd_identifier
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
@@ -47,14 +48,14 @@ class EncryptedFileManager(object):
|
||||
log.info("Started file manager")
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.session.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
|
||||
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
|
||||
return self.session.storage(lbry_file.rowid, new_rate)
|
||||
return self.storage(lbry_file.rowid, new_rate)
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self.session.storage.change_file_status(lbry_file.rowid, status)
|
||||
return self.storage.change_file_status(lbry_file.rowid, status)
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -71,35 +72,36 @@ class EncryptedFileManager(object):
|
||||
return dl
|
||||
|
||||
def _add_to_sd_identifier(self):
|
||||
downloader_factory = ManagedEncryptedFileDownloaderFactory(self)
|
||||
downloader_factory = ManagedEncryptedFileDownloaderFactory(self, self.blob_manager)
|
||||
self.sd_identifier.add_stream_downloader_factory(
|
||||
EncryptedFileStreamType, downloader_factory)
|
||||
|
||||
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
|
||||
stream_name, file_name, download_directory, suggested_file_name):
|
||||
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
|
||||
return ManagedEncryptedFileDownloader(
|
||||
rowid,
|
||||
stream_hash,
|
||||
self.session.peer_finder,
|
||||
self.session.rate_limiter,
|
||||
self.session.blob_manager,
|
||||
self.session.storage,
|
||||
self.peer_finder,
|
||||
self.rate_limiter,
|
||||
self.blob_manager,
|
||||
self.storage,
|
||||
self,
|
||||
payment_rate_manager,
|
||||
self.session.wallet,
|
||||
self.wallet,
|
||||
download_directory,
|
||||
file_name,
|
||||
stream_name=stream_name,
|
||||
sd_hash=sd_hash,
|
||||
key=key,
|
||||
suggested_file_name=suggested_file_name
|
||||
suggested_file_name=suggested_file_name,
|
||||
download_mirrors=download_mirrors
|
||||
)
|
||||
|
||||
def _start_lbry_file(self, file_info, payment_rate_manager, claim_info):
|
||||
def _start_lbry_file(self, file_info, payment_rate_manager, claim_info, download_mirrors=None):
|
||||
lbry_file = self._get_lbry_file(
|
||||
file_info['row_id'], file_info['stream_hash'], payment_rate_manager, file_info['sd_hash'],
|
||||
file_info['key'], file_info['stream_name'], file_info['file_name'], file_info['download_directory'],
|
||||
file_info['suggested_file_name']
|
||||
file_info['suggested_file_name'], download_mirrors
|
||||
)
|
||||
if claim_info:
|
||||
lbry_file.set_claim_info(claim_info)
|
||||
@@ -115,9 +117,9 @@ class EncryptedFileManager(object):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_lbry_files(self):
|
||||
files = yield self.session.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.session.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.session.payment_rate_manager
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
for file_info in files:
|
||||
@@ -144,16 +146,15 @@ class EncryptedFileManager(object):
|
||||
finally:
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop_lbry_files(self):
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
lbry_files = self.lbry_files
|
||||
for lbry_file in lbry_files:
|
||||
yield self._stop_lbry_file(lbry_file)
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = yield get_sd_info(self.session.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
@@ -162,7 +163,7 @@ class EncryptedFileManager(object):
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name']
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info()
|
||||
@@ -172,11 +173,11 @@ class EncryptedFileManager(object):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None):
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.session.payment_rate_manager
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = yield get_sd_info(self.session.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
@@ -184,12 +185,12 @@ class EncryptedFileManager(object):
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = yield self.storage.save_downloaded_file(
|
||||
stream_hash, os.path.basename(file_name.decode('hex')).encode('hex'), download_directory, blob_data_rate
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = yield self.session.storage.get_filename_for_rowid(rowid)
|
||||
file_name = (yield self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name']
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info(include_supports=False)
|
||||
@@ -221,7 +222,7 @@ class EncryptedFileManager(object):
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield self.session.storage.delete_stream(lbry_file.stream_hash)
|
||||
yield self.storage.delete_stream(lbry_file.stream_hash)
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
@@ -248,6 +249,6 @@ class EncryptedFileManager(object):
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
safe_stop_looping_call(self.lbry_file_reflector)
|
||||
yield defer.DeferredList(list(self._stop_lbry_files()))
|
||||
yield self._stop_lbry_files()
|
||||
log.info("Stopped encrypted file manager")
|
||||
defer.returnValue(True)
|
||||
@@ -1,4 +1,4 @@
|
||||
class EncryptedFileStatusReport(object):
|
||||
class EncryptedFileStatusReport:
|
||||
def __init__(self, name, num_completed, num_known, running_status):
|
||||
self.name = name
|
||||
self.num_completed = num_completed
|
||||
@@ -1,4 +0,0 @@
|
||||
from blob_file import BlobFile
|
||||
from creator import BlobFileCreator
|
||||
from writer import HashBlobWriter
|
||||
from reader import HashBlobReader
|
||||
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
import os
|
||||
import logging
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.blob.reader import HashBlobReader
|
||||
|
||||
@@ -12,8 +12,26 @@ log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
class BlobFile(object):
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
class BlobFile:
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
|
||||
@@ -52,7 +70,7 @@ class BlobFile(object):
|
||||
def open_for_writing(self, peer):
|
||||
"""
|
||||
open a blob file to be written by peer, supports concurrent
|
||||
writers, as long as they are from differnt peers.
|
||||
writers, as long as they are from different peers.
|
||||
|
||||
returns tuple of (writer, finished_deferred)
|
||||
|
||||
@@ -60,12 +78,12 @@ class BlobFile(object):
|
||||
finished_deferred - deferred that is fired when write is finished and returns
|
||||
a instance of itself as HashBlob
|
||||
"""
|
||||
if not peer in self.writers:
|
||||
if peer not in self.writers:
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(self.get_length, self.writer_finished)
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return (writer, finished_deferred)
|
||||
return writer, finished_deferred
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None
|
||||
|
||||
@@ -115,7 +133,7 @@ class BlobFile(object):
|
||||
def verified(self):
|
||||
"""
|
||||
Protect verified from being modified by other classes.
|
||||
verified is True if a write to a blob has completed succesfully,
|
||||
verified is True if a write to a blob has completed successfully,
|
||||
or a blob has been read to have the same length as specified
|
||||
in init
|
||||
"""
|
||||
@@ -149,7 +167,7 @@ class BlobFile(object):
|
||||
def writer_finished(self, writer, err=None):
|
||||
def fire_finished_deferred():
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.callback(self)
|
||||
@@ -160,7 +178,7 @@ class BlobFile(object):
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.errback(err)
|
||||
@@ -182,7 +200,7 @@ class BlobFile(object):
|
||||
if writer.len_so_far != self.length:
|
||||
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
|
||||
else:
|
||||
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
|
||||
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(None)
|
||||
else:
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import binascii
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from lbrynet.cryptstream.CryptBlob import StreamBlobDecryptor
|
||||
from lbrynet.interfaces import IBlobHandler
|
||||
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
|
||||
|
||||
|
||||
class CryptBlobHandler(object):
|
||||
implements(IBlobHandler)
|
||||
class CryptBlobHandler:
|
||||
#implements(IBlobHandler)
|
||||
|
||||
def __init__(self, key, write_func):
|
||||
self.key = key
|
||||
@@ -1,15 +1,14 @@
|
||||
import binascii
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IStreamDownloader
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.cryptstream.client.CryptBlobHandler import CryptBlobHandler
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.p2p.client.DownloadManager import DownloadManager
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -34,9 +33,9 @@ class CurrentlyStartingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CryptStreamDownloader(object):
|
||||
class CryptStreamDownloader:
|
||||
|
||||
implements(IStreamDownloader)
|
||||
#implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
key, stream_name):
|
||||
@@ -62,8 +61,8 @@ class CryptStreamDownloader(object):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.key = binascii.unhexlify(key)
|
||||
self.stream_name = binascii.unhexlify(stream_name)
|
||||
self.key = unhexlify(key)
|
||||
self.stream_name = unhexlify(stream_name).decode()
|
||||
self.completed = False
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
@@ -1,16 +1,14 @@
|
||||
import binascii
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.StreamDescriptor import save_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbry_file.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
import os
|
||||
from twisted.internet import defer, threads
|
||||
import logging
|
||||
import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.Error import FileOpenError
|
||||
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -21,11 +19,11 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
storage, payment_rate_manager, wallet, key, stream_name, file_name):
|
||||
CryptStreamDownloader.__init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
self.file_name = binascii.unhexlify(os.path.basename(file_name))
|
||||
self.file_name = os.path.basename(unhexlify(file_name).decode())
|
||||
self._calculated_total_bytes = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -46,8 +44,19 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _start(self):
|
||||
def check_start_succeeded(success):
|
||||
if success:
|
||||
self.starting = False
|
||||
self.stopped = False
|
||||
self.completed = False
|
||||
return True
|
||||
else:
|
||||
return self._start_failed()
|
||||
|
||||
self.download_manager = self._get_download_manager()
|
||||
d = self._setup_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader._start(self))
|
||||
d.addCallback(lambda _: self.download_manager.start_downloading())
|
||||
d.addCallbacks(check_start_succeeded)
|
||||
return d
|
||||
|
||||
def _setup_output(self):
|
||||
@@ -90,8 +99,8 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
self.storage, download_manager)
|
||||
|
||||
|
||||
class EncryptedFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
class EncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
self.peer_finder = peer_finder
|
||||
@@ -128,11 +137,11 @@ class EncryptedFileDownloaderFactory(object):
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name):
|
||||
EncryptedFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = binascii.unhexlify(download_directory)
|
||||
self.file_written_to = os.path.join(self.download_directory, binascii.unhexlify(file_name))
|
||||
super().__init__(stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = unhexlify(download_directory).decode()
|
||||
self.file_written_to = os.path.join(self.download_directory, unhexlify(file_name).decode())
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
@@ -156,7 +165,7 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
self.file_written_to = file_written_to
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise ValueError(
|
||||
raise FileOpenError(
|
||||
"Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." % file_written_to
|
||||
)
|
||||
@@ -183,8 +192,8 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
EncryptedFileDownloaderFactory.__init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = binascii.hexlify(download_directory)
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = hexlify(download_directory.encode())
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
stream_name = stream_info.raw_info['stream_name']
|
||||
@@ -1,14 +1,11 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from lbrynet.interfaces import IMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileMetadataHandler(object):
|
||||
implements(IMetadataHandler)
|
||||
class EncryptedFileMetadataHandler:
|
||||
|
||||
def __init__(self, stream_hash, storage, download_manager):
|
||||
self.stream_hash = stream_hash
|
||||
@@ -1,6 +1,6 @@
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.p2p.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
@@ -8,7 +8,7 @@ def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
EncryptedFileOptions())
|
||||
|
||||
|
||||
class EncryptedFileOptions(object):
|
||||
class EncryptedFileOptions:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@@ -3,12 +3,12 @@ import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileCreator(object):
|
||||
class BlobFileCreator:
|
||||
"""
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we do not know the blob hash beforehand (i.e, when creating
|
||||
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
class HashBlobReader:
|
||||
"""
|
||||
This is a file like reader class that supports
|
||||
read(size) and close()
|
||||
@@ -15,7 +15,7 @@ class HashBlobReader(object):
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warn("Garbage collection was called, but reader for %s was not closed yet",
|
||||
log.warning("Garbage collection was called, but reader for %s was not closed yet",
|
||||
self.read_handle.name)
|
||||
self.close()
|
||||
|
||||
@@ -28,5 +28,3 @@ class HashBlobReader(object):
|
||||
return
|
||||
self.read_handle.close()
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobWriter(object):
|
||||
class HashBlobWriter:
|
||||
def __init__(self, length_getter, finished_cb):
|
||||
self.write_handle = BytesIO()
|
||||
self.length_getter = length_getter
|
||||
@@ -18,7 +18,7 @@ class HashBlobWriter(object):
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warn("Garbage collection was called, but writer was not closed yet")
|
||||
log.warning("Garbage collection was called, but writer was not closed yet")
|
||||
self.close()
|
||||
|
||||
@property
|
||||
@@ -27,7 +27,7 @@ class HashBlobWriter(object):
|
||||
|
||||
def write(self, data):
|
||||
if self.write_handle is None:
|
||||
log.exception("writer has already been closed")
|
||||
log.warning("writer has already been closed")
|
||||
raise IOError('I/O operation on closed file')
|
||||
|
||||
self._hashsum.update(data)
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
# dont touch this. CI server changes this during build/deployment
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
@@ -1,24 +1,15 @@
|
||||
import base58
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import yaml
|
||||
import typing
|
||||
import json
|
||||
import logging
|
||||
import envparse
|
||||
import base58
|
||||
import yaml
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import InvalidCurrencyError, NoSuchDirectoryError
|
||||
from lbrynet.androidhelpers.paths import (
|
||||
android_internal_storage_dir,
|
||||
android_app_internal_storage_dir
|
||||
)
|
||||
|
||||
try:
|
||||
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
|
||||
except (ImportError, ValueError, NameError):
|
||||
# Android platform: NameError: name 'c_wchar' is not defined
|
||||
pass
|
||||
from lbrynet import utils
|
||||
from lbrynet.p2p.Error import InvalidCurrencyError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,6 +20,7 @@ ENV_NAMESPACE = 'LBRY_'
|
||||
LBRYCRD_WALLET = 'lbrycrd'
|
||||
LBRYUM_WALLET = 'lbryum'
|
||||
PTC_WALLET = 'ptc'
|
||||
TORBA_WALLET = 'torba'
|
||||
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
APP_NAME = 'LBRY'
|
||||
@@ -58,108 +50,68 @@ settings_encoders = {
|
||||
'.yml': yaml.safe_dump
|
||||
}
|
||||
|
||||
# set by CLI when the user specifies an alternate config file path
|
||||
conf_file = None
|
||||
|
||||
|
||||
def _win_path_to_bytes(path):
|
||||
"""
|
||||
Encode Windows paths to string. appdirs.user_data_dir()
|
||||
on windows will return unicode path, unlike other platforms
|
||||
which returns string. This will cause problems
|
||||
because we use strings for filenames and combining them with
|
||||
os.path.join() will result in errors.
|
||||
"""
|
||||
for encoding in ('ASCII', 'MBCS'):
|
||||
try:
|
||||
return path.encode(encoding)
|
||||
except (UnicodeEncodeError, LookupError):
|
||||
pass
|
||||
return path
|
||||
|
||||
|
||||
def _get_old_directories(platform_type):
|
||||
directories = {}
|
||||
if platform_type == WINDOWS:
|
||||
appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
|
||||
directories['data'] = os.path.join(appdata, 'lbrynet')
|
||||
directories['lbryum'] = os.path.join(appdata, 'lbryum')
|
||||
directories['download'] = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
elif platform_type == DARWIN:
|
||||
directories['data'] = user_data_dir('LBRY')
|
||||
directories['lbryum'] = os.path.expanduser('~/.lbryum')
|
||||
directories['download'] = os.path.expanduser('~/Downloads')
|
||||
elif platform_type == LINUX:
|
||||
directories['data'] = os.path.expanduser('~/.lbrynet')
|
||||
directories['lbryum'] = os.path.expanduser('~/.lbryum')
|
||||
directories['download'] = os.path.expanduser('~/Downloads')
|
||||
else:
|
||||
raise ValueError('unknown platform value')
|
||||
return directories
|
||||
|
||||
|
||||
def _get_new_directories(platform_type):
|
||||
directories = {}
|
||||
if platform_type == ANDROID:
|
||||
directories['data'] = '%s/lbrynet' % android_app_internal_storage_dir()
|
||||
directories['lbryum'] = '%s/lbryum' % android_app_internal_storage_dir()
|
||||
directories['download'] = '%s/Download' % android_internal_storage_dir()
|
||||
elif platform_type == WINDOWS:
|
||||
directories['data'] = user_data_dir('lbrynet', 'lbry')
|
||||
directories['lbryum'] = user_data_dir('lbryum', 'lbry')
|
||||
directories['download'] = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
elif platform_type == DARWIN:
|
||||
directories = _get_old_directories(platform_type)
|
||||
elif platform_type == LINUX:
|
||||
directories['data'] = user_data_dir('lbry/lbrynet')
|
||||
directories['lbryum'] = user_data_dir('lbry/lbryum')
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir)
|
||||
directories['download'] = re.sub('\"', '', down_dir)
|
||||
except EnvironmentError:
|
||||
directories['download'] = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
|
||||
if not directories['download']:
|
||||
directories['download'] = os.path.expanduser('~/Downloads')
|
||||
else:
|
||||
raise ValueError('unknown platform value')
|
||||
return directories
|
||||
|
||||
|
||||
if 'ANDROID_ARGUMENT' in os.environ:
|
||||
# https://github.com/kivy/kivy/blob/master/kivy/utils.py#L417-L421
|
||||
platform = ANDROID
|
||||
dirs = _get_new_directories(ANDROID)
|
||||
elif 'darwin' in sys.platform:
|
||||
platform = DARWIN
|
||||
dirs = _get_old_directories(DARWIN)
|
||||
elif 'win' in sys.platform:
|
||||
platform = WINDOWS
|
||||
if os.path.isdir(_get_old_directories(WINDOWS)['data']) or \
|
||||
os.path.isdir(_get_old_directories(WINDOWS)['lbryum']):
|
||||
dirs = _get_old_directories(WINDOWS)
|
||||
else:
|
||||
dirs = _get_new_directories(WINDOWS)
|
||||
dirs['data'] = _win_path_to_bytes(dirs['data'])
|
||||
dirs['lbryum'] = _win_path_to_bytes(dirs['lbryum'])
|
||||
dirs['download'] = _win_path_to_bytes(dirs['download'])
|
||||
else:
|
||||
platform = LINUX
|
||||
if os.path.isdir(_get_old_directories(LINUX)['data']) or \
|
||||
os.path.isdir(_get_old_directories(LINUX)['lbryum']):
|
||||
dirs = _get_old_directories(LINUX)
|
||||
else:
|
||||
dirs = _get_new_directories(LINUX)
|
||||
|
||||
default_data_dir = dirs['data']
|
||||
default_lbryum_dir = dirs['lbryum']
|
||||
default_download_dir = dirs['download']
|
||||
|
||||
ICON_PATH = 'icons' if platform is WINDOWS else 'app.icns'
|
||||
|
||||
|
||||
def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
from lbrynet.winpaths import get_path, FOLDERID, UserHandle
|
||||
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
|
||||
# old
|
||||
appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
|
||||
data_dir = os.path.join(appdata, 'lbrynet')
|
||||
lbryum_dir = os.path.join(appdata, 'lbryum')
|
||||
if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
# new
|
||||
data_dir = user_data_dir('lbrynet', 'lbry')
|
||||
lbryum_dir = user_data_dir('lbryum', 'lbry')
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
data_dir = user_data_dir('LBRY')
|
||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
download_dir = None
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except EnvironmentError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
|
||||
if not download_dir:
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
|
||||
# old
|
||||
data_dir = os.path.expanduser('~/.lbrynet')
|
||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||
if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
# new
|
||||
return user_data_dir('lbry/lbrynet'), user_data_dir('lbry/lbryum'), download_dir
|
||||
|
||||
|
||||
def server_port(server_and_port):
|
||||
server, port = server_and_port.split(':')
|
||||
return server, int(port)
|
||||
@@ -168,8 +120,10 @@ def server_port(server_and_port):
|
||||
def server_list(servers):
|
||||
return [server_port(server) for server in servers]
|
||||
|
||||
|
||||
def server_list_reverse(servers):
|
||||
return ["%s:%s" % (server, port) for server, port in servers]
|
||||
return [f"{server}:{port}" for server, port in servers]
|
||||
|
||||
|
||||
class Env(envparse.Env):
|
||||
"""An Env parser that automatically namespaces the variables with LBRY"""
|
||||
@@ -180,11 +134,11 @@ class Env(envparse.Env):
|
||||
self._convert_key(key): self._convert_value(value)
|
||||
for key, value in schema.items()
|
||||
}
|
||||
envparse.Env.__init__(self, **my_schema)
|
||||
super().__init__(**my_schema)
|
||||
|
||||
def __call__(self, key, *args, **kwargs):
|
||||
my_key = self._convert_key(key)
|
||||
return super(Env, self).__call__(my_key, *args, **kwargs)
|
||||
return super().__call__(my_key, *args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _convert_key(key):
|
||||
@@ -242,6 +196,11 @@ FIXED_SETTINGS = {
|
||||
}
|
||||
|
||||
ADJUSTABLE_SETTINGS = {
|
||||
'data_dir': (str, ''), # these blank defaults will be updated to OS specific defaults
|
||||
'wallet_dir': (str, ''),
|
||||
'lbryum_wallet_dir': (str, ''), # to be deprecated
|
||||
'download_directory': (str, ''),
|
||||
|
||||
# By default, daemon will block all cross origin requests
|
||||
# but if this is set, this value will be used for the
|
||||
# Access-Control-Allow-Origin. For example
|
||||
@@ -260,17 +219,15 @@ ADJUSTABLE_SETTINGS = {
|
||||
# will not be made automatically)
|
||||
'auto_renew_claim_height_delta': (int, 0),
|
||||
'cache_time': (int, 150),
|
||||
'data_dir': (str, default_data_dir),
|
||||
'data_rate': (float, .0001), # points/megabyte
|
||||
'delete_blobs_on_remove': (bool, True),
|
||||
'dht_node_port': (int, 4444),
|
||||
'download_directory': (str, default_download_dir),
|
||||
'download_timeout': (int, 180),
|
||||
'download_mirrors': (list, ['blobs.lbry.io']),
|
||||
'is_generous_host': (bool, True),
|
||||
'announce_head_blobs_only': (bool, True),
|
||||
'concurrent_announcers': (int, DEFAULT_CONCURRENT_ANNOUNCERS),
|
||||
'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list, server_list_reverse),
|
||||
'lbryum_wallet_dir': (str, default_lbryum_dir),
|
||||
'max_connections_per_stream': (int, 5),
|
||||
'seek_head_blob_first': (bool, True),
|
||||
# TODO: writing json on the cmd line is a pain, come up with a nicer
|
||||
@@ -287,32 +244,52 @@ ADJUSTABLE_SETTINGS = {
|
||||
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
|
||||
'reflect_uploads': (bool, True),
|
||||
'auto_re_reflect_interval': (int, 86400), # set to 0 to disable
|
||||
'reflector_servers': (list, [('reflector2.lbry.io', 5566)], server_list, server_list_reverse),
|
||||
'run_reflector_server': (bool, False),
|
||||
'reflector_servers': (list, [('reflector.lbry.io', 5566)], server_list, server_list_reverse),
|
||||
'run_reflector_server': (bool, False), # adds `reflector` to components_to_skip unless True
|
||||
'sd_download_timeout': (int, 3),
|
||||
'share_usage_data': (bool, True), # whether to share usage stats and diagnostic info with LBRY
|
||||
'peer_search_timeout': (int, 30),
|
||||
'peer_search_timeout': (int, 60),
|
||||
'use_auth_http': (bool, False),
|
||||
'use_https': (bool, False),
|
||||
'use_upnp': (bool, True),
|
||||
'use_keyring': (bool, False),
|
||||
'wallet': (str, LBRYUM_WALLET),
|
||||
'blockchain_name': (str, 'lbrycrd_main'),
|
||||
'lbryum_servers': (list, [('lbryumx1.lbry.io', 50001), ('lbryumx2.lbry.io',
|
||||
50001)], server_list, server_list_reverse),
|
||||
's3_headers_depth': (int, 96 * 10) # download headers from s3 when the local height is more than 10 chunks behind
|
||||
's3_headers_depth': (int, 96 * 10), # download headers from s3 when the local height is more than 10 chunks behind
|
||||
'components_to_skip': (list, []) # components which will be skipped during start-up of daemon
|
||||
}
|
||||
|
||||
|
||||
class Config(object):
|
||||
def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None,
|
||||
environment=None, cli_settings=None):
|
||||
optional_str = typing.Optional[str]
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, fixed_defaults, adjustable_defaults: typing.Dict, persisted_settings=None, environment=None,
|
||||
cli_settings=None, data_dir: optional_str = None, wallet_dir: optional_str = None,
|
||||
download_dir: optional_str = None, file_name: optional_str = None):
|
||||
self._installation_id = None
|
||||
self._session_id = base58.b58encode(utils.generate_id())
|
||||
self._session_id = base58.b58encode(utils.generate_id()).decode()
|
||||
self._node_id = None
|
||||
|
||||
self._fixed_defaults = fixed_defaults
|
||||
self._adjustable_defaults = adjustable_defaults
|
||||
|
||||
# copy the default adjustable settings
|
||||
self._adjustable_defaults = {k: v for k, v in adjustable_defaults.items()}
|
||||
|
||||
# set the os specific default directories
|
||||
if platform is WINDOWS:
|
||||
self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_windows_directories()
|
||||
elif platform is DARWIN:
|
||||
self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_darwin_directories()
|
||||
elif platform is LINUX:
|
||||
self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_linux_directories()
|
||||
else:
|
||||
assert None not in [data_dir, wallet_dir, download_dir]
|
||||
self.default_data_dir = data_dir
|
||||
self.default_wallet_dir = wallet_dir
|
||||
self.default_download_dir = download_dir
|
||||
|
||||
self._data = {
|
||||
TYPE_DEFAULT: {}, # defaults
|
||||
@@ -334,7 +311,7 @@ class Config(object):
|
||||
|
||||
self._data[TYPE_DEFAULT].update(self._fixed_defaults)
|
||||
self._data[TYPE_DEFAULT].update(
|
||||
{k: v[1] for (k, v) in self._adjustable_defaults.iteritems()})
|
||||
{k: v[1] for (k, v) in self._adjustable_defaults.items()})
|
||||
|
||||
if persisted_settings is None:
|
||||
persisted_settings = {}
|
||||
@@ -349,12 +326,37 @@ class Config(object):
|
||||
cli_settings = {}
|
||||
self._validate_settings(cli_settings)
|
||||
self._data[TYPE_CLI].update(cli_settings)
|
||||
self.file_name = file_name or 'daemon_settings.yml'
|
||||
|
||||
@property
|
||||
def data_dir(self) -> optional_str:
|
||||
data_dir = self.get('data_dir')
|
||||
if not data_dir:
|
||||
return
|
||||
return os.path.expanduser(os.path.expandvars(data_dir))
|
||||
|
||||
@property
|
||||
def download_dir(self) -> optional_str:
|
||||
download_dir = self.get('download_directory')
|
||||
if not download_dir:
|
||||
return
|
||||
return os.path.expanduser(os.path.expandvars(download_dir))
|
||||
|
||||
@property
|
||||
def wallet_dir(self) -> optional_str:
|
||||
if self.get('lbryum_wallet_dir') and not self.get('wallet_dir'):
|
||||
log.warning("'lbryum_wallet_dir' setting will be deprecated, please update to 'wallet_dir'")
|
||||
self['wallet_dir'] = self['lbryum_wallet_dir']
|
||||
wallet_dir = self.get('wallet_dir')
|
||||
if not wallet_dir:
|
||||
return
|
||||
return os.path.expanduser(os.path.expandvars(wallet_dir))
|
||||
|
||||
def __repr__(self):
|
||||
return self.get_current_settings_dict().__repr__()
|
||||
|
||||
def __iter__(self):
|
||||
for k in self._data[TYPE_DEFAULT].iterkeys():
|
||||
for k in self._data[TYPE_DEFAULT].keys():
|
||||
yield k
|
||||
|
||||
def __getitem__(self, name):
|
||||
@@ -378,7 +380,7 @@ class Config(object):
|
||||
|
||||
def _assert_valid_data_type(self, data_type):
|
||||
if data_type not in self._data:
|
||||
raise KeyError('{} in is not a valid data type'.format(data_type))
|
||||
raise KeyError(f'{data_type} in is not a valid data type')
|
||||
|
||||
def get_valid_setting_names(self):
|
||||
return self._data[TYPE_DEFAULT].keys()
|
||||
@@ -388,7 +390,7 @@ class Config(object):
|
||||
|
||||
def _assert_valid_setting(self, name):
|
||||
if not self._is_valid_setting(name):
|
||||
raise KeyError('{} is not a valid setting'.format(name))
|
||||
raise KeyError(f'{name} is not a valid setting')
|
||||
|
||||
def _validate_settings(self, data):
|
||||
invalid_settings = set(data.keys()) - set(self.get_valid_setting_names())
|
||||
@@ -398,7 +400,7 @@ class Config(object):
|
||||
def _assert_editable_setting(self, name):
|
||||
self._assert_valid_setting(name)
|
||||
if name in self._fixed_defaults:
|
||||
raise ValueError('{} is not an editable setting'.format(name))
|
||||
raise ValueError(f'{name} is not an editable setting')
|
||||
|
||||
def _assert_valid_setting_value(self, name, value):
|
||||
if name == "max_key_fee":
|
||||
@@ -408,7 +410,7 @@ class Config(object):
|
||||
elif name == "download_directory":
|
||||
directory = str(value)
|
||||
if not os.path.exists(directory):
|
||||
raise NoSuchDirectoryError(directory)
|
||||
log.warning("download directory '%s' does not exist", directory)
|
||||
|
||||
def is_default(self, name):
|
||||
"""Check if a config value is wasn't specified by the user
|
||||
@@ -453,7 +455,7 @@ class Config(object):
|
||||
for possible_data_type in self._search_order:
|
||||
if name in self._data[possible_data_type]:
|
||||
return self._data[possible_data_type][name]
|
||||
raise KeyError('{} is not a valid setting'.format(name))
|
||||
raise KeyError(f'{name} is not a valid setting')
|
||||
|
||||
def set(self, name, value, data_types=(TYPE_RUNTIME,)):
|
||||
"""Set a config value
|
||||
@@ -477,7 +479,7 @@ class Config(object):
|
||||
self._data[data_type][name] = value
|
||||
|
||||
def update(self, updated_settings, data_types=(TYPE_RUNTIME,)):
|
||||
for k, v in updated_settings.iteritems():
|
||||
for k, v in updated_settings.items():
|
||||
try:
|
||||
self.set(k, v, data_types=data_types)
|
||||
except (KeyError, AssertionError):
|
||||
@@ -491,28 +493,26 @@ class Config(object):
|
||||
|
||||
def get_adjustable_settings_dict(self):
|
||||
return {
|
||||
key: val for key, val in self.get_current_settings_dict().iteritems()
|
||||
key: val for key, val in self.get_current_settings_dict().items()
|
||||
if key in self._adjustable_defaults
|
||||
}
|
||||
|
||||
def save_conf_file_settings(self):
|
||||
if conf_file:
|
||||
path = conf_file
|
||||
else:
|
||||
path = self.get_conf_filename()
|
||||
# reverse the conversions done after loading the settings from the conf
|
||||
# file
|
||||
rev = self._convert_conf_file_lists_reverse(self._data[TYPE_PERSISTED])
|
||||
ext = os.path.splitext(path)[1]
|
||||
ext = os.path.splitext(self.file_name)[1]
|
||||
encoder = settings_encoders.get(ext, False)
|
||||
assert encoder is not False, 'Unknown settings format %s' % ext
|
||||
with open(path, 'w') as settings_file:
|
||||
if not encoder:
|
||||
raise ValueError('Unknown settings format: {}. Available formats: {}'
|
||||
.format(ext, list(settings_encoders.keys())))
|
||||
with open(os.path.join(self.data_dir, self.file_name), 'w') as settings_file:
|
||||
settings_file.write(encoder(rev))
|
||||
|
||||
@staticmethod
|
||||
def _convert_conf_file_lists_reverse(converted):
|
||||
rev = {}
|
||||
for k in converted.iterkeys():
|
||||
for k in converted.keys():
|
||||
if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) == 4:
|
||||
rev[k] = ADJUSTABLE_SETTINGS[k][3](converted[k])
|
||||
else:
|
||||
@@ -522,7 +522,7 @@ class Config(object):
|
||||
@staticmethod
|
||||
def _convert_conf_file_lists(decoded):
|
||||
converted = {}
|
||||
for k, v in decoded.iteritems():
|
||||
for k, v in decoded.items():
|
||||
if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) >= 3:
|
||||
converted[k] = ADJUSTABLE_SETTINGS[k][2](v)
|
||||
else:
|
||||
@@ -534,25 +534,29 @@ class Config(object):
|
||||
settings.node_id = settings.get_node_id()
|
||||
|
||||
def load_conf_file_settings(self):
|
||||
if conf_file:
|
||||
path = conf_file
|
||||
else:
|
||||
path = self.get_conf_filename()
|
||||
path = os.path.join(self.data_dir or self.default_data_dir, self.file_name)
|
||||
if os.path.isfile(path):
|
||||
self._read_conf_file(path)
|
||||
self['data_dir'] = self.data_dir or self.default_data_dir
|
||||
self['download_directory'] = self.download_dir or self.default_download_dir
|
||||
self['wallet_dir'] = self.wallet_dir or self.default_wallet_dir
|
||||
# initialize members depending on config file
|
||||
self.initialize_post_conf_load()
|
||||
|
||||
def _read_conf_file(self, path):
|
||||
if not path or not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
ext = os.path.splitext(path)[1]
|
||||
decoder = settings_decoders.get(ext, False)
|
||||
assert decoder is not False, 'Unknown settings format %s' % ext
|
||||
try:
|
||||
with open(path, 'r') as settings_file:
|
||||
data = settings_file.read()
|
||||
decoded = self._fix_old_conf_file_settings(decoder(data))
|
||||
log.info('Loaded settings file: %s', path)
|
||||
self._validate_settings(decoded)
|
||||
self._data[TYPE_PERSISTED].update(self._convert_conf_file_lists(decoded))
|
||||
except (IOError, OSError) as err:
|
||||
log.info('%s: Failed to update settings from %s', err, path)
|
||||
|
||||
#initialize members depending on config file
|
||||
self.initialize_post_conf_load()
|
||||
if not decoder:
|
||||
raise ValueError('Unknown settings format: {}. Available formats: {}'
|
||||
.format(ext, list(settings_decoders.keys())))
|
||||
with open(path, 'r') as settings_file:
|
||||
data = settings_file.read()
|
||||
decoded = self._fix_old_conf_file_settings(decoder(data))
|
||||
log.info('Loaded settings file: %s', path)
|
||||
self._validate_settings(decoded)
|
||||
self._data[TYPE_PERSISTED].update(self._convert_conf_file_lists(decoded))
|
||||
|
||||
def _fix_old_conf_file_settings(self, settings_dict):
|
||||
if 'API_INTERFACE' in settings_dict:
|
||||
@@ -566,7 +570,7 @@ class Config(object):
|
||||
if 'share_debug_info' in settings_dict:
|
||||
settings_dict['share_usage_data'] = settings_dict['share_debug_info']
|
||||
del settings_dict['share_debug_info']
|
||||
for key in settings_dict.keys():
|
||||
for key in list(settings_dict.keys()):
|
||||
if not self._is_valid_setting(key):
|
||||
log.warning('Ignoring invalid conf file setting: %s', key)
|
||||
del settings_dict[key]
|
||||
@@ -576,9 +580,15 @@ class Config(object):
|
||||
# although there is a risk of a race condition here we don't
|
||||
# expect there to be multiple processes accessing this
|
||||
# directory so the risk can be ignored
|
||||
if not os.path.isdir(self['data_dir']):
|
||||
os.makedirs(self['data_dir'])
|
||||
return self['data_dir']
|
||||
if not os.path.isdir(self.data_dir):
|
||||
os.makedirs(self.data_dir)
|
||||
if not os.path.isdir(os.path.join(self.data_dir, "blobfiles")):
|
||||
os.makedirs(os.path.join(self.data_dir, "blobfiles"))
|
||||
return self.data_dir
|
||||
|
||||
def ensure_wallet_dir(self):
|
||||
if not os.path.isdir(self.wallet_dir):
|
||||
os.makedirs(self.wallet_dir)
|
||||
|
||||
def get_log_filename(self):
|
||||
"""
|
||||
@@ -587,26 +597,18 @@ class Config(object):
|
||||
"""
|
||||
return os.path.join(self.ensure_data_dir(), self['LOG_FILE_NAME'])
|
||||
|
||||
def get_api_connection_string(self):
|
||||
return 'http://%s:%i/%s' % (self['api_host'], self['api_port'], self['API_ADDRESS'])
|
||||
|
||||
def get_ui_address(self):
|
||||
return 'http://%s:%i' % (self['api_host'], self['api_port'])
|
||||
def get_api_connection_string(self, user: str = None, password: str = None) -> str:
|
||||
return 'http%s://%s%s:%i/%s' % (
|
||||
"" if not self['use_https'] else "s",
|
||||
"" if not (user and password) else f"{user}:{password}@",
|
||||
self['api_host'],
|
||||
self['api_port'],
|
||||
self['API_ADDRESS']
|
||||
)
|
||||
|
||||
def get_db_revision_filename(self):
|
||||
return os.path.join(self.ensure_data_dir(), self['DB_REVISION_FILE_NAME'])
|
||||
|
||||
def get_conf_filename(self):
|
||||
data_dir = self.ensure_data_dir()
|
||||
yml_path = os.path.join(data_dir, 'daemon_settings.yml')
|
||||
json_path = os.path.join(data_dir, 'daemon_settings.json')
|
||||
if os.path.isfile(yml_path):
|
||||
return yml_path
|
||||
elif os.path.isfile(json_path):
|
||||
return json_path
|
||||
else:
|
||||
return yml_path
|
||||
|
||||
def get_installation_id(self):
|
||||
install_id_filename = os.path.join(self.ensure_data_dir(), "install_id")
|
||||
if not self._installation_id:
|
||||
@@ -614,7 +616,7 @@ class Config(object):
|
||||
with open(install_id_filename, "r") as install_id_file:
|
||||
self._installation_id = str(install_id_file.read()).strip()
|
||||
if not self._installation_id:
|
||||
self._installation_id = base58.b58encode(utils.generate_id())
|
||||
self._installation_id = base58.b58encode(utils.generate_id()).decode()
|
||||
with open(install_id_filename, "w") as install_id_file:
|
||||
install_id_file.write(self._installation_id)
|
||||
return self._installation_id
|
||||
@@ -628,20 +630,19 @@ class Config(object):
|
||||
if not self._node_id:
|
||||
self._node_id = utils.generate_id()
|
||||
with open(node_id_filename, "w") as node_id_file:
|
||||
node_id_file.write(base58.b58encode(self._node_id))
|
||||
node_id_file.write(base58.b58encode(self._node_id).decode())
|
||||
return self._node_id
|
||||
|
||||
def get_session_id(self):
|
||||
return self._session_id
|
||||
|
||||
|
||||
# type: Config
|
||||
settings = None
|
||||
settings: Config = None
|
||||
|
||||
|
||||
def get_default_env():
|
||||
env_defaults = {}
|
||||
for k, v in ADJUSTABLE_SETTINGS.iteritems():
|
||||
for k, v in ADJUSTABLE_SETTINGS.items():
|
||||
if len(v) == 3:
|
||||
env_defaults[k] = (v[0], None, v[2])
|
||||
elif len(v) == 4:
|
||||
@@ -651,10 +652,18 @@ def get_default_env():
|
||||
return Env(**env_defaults)
|
||||
|
||||
|
||||
def initialize_settings(load_conf_file=True):
|
||||
def initialize_settings(load_conf_file: typing.Optional[bool] = True,
|
||||
data_dir: optional_str = None, wallet_dir: optional_str = None,
|
||||
download_dir: optional_str = None):
|
||||
global settings
|
||||
if settings is None:
|
||||
settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS,
|
||||
environment=get_default_env())
|
||||
environment=get_default_env(), data_dir=data_dir, wallet_dir=wallet_dir,
|
||||
download_dir=download_dir)
|
||||
if load_conf_file:
|
||||
settings.load_conf_file_settings()
|
||||
settings['data_dir'] = settings.data_dir or settings.default_data_dir
|
||||
settings['download_directory'] = settings.download_dir or settings.default_download_dir
|
||||
settings['wallet_dir'] = settings.wallet_dir or settings.default_wallet_dir
|
||||
settings.ensure_data_dir()
|
||||
settings.ensure_wallet_dir()
|
||||
@@ -1,282 +0,0 @@
|
||||
import logging
|
||||
import miniupnpc
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.BlobManager import DiskBlobManager
|
||||
from lbrynet.dht import node, hashannouncer
|
||||
from lbrynet.database.storage import SQLiteStorage
|
||||
from lbrynet.core.RateLimiter import RateLimiter
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager, OnlyFreePaymentsManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Session(object):
|
||||
"""This class manages all important services common to any application that uses the network.
|
||||
|
||||
the hash announcer, which informs other peers that this peer is
|
||||
associated with some hash. Usually, this means this peer has a
|
||||
blob identified by the hash in question, but it can be used for
|
||||
other purposes.
|
||||
|
||||
the peer finder, which finds peers that are associated with some
|
||||
hash.
|
||||
|
||||
the blob manager, which keeps track of which blobs have been
|
||||
downloaded and provides access to them,
|
||||
|
||||
the rate limiter, which attempts to ensure download and upload
|
||||
rates stay below a set maximum
|
||||
|
||||
upnp, which opens holes in compatible firewalls so that remote
|
||||
peers can connect to this peer.
|
||||
"""
|
||||
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None, node_id=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None, hash_announcer=None, blob_dir=None, blob_manager=None,
|
||||
peer_port=None, use_upnp=True, rate_limiter=None, wallet=None, dht_node_class=node.Node,
|
||||
blob_tracker_class=None, payment_rate_manager_class=None, is_generous=True, external_ip=None,
|
||||
storage=None):
|
||||
"""@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
@param db_dir: The directory in which levelDB files should be stored
|
||||
|
||||
@param node_id: The unique ID of this node
|
||||
|
||||
@param peer_manager: An object which keeps track of all known
|
||||
peers. If None, a PeerManager will be created
|
||||
|
||||
@param dht_node_port: The port on which the dht node should
|
||||
listen for incoming connections
|
||||
|
||||
@param known_dht_nodes: A list of nodes which the dht node
|
||||
should use to bootstrap into the dht
|
||||
|
||||
@param peer_finder: An object which is used to look up peers
|
||||
that are associated with some hash. If None, a
|
||||
DHTPeerFinder will be used, which looks for peers in the
|
||||
distributed hash table.
|
||||
|
||||
@param hash_announcer: An object which announces to other
|
||||
peers that this peer is associated with some hash. If
|
||||
None, and peer_port is not None, a DHTHashAnnouncer will
|
||||
be used. If None and peer_port is None, a
|
||||
DummyHashAnnouncer will be used, which will not actually
|
||||
announce anything.
|
||||
|
||||
@param blob_dir: The directory in which blobs will be
|
||||
stored. If None and blob_manager is None, blobs will be
|
||||
stored in memory only.
|
||||
|
||||
@param blob_manager: An object which keeps track of downloaded
|
||||
blobs and provides access to them. If None, and blob_dir
|
||||
is not None, a DiskBlobManager will be used, with the
|
||||
given blob_dir. If None and blob_dir is None, a
|
||||
TempBlobManager will be used, which stores blobs in memory
|
||||
only.
|
||||
|
||||
@param peer_port: The port on which other peers should connect
|
||||
to this peer
|
||||
|
||||
@param use_upnp: Whether or not to try to open a hole in the
|
||||
firewall so that outside peers can connect to this peer's
|
||||
peer_port and dht_node_port
|
||||
|
||||
@param rate_limiter: An object which keeps track of the amount
|
||||
of data transferred to and from this peer, and can limit
|
||||
that rate if desired
|
||||
|
||||
@param wallet: An object which will be used to keep track of
|
||||
expected payments and which will pay peers. If None, a
|
||||
wallet which uses the Point Trader system will be used,
|
||||
which is meant for testing only
|
||||
|
||||
"""
|
||||
self.db_dir = db_dir
|
||||
self.node_id = node_id
|
||||
self.peer_manager = peer_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.hash_announcer = hash_announcer
|
||||
self.dht_node_port = dht_node_port
|
||||
self.known_dht_nodes = known_dht_nodes
|
||||
if self.known_dht_nodes is None:
|
||||
self.known_dht_nodes = []
|
||||
self.blob_dir = blob_dir
|
||||
self.blob_manager = blob_manager
|
||||
# self.blob_tracker = None
|
||||
# self.blob_tracker_class = blob_tracker_class or BlobAvailabilityTracker
|
||||
self.peer_port = peer_port
|
||||
self.use_upnp = use_upnp
|
||||
self.rate_limiter = rate_limiter
|
||||
self.external_ip = external_ip
|
||||
self.upnp_redirects = []
|
||||
self.wallet = wallet
|
||||
self.dht_node_class = dht_node_class
|
||||
self.dht_node = None
|
||||
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
|
||||
self.payment_rate_manager = OnlyFreePaymentsManager()
|
||||
# self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager
|
||||
# self.is_generous = is_generous
|
||||
self.storage = storage or SQLiteStorage(self.db_dir)
|
||||
|
||||
def setup(self):
|
||||
"""Create the blob directory and database if necessary, start all desired services"""
|
||||
|
||||
log.debug("Starting session.")
|
||||
|
||||
if self.node_id is None:
|
||||
self.node_id = generate_id()
|
||||
|
||||
if self.use_upnp is True:
|
||||
d = self._try_upnp()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: self.storage.setup())
|
||||
d.addCallback(lambda _: self._setup_dht())
|
||||
d.addCallback(lambda _: self._setup_other_components())
|
||||
return d
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop all services"""
|
||||
log.info('Stopping session.')
|
||||
ds = []
|
||||
if self.hash_announcer:
|
||||
self.hash_announcer.stop()
|
||||
# if self.blob_tracker is not None:
|
||||
# ds.append(defer.maybeDeferred(self.blob_tracker.stop))
|
||||
if self.dht_node is not None:
|
||||
ds.append(defer.maybeDeferred(self.dht_node.stop))
|
||||
if self.rate_limiter is not None:
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
if self.wallet is not None:
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.use_upnp is True:
|
||||
ds.append(defer.maybeDeferred(self._unset_upnp))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _try_upnp(self):
|
||||
|
||||
log.debug("In _try_upnp")
|
||||
|
||||
def get_free_port(upnp, port, protocol):
|
||||
# returns an existing mapping if it exists
|
||||
mapping = upnp.getspecificportmapping(port, protocol)
|
||||
if not mapping:
|
||||
return port
|
||||
if upnp.lanaddr == mapping[0]:
|
||||
return mapping[1]
|
||||
return get_free_port(upnp, port + 1, protocol)
|
||||
|
||||
def get_port_mapping(upnp, port, protocol, description):
|
||||
# try to map to the requested port, if there is already a mapping use the next external
|
||||
# port available
|
||||
if protocol not in ['UDP', 'TCP']:
|
||||
raise Exception("invalid protocol")
|
||||
port = get_free_port(upnp, port, protocol)
|
||||
if isinstance(port, tuple):
|
||||
log.info("Found existing UPnP redirect %s:%i (%s) to %s:%i, using it",
|
||||
self.external_ip, port, protocol, upnp.lanaddr, port)
|
||||
return port
|
||||
upnp.addportmapping(port, protocol, upnp.lanaddr, port,
|
||||
description, '')
|
||||
log.info("Set UPnP redirect %s:%i (%s) to %s:%i", self.external_ip, port,
|
||||
protocol, upnp.lanaddr, port)
|
||||
return port
|
||||
|
||||
def threaded_try_upnp():
|
||||
if self.use_upnp is False:
|
||||
log.debug("Not using upnp")
|
||||
return False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
external_ip = u.externalipaddress()
|
||||
if external_ip != '0.0.0.0' and not self.external_ip:
|
||||
# best not to rely on this external ip, the router can be behind layers of NATs
|
||||
self.external_ip = external_ip
|
||||
if self.peer_port:
|
||||
self.peer_port = get_port_mapping(u, self.peer_port, 'TCP', 'LBRY peer port')
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
if self.dht_node_port:
|
||||
self.dht_node_port = get_port_mapping(u, self.dht_node_port, 'UDP', 'LBRY DHT port')
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
return True
|
||||
return False
|
||||
|
||||
def upnp_failed(err):
|
||||
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
return False
|
||||
|
||||
d = threads.deferToThread(threaded_try_upnp)
|
||||
d.addErrback(upnp_failed)
|
||||
return d
|
||||
|
||||
def _setup_dht(self): # does not block startup, the dht will re-attempt if necessary
|
||||
self.dht_node = self.dht_node_class(
|
||||
node_id=self.node_id,
|
||||
udpPort=self.dht_node_port,
|
||||
externalIP=self.external_ip,
|
||||
peerPort=self.peer_port,
|
||||
peer_manager=self.peer_manager,
|
||||
peer_finder=self.peer_finder,
|
||||
)
|
||||
if not self.hash_announcer:
|
||||
self.hash_announcer = hashannouncer.DHTHashAnnouncer(self.dht_node, self.storage)
|
||||
self.peer_manager = self.dht_node.peer_manager
|
||||
self.peer_finder = self.dht_node.peer_finder
|
||||
d = self.dht_node.start(self.known_dht_nodes)
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
d.addCallback(lambda _: self.hash_announcer.start())
|
||||
|
||||
def _setup_other_components(self):
|
||||
log.debug("Setting up the rest of the components")
|
||||
|
||||
if self.rate_limiter is None:
|
||||
self.rate_limiter = RateLimiter()
|
||||
|
||||
if self.blob_manager is None:
|
||||
if self.blob_dir is None:
|
||||
raise Exception(
|
||||
"TempBlobManager is no longer supported, specify BlobManager or db_dir")
|
||||
else:
|
||||
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage, self.dht_node._dataStore)
|
||||
|
||||
# if self.blob_tracker is None:
|
||||
# self.blob_tracker = self.blob_tracker_class(
|
||||
# self.blob_manager, self.dht_node.peer_finder, self.dht_node
|
||||
# )
|
||||
# if self.payment_rate_manager is None:
|
||||
# self.payment_rate_manager = self.payment_rate_manager_class(
|
||||
# self.base_payment_rate_manager, self.blob_tracker, self.is_generous
|
||||
# )
|
||||
|
||||
self.rate_limiter.start()
|
||||
d = self.blob_manager.setup()
|
||||
d.addCallback(lambda _: self.wallet.start())
|
||||
# d.addCallback(lambda _: self.blob_tracker.start())
|
||||
return d
|
||||
|
||||
def _unset_upnp(self):
|
||||
log.info("Unsetting upnp for session")
|
||||
|
||||
def threaded_unset_upnp():
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
for port, protocol in self.upnp_redirects:
|
||||
if u.getspecificportmapping(port, protocol) is None:
|
||||
log.warning(
|
||||
"UPnP redirect for %s %d was removed by something else.",
|
||||
protocol, port)
|
||||
else:
|
||||
u.deleteportmapping(port, protocol)
|
||||
log.info("Removed UPnP redirect for %s %d.", protocol, port)
|
||||
self.upnp_redirects = []
|
||||
|
||||
d = threads.deferToThread(threaded_unset_upnp)
|
||||
d.addErrback(lambda err: str(err))
|
||||
return d
|
||||
@@ -1,17 +0,0 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_read_handle(path):
|
||||
"""
|
||||
Get os independent read handle for a file
|
||||
"""
|
||||
|
||||
if os.name == "nt":
|
||||
file_mode = 'rb'
|
||||
else:
|
||||
file_mode = 'r'
|
||||
read_handle = open(path, file_mode)
|
||||
yield read_handle
|
||||
read_handle.close()
|
||||
@@ -1,54 +0,0 @@
|
||||
import platform
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
from urllib2 import urlopen, URLError
|
||||
from lbryschema import __version__ as lbryschema_version
|
||||
from lbryum import __version__ as LBRYUM_VERSION
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from lbrynet.conf import ROOT_DIR
|
||||
|
||||
|
||||
def get_lbrynet_version():
|
||||
if build_type.BUILD == "dev":
|
||||
try:
|
||||
with open(os.devnull, 'w') as devnull:
|
||||
git_dir = ROOT_DIR + '/.git'
|
||||
return subprocess.check_output(
|
||||
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
|
||||
stderr=devnull
|
||||
).strip().lstrip('v')
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
print "failed to get version from git"
|
||||
return lbrynet_version
|
||||
|
||||
|
||||
def get_platform(get_ip=True):
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": get_lbrynet_version(),
|
||||
"lbryum_version": LBRYUM_VERSION,
|
||||
"lbryschema_version": lbryschema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
|
||||
# TODO: remove this from get_platform and add a get_external_ip function using treq
|
||||
if get_ip:
|
||||
try:
|
||||
response = json.loads(urlopen("https://api.lbry.io/ip").read())
|
||||
if not response['success']:
|
||||
raise URLError("failed to get external ip")
|
||||
p['ip'] = response['data']['ip']
|
||||
except (URLError, AssertionError):
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
@@ -1,8 +0,0 @@
|
||||
"""
|
||||
Classes and functions for dealing with Crypt Streams.
|
||||
|
||||
Crypt Streams are encrypted blobs and metadata tying those blobs together. At least some of the
|
||||
metadata is generally stored in a Stream Descriptor File, for example containing a public key
|
||||
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
|
||||
may or may not be present.
|
||||
"""
|
||||
@@ -0,0 +1,109 @@
|
||||
import os
|
||||
import sys
|
||||
import inspect
|
||||
import logging
|
||||
TRACE = 5
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging{}__init__{}".format(os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
|
||||
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to substitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
def install_logger():
|
||||
current = logging.getLoggerClass()
|
||||
if current is not Logger:
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
@@ -1,246 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import colorama
|
||||
from docopt import docopt
|
||||
from collections import OrderedDict
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.daemon.auth.client import JSONRPCException, LBRYAPIClient, AuthAPIClient
|
||||
from lbrynet.daemon.Daemon import LOADING_WALLET_CODE, Daemon
|
||||
from lbrynet.core.system_info import get_platform
|
||||
from jsonrpc.common import RPCError
|
||||
from requests.exceptions import ConnectionError
|
||||
from urllib2 import URLError, HTTPError
|
||||
from httplib import UNAUTHORIZED
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_flag_vals(flag_names, parsed_args):
|
||||
kwargs = OrderedDict()
|
||||
for key, arg in parsed_args.iteritems():
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--"):
|
||||
if remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif key in flag_names:
|
||||
if remove_brackets(flag_names[key]) not in kwargs:
|
||||
k = remove_brackets(flag_names[key])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = guess_type(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def main():
|
||||
argv = sys.argv[1:]
|
||||
|
||||
# check if a config file has been specified. If so, shift
|
||||
# all the arguments so that the parsing can continue without
|
||||
# noticing
|
||||
if len(argv) and argv[0] == "--conf":
|
||||
if len(argv) < 2:
|
||||
print_error("No config file specified for --conf option")
|
||||
print_help()
|
||||
return
|
||||
|
||||
conf.conf_file = argv[1]
|
||||
argv = argv[2:]
|
||||
|
||||
if len(argv):
|
||||
method, args = argv[0], argv[1:]
|
||||
else:
|
||||
print_help()
|
||||
return
|
||||
|
||||
if method in ['help', '--help', '-h']:
|
||||
if len(args) == 1:
|
||||
print_help_for_command(args[0])
|
||||
else:
|
||||
print_help()
|
||||
return
|
||||
|
||||
elif method in ['version', '--version']:
|
||||
print utils.json_dumps_pretty(get_platform(get_ip=False))
|
||||
return
|
||||
|
||||
if method not in Daemon.callable_methods:
|
||||
if method not in Daemon.deprecated_methods:
|
||||
print_error("\"%s\" is not a valid command." % method)
|
||||
return
|
||||
new_method = Daemon.deprecated_methods[method]._new_command
|
||||
print_error("\"%s\" is deprecated, using \"%s\"." % (method, new_method))
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
if hasattr(fn, "_flags"):
|
||||
flag_names = fn._flags
|
||||
else:
|
||||
flag_names = {}
|
||||
|
||||
parsed = docopt(fn.__doc__, args)
|
||||
kwargs = set_flag_vals(flag_names, parsed)
|
||||
colorama.init()
|
||||
conf.initialize_settings()
|
||||
|
||||
try:
|
||||
api = LBRYAPIClient.get_client()
|
||||
status = api.status()
|
||||
except (URLError, ConnectionError) as err:
|
||||
if isinstance(err, HTTPError) and err.code == UNAUTHORIZED:
|
||||
api = AuthAPIClient.config()
|
||||
# this can happen if the daemon is using auth with the --http-auth flag
|
||||
# when the config setting is to not use it
|
||||
try:
|
||||
status = api.status()
|
||||
except:
|
||||
print_error("Daemon requires authentication, but none was provided.",
|
||||
suggest_help=False)
|
||||
return 1
|
||||
else:
|
||||
print_error("Could not connect to daemon. Are you sure it's running?",
|
||||
suggest_help=False)
|
||||
return 1
|
||||
|
||||
status_code = status['startup_status']['code']
|
||||
|
||||
if status_code != "started" and method not in Daemon.allowed_during_startup:
|
||||
print "Daemon is in the process of starting. Please try again in a bit."
|
||||
message = status['startup_status']['message']
|
||||
if message:
|
||||
if (
|
||||
status['startup_status']['code'] == LOADING_WALLET_CODE
|
||||
and status['blockchain_status']['blocks_behind'] > 0
|
||||
):
|
||||
message += '. Blocks left: ' + str(status['blockchain_status']['blocks_behind'])
|
||||
print " Status: " + message
|
||||
return 1
|
||||
|
||||
# TODO: check if port is bound. Error if its not
|
||||
|
||||
try:
|
||||
result = api.call(method, kwargs)
|
||||
if isinstance(result, basestring):
|
||||
# printing the undumped string is prettier
|
||||
print result
|
||||
else:
|
||||
print utils.json_dumps_pretty(result)
|
||||
except (RPCError, KeyError, JSONRPCException, HTTPError) as err:
|
||||
if isinstance(err, HTTPError):
|
||||
error_body = err.read()
|
||||
try:
|
||||
error_data = json.loads(error_body)
|
||||
except ValueError:
|
||||
print (
|
||||
"There was an error, and the response was not valid JSON.\n" +
|
||||
"Raw JSONRPC response:\n" + error_body
|
||||
)
|
||||
return 1
|
||||
|
||||
print_error(error_data['error']['message'] + "\n", suggest_help=False)
|
||||
|
||||
if 'data' in error_data['error'] and 'traceback' in error_data['error']['data']:
|
||||
print "Here's the traceback for the error you encountered:"
|
||||
print "\n".join(error_data['error']['data']['traceback'])
|
||||
|
||||
print_help_for_command(method)
|
||||
elif isinstance(err, RPCError):
|
||||
print_error(err.msg, suggest_help=False)
|
||||
# print_help_for_command(method)
|
||||
else:
|
||||
print_error("Something went wrong\n", suggest_help=False)
|
||||
print str(err)
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
def guess_type(x, key=None):
|
||||
if not isinstance(x, (unicode, str)):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x in ('true', 'True', 'TRUE'):
|
||||
return True
|
||||
if x in ('false', 'False', 'FALSE'):
|
||||
return False
|
||||
if '.' in x:
|
||||
try:
|
||||
return float(x)
|
||||
except ValueError:
|
||||
# not a float
|
||||
pass
|
||||
try:
|
||||
return int(x)
|
||||
except ValueError:
|
||||
return x
|
||||
|
||||
|
||||
def print_help_suggestion():
|
||||
print "See `{} help` for more information.".format(os.path.basename(sys.argv[0]))
|
||||
|
||||
|
||||
def print_error(message, suggest_help=True):
|
||||
error_style = colorama.Style.BRIGHT + colorama.Fore.RED
|
||||
print error_style + "ERROR: " + message + colorama.Style.RESET_ALL
|
||||
if suggest_help:
|
||||
print_help_suggestion()
|
||||
|
||||
|
||||
def print_help():
|
||||
print "\n".join([
|
||||
"NAME",
|
||||
" lbrynet-cli - LBRY command line client.",
|
||||
"",
|
||||
"USAGE",
|
||||
" lbrynet-cli [--conf <config file>] <command> [<args>]",
|
||||
"",
|
||||
"EXAMPLES",
|
||||
" lbrynet-cli commands # list available commands",
|
||||
" lbrynet-cli status # get daemon status",
|
||||
" lbrynet-cli --conf ~/l1.conf status # like above but using ~/l1.conf as config file",
|
||||
" lbrynet-cli resolve_name what # resolve a name",
|
||||
" lbrynet-cli help resolve_name # get help for a command",
|
||||
])
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print "Help for %s method:\n%s" % (command, fn.__doc__)
|
||||
|
||||
|
||||
def wrap_list_to_term_width(l, width=None, separator=', ', prefix=''):
|
||||
if width is None:
|
||||
try:
|
||||
_, width = os.popen('stty size', 'r').read().split()
|
||||
width = int(width)
|
||||
except:
|
||||
pass
|
||||
if not width:
|
||||
width = 80
|
||||
|
||||
lines = []
|
||||
curr_line = ''
|
||||
for item in l:
|
||||
new_line = curr_line + item + separator
|
||||
if len(new_line) + len(prefix) > width:
|
||||
lines.append(curr_line)
|
||||
curr_line = item + separator
|
||||
else:
|
||||
curr_line = new_line
|
||||
lines.append(curr_line)
|
||||
|
||||
ret = prefix + ("\n" + prefix).join(lines)
|
||||
if ret.endswith(separator):
|
||||
ret = ret[:-len(separator)]
|
||||
return ret
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,124 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
|
||||
# This needs to happen before anything else
|
||||
if 'win' in sys.platform:
|
||||
import certifi
|
||||
os.environ['SSL_CERT_FILE'] = certifi.where()
|
||||
|
||||
from lbrynet.core import log_support
|
||||
|
||||
import argparse
|
||||
import logging.handlers
|
||||
|
||||
from twisted.internet import defer, reactor
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet import analytics
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils, system_info
|
||||
from lbrynet.daemon.DaemonServer import DaemonServer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def test_internet_connection():
|
||||
return utils.check_connection()
|
||||
|
||||
|
||||
def start():
|
||||
"""The primary entry point for launching the daemon."""
|
||||
|
||||
# postpone loading the config file to after the CLI arguments
|
||||
# have been parsed, as they may contain an alternate config file location
|
||||
conf.initialize_settings(load_conf_file=False)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument(
|
||||
"--conf",
|
||||
help="specify an alternative configuration file",
|
||||
type=str,
|
||||
default=None
|
||||
)
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
parser.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
parser.add_argument(
|
||||
'--version', action="store_true",
|
||||
help='Show daemon version and quit'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
update_settings_from_args(args)
|
||||
|
||||
conf.settings.load_conf_file_settings()
|
||||
|
||||
if args.version:
|
||||
version = system_info.get_platform(get_ip=False)
|
||||
version['installation_id'] = conf.settings.installation_id
|
||||
print utils.json_dumps_pretty(version)
|
||||
return
|
||||
|
||||
lbrynet_log = conf.settings.get_log_filename()
|
||||
log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
|
||||
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
|
||||
|
||||
try:
|
||||
log.debug('Checking for an existing lbrynet daemon instance')
|
||||
JSONRPCProxy.from_url(conf.settings.get_api_connection_string()).status()
|
||||
log.info("lbrynet-daemon is already running")
|
||||
return
|
||||
except Exception:
|
||||
log.debug('No lbrynet instance found, continuing to start')
|
||||
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if test_internet_connection():
|
||||
analytics_manager = analytics.Manager.new_instance()
|
||||
start_server_and_listen(analytics_manager)
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
|
||||
def update_settings_from_args(args):
|
||||
if args.conf:
|
||||
conf.conf_file = args.conf
|
||||
|
||||
if args.useauth:
|
||||
conf.settings.update({
|
||||
'use_auth_http': args.useauth,
|
||||
}, data_types=(conf.TYPE_CLI,))
|
||||
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_server_and_listen(analytics_manager):
|
||||
"""
|
||||
Args:
|
||||
use_auth: set to true to enable http authentication
|
||||
analytics_manager: to send analytics
|
||||
"""
|
||||
analytics_manager.send_server_startup()
|
||||
daemon_server = DaemonServer(analytics_manager)
|
||||
try:
|
||||
yield daemon_server.start(conf.settings['use_auth_http'])
|
||||
analytics_manager.send_server_startup_success()
|
||||
except Exception as e:
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
analytics_manager.send_server_startup_error(str(e))
|
||||
daemon_server.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start()
|
||||
@@ -1,77 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.internet import defer, reactor, error
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
from lbrynet.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
|
||||
from lbrynet.daemon.auth.util import initialize_api_key_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IndexResource(resource.Resource):
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
|
||||
class DaemonServer(object):
|
||||
def __init__(self, analytics_manager=None):
|
||||
self._daemon = None
|
||||
self.root = None
|
||||
self.server_port = None
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
def _setup_server(self, use_auth):
|
||||
self.root = IndexResource()
|
||||
self._daemon = Daemon(self.analytics_manager)
|
||||
self.root.putChild("", self._daemon)
|
||||
# TODO: DEPRECATED, remove this and just serve the API at the root
|
||||
self.root.putChild(conf.settings['API_ADDRESS'], self._daemon)
|
||||
|
||||
lbrynet_server = get_site_base(use_auth, self.root)
|
||||
|
||||
try:
|
||||
self.server_port = reactor.listenTCP(
|
||||
conf.settings['api_port'], lbrynet_server, interface=conf.settings['api_host'])
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
except error.CannotListenError:
|
||||
log.info('Daemon already running, exiting app')
|
||||
raise
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, use_auth):
|
||||
yield self._setup_server(use_auth)
|
||||
yield self._daemon.setup()
|
||||
|
||||
def stop(self):
|
||||
if reactor.running:
|
||||
log.info("Stopping the reactor")
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
|
||||
def get_site_base(use_auth, root):
|
||||
if use_auth:
|
||||
log.info("Using authenticated API")
|
||||
root = create_auth_session(root)
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
return server.Site(root)
|
||||
|
||||
|
||||
def create_auth_session(root):
|
||||
pw_path = os.path.join(conf.settings['data_dir'], ".api_keys")
|
||||
initialize_api_key_file(pw_path)
|
||||
checker = PasswordChecker.load_file(pw_path)
|
||||
realm = HttpPasswordRealm(root)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
factory = guard.BasicCredentialFactory('Login to lbrynet api')
|
||||
_lbrynet_server = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
|
||||
return _lbrynet_server
|
||||
@@ -1,78 +0,0 @@
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.core import file_utils
|
||||
from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Publisher(object):
|
||||
def __init__(self, session, lbry_file_manager, wallet, certificate_id):
|
||||
self.session = session
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.wallet = wallet
|
||||
self.certificate_id = certificate_id
|
||||
self.lbry_file = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def create_and_publish_stream(self, name, bid, claim_dict, file_path, claim_address=None,
|
||||
change_address=None):
|
||||
"""Create lbry file and make claim"""
|
||||
log.info('Starting publish for %s', name)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception("File {} not found".format(file_path))
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception("Cannot publish empty file {}".format(file_path))
|
||||
|
||||
file_name = os.path.basename(file_path)
|
||||
with file_utils.get_read_handle(file_path) as read_handle:
|
||||
self.lbry_file = yield create_lbry_file(self.session, self.lbry_file_manager, file_name,
|
||||
read_handle)
|
||||
|
||||
if 'source' not in claim_dict['stream']:
|
||||
claim_dict['stream']['source'] = {}
|
||||
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = get_content_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
claim_out = yield self.make_claim(name, bid, claim_dict, claim_address, change_address)
|
||||
|
||||
# check if we have a file already for this claim (if this is a publish update with a new stream)
|
||||
old_stream_hashes = yield self.session.storage.get_old_stream_hashes_for_claim_id(claim_out['claim_id'],
|
||||
self.lbry_file.stream_hash)
|
||||
if old_stream_hashes:
|
||||
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
|
||||
list(self.lbry_file_manager.lbry_files)):
|
||||
yield self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False)
|
||||
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
|
||||
|
||||
yield self.session.storage.save_content_claim(
|
||||
self.lbry_file.stream_hash, "%s:%i" % (claim_out['txid'], claim_out['nout'])
|
||||
)
|
||||
defer.returnValue(claim_out)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def publish_stream(self, name, bid, claim_dict, stream_hash, claim_address=None, change_address=None):
|
||||
"""Make a claim without creating a lbry file"""
|
||||
claim_out = yield self.make_claim(name, bid, claim_dict, claim_address, change_address)
|
||||
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
|
||||
yield self.session.storage.save_content_claim(stream_hash, "%s:%i" % (claim_out['txid'],
|
||||
claim_out['nout']))
|
||||
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
|
||||
defer.returnValue(claim_out)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def make_claim(self, name, bid, claim_dict, claim_address=None, change_address=None):
|
||||
claim_out = yield self.wallet.claim_name(name, bid, claim_dict,
|
||||
certificate_id=self.certificate_id,
|
||||
claim_address=claim_address,
|
||||
change_address=change_address)
|
||||
defer.returnValue(claim_out)
|
||||
|
||||
|
||||
def get_content_type(filename):
|
||||
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
|
||||
@@ -1,3 +0,0 @@
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
|
||||
get_client = LBRYAPIClient.get_client
|
||||
@@ -1,139 +0,0 @@
|
||||
import os
|
||||
import json
|
||||
import urlparse
|
||||
import requests
|
||||
from requests.cookies import RequestsCookieJar
|
||||
import logging
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
from lbrynet import conf
|
||||
from lbrynet.daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
|
||||
def copy_cookies(cookies):
|
||||
result = RequestsCookieJar()
|
||||
result.update(cookies)
|
||||
return result
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
Exception.__init__(self)
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
class AuthAPIClient(object):
|
||||
def __init__(self, key, timeout, connection, count, cookies, url, login_url):
|
||||
self.__api_key = key
|
||||
self.__service_url = login_url
|
||||
self.__id_count = count
|
||||
self.__url = url
|
||||
self.__conn = connection
|
||||
self.__cookies = copy_cookies(cookies)
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
raise AttributeError(name)
|
||||
|
||||
def f(*args, **kwargs):
|
||||
return self.call(name, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
def call(self, method, params=None):
|
||||
params = params or {}
|
||||
self.__id_count += 1
|
||||
pre_auth_post_data = {
|
||||
'version': '2',
|
||||
'method': method,
|
||||
'params': params,
|
||||
'id': self.__id_count
|
||||
}
|
||||
to_auth = get_auth_message(pre_auth_post_data)
|
||||
pre_auth_post_data.update({'hmac': self.__api_key.get_hmac(to_auth)})
|
||||
post_data = json.dumps(pre_auth_post_data)
|
||||
cookies = copy_cookies(self.__cookies)
|
||||
req = requests.Request(
|
||||
method='POST', url=self.__service_url, data=post_data, cookies=cookies,
|
||||
headers={
|
||||
'Host': self.__url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
)
|
||||
http_response = self.__conn.send(req.prepare())
|
||||
if http_response is None:
|
||||
raise JSONRPCException({
|
||||
'code': -342, 'message': 'missing HTTP response from server'})
|
||||
http_response.raise_for_status()
|
||||
next_secret = http_response.headers.get(LBRY_SECRET, False)
|
||||
if next_secret:
|
||||
self.__api_key.secret = next_secret
|
||||
self.__cookies = copy_cookies(http_response.cookies)
|
||||
response = http_response.json()
|
||||
if response.get('error') is not None:
|
||||
raise JSONRPCException(response['error'])
|
||||
elif 'result' not in response:
|
||||
raise JSONRPCException({
|
||||
'code': -343, 'message': 'missing JSON-RPC result'})
|
||||
else:
|
||||
return response['result']
|
||||
|
||||
@classmethod
|
||||
def config(cls, key_name=None, key=None, pw_path=None, timeout=HTTP_TIMEOUT, connection=None, count=0,
|
||||
cookies=None, auth=None, url=None, login_url=None):
|
||||
|
||||
api_key_name = key_name or API_KEY_NAME
|
||||
pw_path = os.path.join(conf.settings['data_dir'], ".api_keys") if not pw_path else pw_path
|
||||
if not key:
|
||||
keys = load_api_keys(pw_path)
|
||||
api_key = keys.get(api_key_name, False)
|
||||
else:
|
||||
api_key = APIKey(name=api_key_name, secret=key)
|
||||
if login_url is None:
|
||||
service_url = "http://%s:%s@%s:%i/%s" % (api_key_name,
|
||||
api_key.secret,
|
||||
conf.settings['api_host'],
|
||||
conf.settings['api_port'],
|
||||
conf.settings['API_ADDRESS'])
|
||||
else:
|
||||
service_url = login_url
|
||||
id_count = count
|
||||
|
||||
if auth is None and connection is None and cookies is None and url is None:
|
||||
# This is a new client instance, start an authenticated session
|
||||
url = urlparse.urlparse(service_url)
|
||||
conn = requests.Session()
|
||||
req = requests.Request(method='POST',
|
||||
url=service_url,
|
||||
headers={'Host': url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'},)
|
||||
r = req.prepare()
|
||||
http_response = conn.send(r)
|
||||
cookies = RequestsCookieJar()
|
||||
cookies.update(http_response.cookies)
|
||||
uid = cookies.get(TWISTED_SESSION)
|
||||
api_key = APIKey.new(seed=uid)
|
||||
else:
|
||||
# This is a client that already has a session, use it
|
||||
conn = connection
|
||||
if not cookies.get(LBRY_SECRET):
|
||||
raise Exception("Missing cookie")
|
||||
secret = cookies.get(LBRY_SECRET)
|
||||
api_key = APIKey(secret, api_key_name)
|
||||
return cls(api_key, timeout, conn, id_count, cookies, url, service_url)
|
||||
|
||||
|
||||
class LBRYAPIClient(object):
|
||||
@staticmethod
|
||||
def get_client():
|
||||
if not conf.settings:
|
||||
conf.initialize_settings()
|
||||
return AuthAPIClient.config() if conf.settings['use_auth_http'] else \
|
||||
JSONRPCProxy.from_url(conf.settings.get_api_connection_string())
|
||||
@@ -1,91 +0,0 @@
|
||||
import base58
|
||||
import hmac
|
||||
import hashlib
|
||||
import yaml
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
API_KEY_NAME = "api"
|
||||
|
||||
|
||||
def sha(x):
|
||||
h = hashlib.sha256(x).digest()
|
||||
return base58.b58encode(h)
|
||||
|
||||
|
||||
def generate_key(x=None):
|
||||
if x is None:
|
||||
return sha(os.urandom(256))
|
||||
else:
|
||||
return sha(x)
|
||||
|
||||
|
||||
class APIKey(object):
|
||||
def __init__(self, secret, name, expiration=None):
|
||||
self.secret = secret
|
||||
self.name = name
|
||||
self.expiration = expiration
|
||||
|
||||
@classmethod
|
||||
def new(cls, seed=None, name=None, expiration=None):
|
||||
secret = generate_key(seed)
|
||||
key_name = name if name else sha(secret)
|
||||
return APIKey(secret, key_name, expiration)
|
||||
|
||||
def _raw_key(self):
|
||||
return base58.b58decode(self.secret)
|
||||
|
||||
def get_hmac(self, message):
|
||||
decoded_key = self._raw_key()
|
||||
signature = hmac.new(decoded_key, message, hashlib.sha256)
|
||||
return base58.b58encode(signature.digest())
|
||||
|
||||
def compare_hmac(self, message, token):
|
||||
decoded_token = base58.b58decode(token)
|
||||
target = base58.b58decode(self.get_hmac(message))
|
||||
|
||||
try:
|
||||
if len(decoded_token) != len(target):
|
||||
return False
|
||||
return hmac.compare_digest(decoded_token, target)
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def load_api_keys(path):
|
||||
if not os.path.isfile(path):
|
||||
raise Exception("Invalid api key path")
|
||||
|
||||
with open(path, "r") as f:
|
||||
data = yaml.load(f.read())
|
||||
|
||||
keys_for_return = {}
|
||||
for key_name in data:
|
||||
key = data[key_name]
|
||||
secret = key['secret']
|
||||
expiration = key['expiration']
|
||||
keys_for_return.update({key_name: APIKey(secret, key_name, expiration)})
|
||||
return keys_for_return
|
||||
|
||||
|
||||
def save_api_keys(keys, path):
|
||||
with open(path, "w") as f:
|
||||
key_dict = {keys[key_name].name: {'secret': keys[key_name].secret,
|
||||
'expiration': keys[key_name].expiration}
|
||||
for key_name in keys}
|
||||
data = yaml.safe_dump(key_dict)
|
||||
f.write(data)
|
||||
|
||||
|
||||
def initialize_api_key_file(key_path):
|
||||
keys = {}
|
||||
new_api_key = APIKey.new(name=API_KEY_NAME)
|
||||
keys.update({new_api_key.name: new_api_key})
|
||||
save_api_keys(keys, key_path)
|
||||
|
||||
|
||||
def get_auth_message(message_dict):
|
||||
return json.dumps(message_dict, sort_keys=True)
|
||||
@@ -8,7 +8,7 @@ DELAY_INCREMENT = 0.0001
|
||||
QUEUE_SIZE_THRESHOLD = 100
|
||||
|
||||
|
||||
class CallLaterManager(object):
|
||||
class CallLaterManager:
|
||||
def __init__(self, callLater):
|
||||
"""
|
||||
:param callLater: (IReactorTime.callLater)
|
||||
@@ -29,6 +29,8 @@ rpcTimeout = 5
|
||||
|
||||
# number of rpc attempts to make before a timeout results in the node being removed as a contact
|
||||
rpcAttempts = 5
|
||||
# time window to count failures (in seconds)
|
||||
rpcAttemptsPruningTimeWindow = 600
|
||||
|
||||
# Delay between iterations of iterative node lookups (for loose parallelism) (in seconds)
|
||||
iterativeLookupDelay = rpcTimeout / 2
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import reduce
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
try:
|
||||
ip = ipaddress.ip_address(address.decode()) # this needs to be unicode, thus the decode()
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class _Contact(object):
|
||||
class _Contact:
|
||||
""" Encapsulation for remote contact
|
||||
|
||||
This class contains information on a single remote contact, and also
|
||||
@@ -19,8 +21,8 @@ class _Contact(object):
|
||||
|
||||
def __init__(self, contactManager, id, ipAddress, udpPort, networkProtocol, firstComm):
|
||||
if id is not None:
|
||||
if not len(id) == constants.key_bits / 8:
|
||||
raise ValueError("invalid node id: %s" % id.encode('hex'))
|
||||
if not len(id) == constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: {}".format(hexlify(id).decode()))
|
||||
if not 0 <= udpPort <= 65536:
|
||||
raise ValueError("invalid port")
|
||||
if not is_valid_ipv4(ipAddress):
|
||||
@@ -38,7 +40,7 @@ class _Contact(object):
|
||||
self._token = (None, 0) # token, timestamp
|
||||
|
||||
def update_token(self, token):
|
||||
self._token = token, self.getTime()
|
||||
self._token = token, self.getTime() if token else 0
|
||||
|
||||
@property
|
||||
def token(self):
|
||||
@@ -56,7 +58,7 @@ class _Contact(object):
|
||||
def log_id(self, short=True):
|
||||
if not self.id:
|
||||
return "not initialized"
|
||||
id_hex = self.id.encode('hex')
|
||||
id_hex = hexlify(self.id)
|
||||
return id_hex if not short else id_hex[:8]
|
||||
|
||||
@property
|
||||
@@ -65,7 +67,7 @@ class _Contact(object):
|
||||
|
||||
@property
|
||||
def lastFailed(self):
|
||||
return self._contactManager._rpc_failures.get((self.address, self.port), [None])[-1]
|
||||
return (self.failures or [None])[-1]
|
||||
|
||||
@property
|
||||
def failures(self):
|
||||
@@ -95,25 +97,17 @@ class _Contact(object):
|
||||
return None
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, _Contact):
|
||||
return self.id == other.id
|
||||
elif isinstance(other, str):
|
||||
return self.id == other
|
||||
else:
|
||||
return False
|
||||
if not isinstance(other, _Contact):
|
||||
raise TypeError("invalid type to compare with Contact: %s" % str(type(other)))
|
||||
return (self.id, self.address, self.port) == (other.id, other.address, other.port)
|
||||
|
||||
def __ne__(self, other):
|
||||
if isinstance(other, _Contact):
|
||||
return self.id != other.id
|
||||
elif isinstance(other, str):
|
||||
return self.id != other
|
||||
else:
|
||||
return True
|
||||
def __hash__(self):
|
||||
return hash((self.id, self.address, self.port))
|
||||
|
||||
def compact_ip(self):
|
||||
compact_ip = reduce(
|
||||
lambda buff, x: buff + bytearray([int(x)]), self.address.split('.'), bytearray())
|
||||
return str(compact_ip)
|
||||
return compact_ip
|
||||
|
||||
def set_id(self, id):
|
||||
if not self._id:
|
||||
@@ -156,12 +150,12 @@ class _Contact(object):
|
||||
raise AttributeError("unknown command: %s" % name)
|
||||
|
||||
def _sendRPC(*args, **kwargs):
|
||||
return self._networkProtocol.sendRPC(self, name, args)
|
||||
return self._networkProtocol.sendRPC(self, name.encode(), args)
|
||||
|
||||
return _sendRPC
|
||||
|
||||
|
||||
class ContactManager(object):
|
||||
class ContactManager:
|
||||
def __init__(self, get_time=None):
|
||||
if not get_time:
|
||||
from twisted.internet import reactor
|
||||
@@ -171,12 +165,11 @@ class ContactManager(object):
|
||||
self._rpc_failures = {}
|
||||
|
||||
def get_contact(self, id, address, port):
|
||||
for contact in self._contacts.itervalues():
|
||||
for contact in self._contacts.values():
|
||||
if contact.id == id and contact.address == address and contact.port == port:
|
||||
return contact
|
||||
|
||||
def make_contact(self, id, ipAddress, udpPort, networkProtocol, firstComm=0):
|
||||
ipAddress = str(ipAddress)
|
||||
contact = self.get_contact(id, ipAddress, udpPort)
|
||||
if contact:
|
||||
return contact
|
||||
@@ -185,5 +178,12 @@ class ContactManager(object):
|
||||
return contact
|
||||
|
||||
def is_ignored(self, origin_tuple):
|
||||
failed_rpc_count = len(self._rpc_failures.get(origin_tuple, []))
|
||||
failed_rpc_count = len(self._prune_failures(origin_tuple))
|
||||
return failed_rpc_count > constants.rpcAttempts
|
||||
|
||||
def _prune_failures(self, origin_tuple):
|
||||
# Prunes recorded failures to the last time window of attempts
|
||||
pruning_limit = self._get_time() - constants.rpcAttemptsPruningTimeWindow
|
||||
pruned = list(filter(lambda t: t >= pruning_limit, self._rpc_failures.get(origin_tuple, [])))
|
||||
self._rpc_failures[origin_tuple] = pruned
|
||||
return pruned
|
||||
@@ -1,27 +1,21 @@
|
||||
import UserDict
|
||||
import constants
|
||||
from interface import IDataStore
|
||||
from zope.interface import implements
|
||||
from collections import UserDict
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class DictDataStore(UserDict.DictMixin):
|
||||
class DictDataStore(UserDict):
|
||||
""" A datastore using an in-memory Python dictionary """
|
||||
implements(IDataStore)
|
||||
#implements(IDataStore)
|
||||
|
||||
def __init__(self, getTime=None):
|
||||
# Dictionary format:
|
||||
# { <key>: (<contact>, <value>, <lastPublished>, <originallyPublished> <originalPublisherID>) }
|
||||
self._dict = {}
|
||||
super().__init__()
|
||||
if not getTime:
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self.completed_blobs = set()
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
return self._dict.keys()
|
||||
|
||||
def filter_bad_and_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
@@ -29,41 +23,44 @@ class DictDataStore(UserDict.DictMixin):
|
||||
return filter(
|
||||
lambda peer:
|
||||
self._getTime() - peer[3] < constants.dataExpireTimeout and peer[0].contact_is_good is not False,
|
||||
self._dict[key]
|
||||
self[key]
|
||||
)
|
||||
|
||||
def filter_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
return filter(lambda peer: self._getTime() - peer[3] < constants.dataExpireTimeout, self._dict[key])
|
||||
return filter(lambda peer: self._getTime() - peer[3] < constants.dataExpireTimeout, self[key])
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
for key in self._dict.keys():
|
||||
unexpired_peers = self.filter_expired_peers(key)
|
||||
expired_keys = []
|
||||
for key in self.keys():
|
||||
unexpired_peers = list(self.filter_expired_peers(key))
|
||||
if not unexpired_peers:
|
||||
del self._dict[key]
|
||||
expired_keys.append(key)
|
||||
else:
|
||||
self._dict[key] = unexpired_peers
|
||||
self[key] = unexpired_peers
|
||||
for key in expired_keys:
|
||||
del self[key]
|
||||
|
||||
def hasPeersForBlob(self, key):
|
||||
return True if key in self._dict and len(self.filter_bad_and_expired_peers(key)) else False
|
||||
return bool(key in self and len(tuple(self.filter_bad_and_expired_peers(key))))
|
||||
|
||||
def addPeerToBlob(self, contact, key, compact_address, lastPublished, originallyPublished, originalPublisherID):
|
||||
if key in self._dict:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self._dict[key]):
|
||||
self._dict[key].append(
|
||||
if key in self:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self[key]):
|
||||
self[key].append(
|
||||
(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)
|
||||
)
|
||||
else:
|
||||
self._dict[key] = [(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)]
|
||||
self[key] = [(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)]
|
||||
|
||||
def getPeersForBlob(self, key):
|
||||
return [] if key not in self._dict else [val[1] for val in self.filter_bad_and_expired_peers(key)]
|
||||
return [] if key not in self else [val[1] for val in self.filter_bad_and_expired_peers(key)]
|
||||
|
||||
def getStoringContacts(self):
|
||||
contacts = set()
|
||||
for key in self._dict:
|
||||
for values in self._dict[key]:
|
||||
for key in self:
|
||||
for values in self[key]:
|
||||
contacts.add(values[0])
|
||||
return list(contacts)
|
||||
@@ -1,21 +1,21 @@
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class Distance(object):
|
||||
class Distance:
|
||||
"""Calculate the XOR result between two string variables.
|
||||
|
||||
Frequently we re-use one of the points so as an optimization
|
||||
we pre-calculate the long value of that point.
|
||||
we pre-calculate the value of that point.
|
||||
"""
|
||||
|
||||
def __init__(self, key):
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.key = key
|
||||
self.val_key_one = long(key.encode('hex'), 16)
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two):
|
||||
val_key_two = long(key_two.encode('hex'), 16)
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
def is_closer(self, a, b):
|
||||
|
||||
@@ -1,134 +1,75 @@
|
||||
from error import DecodeError
|
||||
from lbrynet.dht.error import DecodeError
|
||||
|
||||
|
||||
class Encoding(object):
|
||||
""" Interface for RPC message encoders/decoders
|
||||
|
||||
All encoding implementations used with this library should inherit and
|
||||
implement this.
|
||||
"""
|
||||
|
||||
def encode(self, data):
|
||||
""" Encode the specified data
|
||||
|
||||
@param data: The data to encode
|
||||
This method has to support encoding of the following
|
||||
types: C{str}, C{int} and C{long}
|
||||
Any additional data types may be supported as long as the
|
||||
implementing class's C{decode()} method can successfully
|
||||
decode them.
|
||||
|
||||
@return: The encoded data
|
||||
@rtype: str
|
||||
"""
|
||||
|
||||
def decode(self, data):
|
||||
""" Decode the specified data string
|
||||
|
||||
@param data: The data (byte string) to decode.
|
||||
@type data: str
|
||||
|
||||
@return: The decoded data (in its correct type)
|
||||
"""
|
||||
def bencode(data):
|
||||
""" Encoder implementation of the Bencode algorithm (Bittorrent). """
|
||||
if isinstance(data, int):
|
||||
return b'i%de' % data
|
||||
elif isinstance(data, (bytes, bytearray)):
|
||||
return b'%d:%s' % (len(data), data)
|
||||
elif isinstance(data, str):
|
||||
return b'%d:%s' % (len(data), data.encode())
|
||||
elif isinstance(data, (list, tuple)):
|
||||
encoded_list_items = b''
|
||||
for item in data:
|
||||
encoded_list_items += bencode(item)
|
||||
return b'l%se' % encoded_list_items
|
||||
elif isinstance(data, dict):
|
||||
encoded_dict_items = b''
|
||||
keys = data.keys()
|
||||
for key in sorted(keys):
|
||||
encoded_dict_items += bencode(key)
|
||||
encoded_dict_items += bencode(data[key])
|
||||
return b'd%se' % encoded_dict_items
|
||||
else:
|
||||
raise TypeError("Cannot bencode '%s' object" % type(data))
|
||||
|
||||
|
||||
class Bencode(Encoding):
|
||||
""" Implementation of a Bencode-based algorithm (Bencode is the encoding
|
||||
algorithm used by Bittorrent).
|
||||
def bdecode(data):
|
||||
""" Decoder implementation of the Bencode algorithm. """
|
||||
assert type(data) == bytes # fixme: _maybe_ remove this after porting
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
try:
|
||||
return _decode_recursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError(str(e))
|
||||
|
||||
@note: This algorithm differs from the "official" Bencode algorithm in
|
||||
that it can encode/decode floating point values in addition to
|
||||
integers.
|
||||
"""
|
||||
|
||||
def encode(self, data):
|
||||
""" Encoder implementation of the Bencode algorithm
|
||||
|
||||
@param data: The data to encode
|
||||
@type data: int, long, tuple, list, dict or str
|
||||
|
||||
@return: The encoded data
|
||||
@rtype: str
|
||||
"""
|
||||
if isinstance(data, (int, long)):
|
||||
return 'i%de' % data
|
||||
elif isinstance(data, str):
|
||||
return '%d:%s' % (len(data), data)
|
||||
elif isinstance(data, (list, tuple)):
|
||||
encodedListItems = ''
|
||||
for item in data:
|
||||
encodedListItems += self.encode(item)
|
||||
return 'l%se' % encodedListItems
|
||||
elif isinstance(data, dict):
|
||||
encodedDictItems = ''
|
||||
keys = data.keys()
|
||||
keys.sort()
|
||||
for key in keys:
|
||||
encodedDictItems += self.encode(key) # TODO: keys should always be bytestrings
|
||||
encodedDictItems += self.encode(data[key])
|
||||
return 'd%se' % encodedDictItems
|
||||
else:
|
||||
print data
|
||||
raise TypeError("Cannot bencode '%s' object" % type(data))
|
||||
|
||||
def decode(self, data):
|
||||
""" Decoder implementation of the Bencode algorithm
|
||||
|
||||
@param data: The encoded data
|
||||
@type data: str
|
||||
|
||||
@note: This is a convenience wrapper for the recursive decoding
|
||||
algorithm, C{_decodeRecursive}
|
||||
|
||||
@return: The decoded data, as a native Python type
|
||||
@rtype: int, list, dict or str
|
||||
"""
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
def _decode_recursive(data, start_index=0):
|
||||
if data[start_index] == ord('i'):
|
||||
end_pos = data[start_index:].find(b'e') + start_index
|
||||
return int(data[start_index + 1:end_pos]), end_pos + 1
|
||||
elif data[start_index] == ord('l'):
|
||||
start_index += 1
|
||||
decoded_list = []
|
||||
while data[start_index] != ord('e'):
|
||||
list_data, start_index = _decode_recursive(data, start_index)
|
||||
decoded_list.append(list_data)
|
||||
return decoded_list, start_index + 1
|
||||
elif data[start_index] == ord('d'):
|
||||
start_index += 1
|
||||
decoded_dict = {}
|
||||
while data[start_index] != ord('e'):
|
||||
key, start_index = _decode_recursive(data, start_index)
|
||||
value, start_index = _decode_recursive(data, start_index)
|
||||
decoded_dict[key] = value
|
||||
return decoded_dict, start_index
|
||||
elif data[start_index] == ord('f'):
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
end_pos = data[start_index:].find(b'e') + start_index
|
||||
return float(data[start_index + 1:end_pos]), end_pos + 1
|
||||
elif data[start_index] == ord('n'):
|
||||
# This (None/NULL data type) is a non-standard extension
|
||||
# to the original Bencode algorithm
|
||||
return None, start_index + 1
|
||||
else:
|
||||
split_pos = data[start_index:].find(b':') + start_index
|
||||
try:
|
||||
return self._decodeRecursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError(e.message)
|
||||
|
||||
@staticmethod
|
||||
def _decodeRecursive(data, startIndex=0):
|
||||
""" Actual implementation of the recursive Bencode algorithm
|
||||
|
||||
Do not call this; use C{decode()} instead
|
||||
"""
|
||||
if data[startIndex] == 'i':
|
||||
endPos = data[startIndex:].find('e') + startIndex
|
||||
return int(data[startIndex + 1:endPos]), endPos + 1
|
||||
elif data[startIndex] == 'l':
|
||||
startIndex += 1
|
||||
decodedList = []
|
||||
while data[startIndex] != 'e':
|
||||
listData, startIndex = Bencode._decodeRecursive(data, startIndex)
|
||||
decodedList.append(listData)
|
||||
return decodedList, startIndex + 1
|
||||
elif data[startIndex] == 'd':
|
||||
startIndex += 1
|
||||
decodedDict = {}
|
||||
while data[startIndex] != 'e':
|
||||
key, startIndex = Bencode._decodeRecursive(data, startIndex)
|
||||
value, startIndex = Bencode._decodeRecursive(data, startIndex)
|
||||
decodedDict[key] = value
|
||||
return decodedDict, startIndex
|
||||
elif data[startIndex] == 'f':
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
endPos = data[startIndex:].find('e') + startIndex
|
||||
return float(data[startIndex + 1:endPos]), endPos + 1
|
||||
elif data[startIndex] == 'n':
|
||||
# This (None/NULL data type) is a non-standard extension
|
||||
# to the original Bencode algorithm
|
||||
return None, startIndex + 1
|
||||
else:
|
||||
splitPos = data[startIndex:].find(':') + startIndex
|
||||
try:
|
||||
length = int(data[startIndex:splitPos])
|
||||
except ValueError, e:
|
||||
raise DecodeError, e
|
||||
startIndex = splitPos + 1
|
||||
endPos = startIndex + length
|
||||
bytes = data[startIndex:endPos]
|
||||
return bytes, endPos
|
||||
length = int(data[start_index:split_pos])
|
||||
except ValueError:
|
||||
raise DecodeError()
|
||||
start_index = split_pos + 1
|
||||
end_pos = start_index + length
|
||||
b = data[start_index:end_pos]
|
||||
return b, end_pos
|
||||
@@ -1,10 +1,10 @@
|
||||
import binascii
|
||||
import exceptions
|
||||
#import exceptions
|
||||
|
||||
# this is a dict of {"exceptions.<exception class name>": exception class} items used to raise
|
||||
# remote built-in exceptions locally
|
||||
BUILTIN_EXCEPTIONS = {
|
||||
"exceptions.%s" % e: getattr(exceptions, e) for e in dir(exceptions) if not e.startswith("_")
|
||||
# "exceptions.%s" % e: getattr(exceptions, e) for e in dir(exceptions) if not e.startswith("_")
|
||||
}
|
||||
|
||||
|
||||
@@ -13,14 +13,12 @@ class DecodeError(Exception):
|
||||
Should be raised by an C{Encoding} implementation if decode operation
|
||||
fails
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class BucketFull(Exception):
|
||||
"""
|
||||
Raised when the bucket is full
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class UnknownRemoteException(Exception):
|
||||
@@ -37,7 +35,7 @@ class TimeoutError(Exception):
|
||||
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
|
||||
else:
|
||||
msg = 'Timeout connecting to uninitialized node'
|
||||
Exception.__init__(self, msg)
|
||||
super().__init__(msg)
|
||||
self.remote_contact_id = remote_contact_id
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ class IDataStore(Interface):
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
pass
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
pass
|
||||
@@ -59,7 +58,7 @@ class IRoutingTable(Interface):
|
||||
@param count: the amount of contacts to return
|
||||
@type count: int
|
||||
@param _rpcNodeID: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the paramater will get
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type _rpcNodeID: str
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from distance import Distance
|
||||
from error import TimeoutError
|
||||
import constants
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -14,10 +14,18 @@ def get_contact(contact_list, node_id, address, port):
|
||||
raise IndexError(node_id)
|
||||
|
||||
|
||||
class _IterativeFind(object):
|
||||
def expand_peer(compact_peer_info):
|
||||
host = "{}.{}.{}.{}".format(*compact_peer_info[:4])
|
||||
port = int.from_bytes(compact_peer_info[4:6], 'big')
|
||||
peer_node_id = compact_peer_info[6:]
|
||||
return (peer_node_id, host, port)
|
||||
|
||||
|
||||
class _IterativeFind:
|
||||
# TODO: use polymorphism to search for a value or node
|
||||
# instead of using a find_value flag
|
||||
def __init__(self, node, shortlist, key, rpc):
|
||||
def __init__(self, node, shortlist, key, rpc, exclude=None):
|
||||
self.exclude = set(exclude or [])
|
||||
self.node = node
|
||||
self.finished_deferred = defer.Deferred()
|
||||
# all distance operations in this class only care about the distance
|
||||
@@ -29,7 +37,7 @@ class _IterativeFind(object):
|
||||
# Shortlist of contact objects (the k closest known contacts to the key from the routing table)
|
||||
self.shortlist = shortlist
|
||||
# The search key
|
||||
self.key = str(key)
|
||||
self.key = key
|
||||
# The rpc method name (findValue or findNode)
|
||||
self.rpc = rpc
|
||||
# List of active queries; len() indicates number of active probes
|
||||
@@ -59,28 +67,28 @@ class _IterativeFind(object):
|
||||
|
||||
def getContactTriples(self, result):
|
||||
if self.is_find_value_request:
|
||||
contact_triples = result['contacts']
|
||||
contact_triples = result[b'contacts']
|
||||
else:
|
||||
contact_triples = result
|
||||
for contact_tup in contact_triples:
|
||||
if not isinstance(contact_tup, (list, tuple)) or len(contact_tup) != 3:
|
||||
raise ValueError("invalid contact triple")
|
||||
contact_tup[1] = contact_tup[1].decode() # ips are strings
|
||||
return contact_triples
|
||||
|
||||
def sortByDistance(self, contact_list):
|
||||
"""Sort the list of contacts in order by distance from key"""
|
||||
contact_list.sort(key=lambda c: self.distance(c.id))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def extendShortlist(self, contact, result):
|
||||
# The "raw response" tuple contains the response message and the originating address info
|
||||
originAddress = (contact.address, contact.port)
|
||||
if self.finished_deferred.called:
|
||||
defer.returnValue(contact.id)
|
||||
return contact.id
|
||||
if self.node.contact_manager.is_ignored(originAddress):
|
||||
raise ValueError("contact is ignored")
|
||||
if contact.id == self.node.node_id:
|
||||
defer.returnValue(contact.id)
|
||||
return contact.id
|
||||
|
||||
if contact not in self.active_contacts:
|
||||
self.active_contacts.append(contact)
|
||||
@@ -93,24 +101,28 @@ class _IterativeFind(object):
|
||||
# we are looking for before treating it as a list of contact triples
|
||||
if self.is_find_value_request and self.key in result:
|
||||
# We have found the value
|
||||
self.find_value_result[self.key] = result[self.key]
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
for peer in result[self.key]:
|
||||
node_id, host, port = expand_peer(peer)
|
||||
if (host, port) not in self.exclude:
|
||||
self.find_value_result.setdefault(self.key, []).append((node_id, host, port))
|
||||
if self.find_value_result:
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
else:
|
||||
if self.is_find_value_request:
|
||||
# We are looking for a value, and the remote node didn't have it
|
||||
# - mark it as the closest "empty" node, if it is
|
||||
# TODO: store to this peer after finding the value as per the kademlia spec
|
||||
if 'closestNodeNoValue' in self.find_value_result:
|
||||
if b'closestNodeNoValue' in self.find_value_result:
|
||||
if self.is_closer(contact):
|
||||
self.find_value_result['closestNodeNoValue'] = contact
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
else:
|
||||
self.find_value_result['closestNodeNoValue'] = contact
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
contactTriples = self.getContactTriples(result)
|
||||
for contactTriple in contactTriples:
|
||||
if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
|
||||
continue
|
||||
elif self.node.contact_manager.is_ignored((contactTriple[1], contactTriple[2])):
|
||||
raise ValueError("contact is ignored")
|
||||
continue
|
||||
else:
|
||||
found_contact = self.node.contact_manager.make_contact(contactTriple[0], contactTriple[1],
|
||||
contactTriple[2], self.node._protocol)
|
||||
@@ -121,14 +133,14 @@ class _IterativeFind(object):
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
|
||||
defer.returnValue(contact.id)
|
||||
return contact.id
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def probeContact(self, contact):
|
||||
fn = getattr(contact, self.rpc)
|
||||
try:
|
||||
response = yield fn(self.key)
|
||||
result = yield self.extendShortlist(contact, response)
|
||||
result = self.extendShortlist(contact, response)
|
||||
defer.returnValue(result)
|
||||
except (TimeoutError, defer.CancelledError, ValueError, IndexError):
|
||||
defer.returnValue(contact.id)
|
||||
@@ -160,6 +172,9 @@ class _IterativeFind(object):
|
||||
already_contacted_addresses = {(c.address, c.port) for c in self.already_contacted}
|
||||
to_remove = []
|
||||
for contact in self.shortlist:
|
||||
if self.node.contact_manager.is_ignored((contact.address, contact.port)):
|
||||
to_remove.append(contact) # a contact became bad during iteration
|
||||
continue
|
||||
if (contact.address, contact.port) not in already_contacted_addresses:
|
||||
self.already_contacted.append(contact)
|
||||
to_remove.append(contact)
|
||||
@@ -188,8 +203,11 @@ class _IterativeFind(object):
|
||||
|
||||
elif not self.finished_deferred.called and not self.active_probes or self.should_stop():
|
||||
# If no probes were sent, there will not be any improvement, so we're done
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
if self.is_find_value_request:
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
else:
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
elif not self.finished_deferred.called:
|
||||
# Force the next iteration
|
||||
self.searchIteration()
|
||||
@@ -206,7 +224,7 @@ class _IterativeFind(object):
|
||||
self.pending_iteration_calls.append(cancel)
|
||||
|
||||
|
||||
def iterativeFind(node, shortlist, key, rpc):
|
||||
helper = _IterativeFind(node, shortlist, key, rpc)
|
||||
def iterativeFind(node, shortlist, key, rpc, exclude=None):
|
||||
helper = _IterativeFind(node, shortlist, key, rpc, exclude)
|
||||
helper.searchIteration(0)
|
||||
return helper.finished_deferred
|
||||
@@ -1,12 +1,13 @@
|
||||
import logging
|
||||
import constants
|
||||
from distance import Distance
|
||||
from error import BucketFull
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import BucketFull
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket(object):
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
@@ -135,8 +136,8 @@ class KBucket(object):
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
if isinstance(key, str):
|
||||
key = long(key.encode('hex'), 16)
|
||||
if isinstance(key, bytes):
|
||||
key = int.from_bytes(key, 'big')
|
||||
return self.rangeMin <= key < self.rangeMax
|
||||
|
||||
def __len__(self):
|
||||
|
||||
@@ -7,10 +7,10 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import msgtypes
|
||||
from lbrynet.dht import msgtypes
|
||||
|
||||
|
||||
class MessageTranslator(object):
|
||||
class MessageTranslator:
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
Classes inheriting from this should provide a translation services between
|
||||
|
||||
@@ -7,17 +7,17 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from lbrynet.core.utils import generate_id
|
||||
import constants
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class Message(object):
|
||||
class Message:
|
||||
""" Base class for messages - all "unknown" messages use this class """
|
||||
|
||||
def __init__(self, rpcID, nodeID):
|
||||
if len(rpcID) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id: %i bytes (expected 20)" % len(rpcID))
|
||||
if len(nodeID) != constants.key_bits / 8:
|
||||
if len(nodeID) != constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: %i bytes (expected 48)" % len(nodeID))
|
||||
self.id = rpcID
|
||||
self.nodeID = nodeID
|
||||
@@ -29,7 +29,7 @@ class RequestMessage(Message):
|
||||
def __init__(self, nodeID, method, methodArgs, rpcID=None):
|
||||
if rpcID is None:
|
||||
rpcID = generate_id()[:constants.rpc_id_length]
|
||||
Message.__init__(self, rpcID, nodeID)
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.request = method
|
||||
self.args = methodArgs
|
||||
|
||||
@@ -38,7 +38,7 @@ class ResponseMessage(Message):
|
||||
""" Message containing the result from a successful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, response):
|
||||
Message.__init__(self, rpcID, nodeID)
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.response = response
|
||||
|
||||
|
||||
@@ -46,8 +46,7 @@ class ErrorMessage(ResponseMessage):
|
||||
""" Message containing the error from an unsuccessful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
|
||||
ResponseMessage.__init__(self, rpcID, nodeID, errorMessage)
|
||||
super().__init__(rpcID, nodeID, errorMessage)
|
||||
if isinstance(exceptionType, type):
|
||||
self.exceptionType = '%s.%s' % (exceptionType.__module__, exceptionType.__name__)
|
||||
else:
|
||||
self.exceptionType = exceptionType
|
||||
exceptionType = (f'{exceptionType.__module__}.{exceptionType.__name__}').encode()
|
||||
self.exceptionType = exceptionType
|
||||
@@ -1,40 +1,20 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
import binascii
|
||||
import hashlib
|
||||
import struct
|
||||
import logging
|
||||
from functools import reduce
|
||||
|
||||
from twisted.internet import defer, error, task
|
||||
|
||||
from lbrynet.core.utils import generate_id, DeferredDict
|
||||
from lbrynet.core.call_later_manager import CallLaterManager
|
||||
from lbrynet.core.PeerManager import PeerManager
|
||||
from error import TimeoutError
|
||||
import constants
|
||||
import routingtable
|
||||
import datastore
|
||||
import protocol
|
||||
from peerfinder import DHTPeerFinder
|
||||
from contact import ContactManager
|
||||
from iterativefind import iterativeFind
|
||||
|
||||
from lbrynet.utils import generate_id, DeferredDict
|
||||
from lbrynet.dht.call_later_manager import CallLaterManager
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants, routingtable, datastore, protocol
|
||||
from lbrynet.dht.contact import ContactManager
|
||||
from lbrynet.dht.iterativefind import iterativeFind
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def expand_peer(compact_peer_info):
|
||||
host = ".".join([str(ord(d)) for d in compact_peer_info[:4]])
|
||||
port, = struct.unpack('>H', compact_peer_info[4:6])
|
||||
peer_node_id = compact_peer_info[6:]
|
||||
return (peer_node_id, host, port)
|
||||
|
||||
|
||||
def rpcmethod(func):
|
||||
""" Decorator to expose Node methods as remote procedure calls
|
||||
|
||||
@@ -45,7 +25,7 @@ def rpcmethod(func):
|
||||
return func
|
||||
|
||||
|
||||
class MockKademliaHelper(object):
|
||||
class MockKademliaHelper:
|
||||
def __init__(self, clock=None, callLater=None, resolve=None, listenUDP=None):
|
||||
if not listenUDP or not resolve or not callLater or not clock:
|
||||
from twisted.internet import reactor
|
||||
@@ -98,8 +78,8 @@ class Node(MockKademliaHelper):
|
||||
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
|
||||
routingTableClass=None, networkProtocol=None,
|
||||
externalIP=None, peerPort=3333, listenUDP=None,
|
||||
callLater=None, resolve=None, clock=None, peer_finder=None,
|
||||
peer_manager=None):
|
||||
callLater=None, resolve=None, clock=None,
|
||||
interface='', externalUDPPort=None):
|
||||
"""
|
||||
@param dataStore: The data store to use. This must be class inheriting
|
||||
from the C{DataStore} interface (or providing the
|
||||
@@ -125,9 +105,10 @@ class Node(MockKademliaHelper):
|
||||
@param peerPort: the port at which this node announces it has a blob for
|
||||
"""
|
||||
|
||||
MockKademliaHelper.__init__(self, clock, callLater, resolve, listenUDP)
|
||||
super().__init__(clock, callLater, resolve, listenUDP)
|
||||
self.node_id = node_id or self._generateID()
|
||||
self.port = udpPort
|
||||
self._listen_interface = interface
|
||||
self._change_token_lc = self.get_looping_call(self.change_token)
|
||||
self._refresh_node_lc = self.get_looping_call(self._refreshNode)
|
||||
self._refresh_contacts_lc = self.get_looping_call(self._refreshContacts)
|
||||
@@ -138,25 +119,23 @@ class Node(MockKademliaHelper):
|
||||
else:
|
||||
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
|
||||
|
||||
# Initialize this node's network access mechanisms
|
||||
if networkProtocol is None:
|
||||
self._protocol = protocol.KademliaProtocol(self)
|
||||
else:
|
||||
self._protocol = networkProtocol
|
||||
# Initialize the data storage mechanism used by this node
|
||||
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
|
||||
self.token_secret = self._generateID()
|
||||
self.old_token_secret = None
|
||||
self.externalIP = externalIP
|
||||
self.peerPort = peerPort
|
||||
self.externalUDPPort = externalUDPPort or self.port
|
||||
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
|
||||
self.peer_manager = peer_manager or PeerManager()
|
||||
self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager)
|
||||
self._join_deferred = None
|
||||
|
||||
def __del__(self):
|
||||
log.warning("unclean shutdown of the dht node")
|
||||
if hasattr(self, "_listeningPort") and self._listeningPort is not None:
|
||||
self._listeningPort.stopListening()
|
||||
#def __del__(self):
|
||||
# log.warning("unclean shutdown of the dht node")
|
||||
# if hasattr(self, "_listeningPort") and self._listeningPort is not None:
|
||||
# self._listeningPort.stopListening()
|
||||
|
||||
def __str__(self):
|
||||
return '<%s.%s object; ID: %s, IP address: %s, UDP port: %d>' % (
|
||||
self.__module__, self.__class__.__name__, binascii.hexlify(self.node_id), self.externalIP, self.port)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
@@ -171,7 +150,8 @@ class Node(MockKademliaHelper):
|
||||
def start_listening(self):
|
||||
if not self._listeningPort:
|
||||
try:
|
||||
self._listeningPort = self.reactor_listenUDP(self.port, self._protocol)
|
||||
self._listeningPort = self.reactor_listenUDP(self.port, self._protocol,
|
||||
interface=self._listen_interface)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
|
||||
@@ -200,7 +180,7 @@ class Node(MockKademliaHelper):
|
||||
if not known_node_resolution:
|
||||
known_node_resolution = yield _resolve_seeds()
|
||||
# we are one of the seed nodes, don't add ourselves
|
||||
if (self.externalIP, self.port) in known_node_resolution.itervalues():
|
||||
if (self.externalIP, self.port) in known_node_resolution.values():
|
||||
del known_node_resolution[(self.externalIP, self.port)]
|
||||
known_node_addresses.remove((self.externalIP, self.port))
|
||||
|
||||
@@ -213,7 +193,7 @@ class Node(MockKademliaHelper):
|
||||
def _initialize_routing():
|
||||
bootstrap_contacts = []
|
||||
contact_addresses = {(c.address, c.port): c for c in self.contacts}
|
||||
for (host, port), ip_address in known_node_resolution.iteritems():
|
||||
for (host, port), ip_address in known_node_resolution.items():
|
||||
if (host, port) not in contact_addresses:
|
||||
# Create temporary contact information for the list of addresses of known nodes
|
||||
# The contact node id will be set with the responding node id when we initialize it to None
|
||||
@@ -228,7 +208,7 @@ class Node(MockKademliaHelper):
|
||||
if not bootstrap_contacts:
|
||||
log.warning("no bootstrap contacts to ping")
|
||||
ping_result = yield _ping_contacts(bootstrap_contacts)
|
||||
shortlist = ping_result.keys()
|
||||
shortlist = list(ping_result.keys())
|
||||
if not shortlist:
|
||||
log.warning("failed to ping %i bootstrap contacts", len(bootstrap_contacts))
|
||||
defer.returnValue(None)
|
||||
@@ -236,10 +216,10 @@ class Node(MockKademliaHelper):
|
||||
# find the closest peers to us
|
||||
closest = yield self._iterativeFind(self.node_id, shortlist if not self.contacts else None)
|
||||
yield _ping_contacts(closest)
|
||||
# # query random hashes in our bucket key ranges to fill or split them
|
||||
# random_ids_in_range = self._routingTable.getRefreshList()
|
||||
# while random_ids_in_range:
|
||||
# yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
# query random hashes in our bucket key ranges to fill or split them
|
||||
random_ids_in_range = self._routingTable.getRefreshList()
|
||||
while random_ids_in_range:
|
||||
yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -263,7 +243,7 @@ class Node(MockKademliaHelper):
|
||||
yield _iterative_join()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, known_node_addresses=None):
|
||||
def start(self, known_node_addresses=None, block_on_join=False):
|
||||
""" Causes the Node to attempt to join the DHT network by contacting the
|
||||
known DHT nodes. This can be called multiple times if the previous attempt
|
||||
has failed or if the Node has lost all the contacts.
|
||||
@@ -278,8 +258,13 @@ class Node(MockKademliaHelper):
|
||||
self.start_listening()
|
||||
yield self._protocol._listening
|
||||
# TODO: Refresh all k-buckets further away than this node's closest neighbour
|
||||
yield self.joinNetwork(known_node_addresses or [])
|
||||
d = self.joinNetwork(known_node_addresses or [])
|
||||
d.addCallback(lambda _: self.start_looping_calls())
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
if block_on_join:
|
||||
yield d
|
||||
|
||||
def start_looping_calls(self):
|
||||
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)
|
||||
# Start refreshing k-buckets periodically, if necessary
|
||||
self.safe_start_looping_call(self._refresh_node_lc, constants.checkRefreshInterval)
|
||||
@@ -305,25 +290,24 @@ class Node(MockKademliaHelper):
|
||||
@defer.inlineCallbacks
|
||||
def storeToContact(self, blob_hash, contact):
|
||||
try:
|
||||
token = contact.token
|
||||
if not token:
|
||||
find_value_response = yield contact.findValue(blob_hash)
|
||||
token = find_value_response['token']
|
||||
contact.update_token(token)
|
||||
res = yield contact.store(blob_hash, token, self.peerPort, self.node_id, 0)
|
||||
if res != "OK":
|
||||
if not contact.token:
|
||||
yield contact.findValue(blob_hash)
|
||||
res = yield contact.store(blob_hash, contact.token, self.peerPort, self.node_id, 0)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
defer.returnValue(True)
|
||||
log.debug("Stored %s to %s (%s)", binascii.hexlify(blob_hash), contact.log_id(), contact.address)
|
||||
return True
|
||||
except protocol.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s",
|
||||
binascii.hexlify(blob_hash), contact.log_id())
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err.message)
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
contact.update_token(None)
|
||||
log.error("Unexpected error while storing blob_hash %s at %s: %s",
|
||||
binascii.hexlify(blob_hash), contact, err)
|
||||
defer.returnValue(False)
|
||||
return False
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def announceHaveBlob(self, blob_hash):
|
||||
@@ -332,9 +316,7 @@ class Node(MockKademliaHelper):
|
||||
if not self.externalIP:
|
||||
raise Exception("Cannot determine external IP: %s" % self.externalIP)
|
||||
stored_to = yield DeferredDict({contact: self.storeToContact(blob_hash, contact) for contact in contacts})
|
||||
contacted_node_ids = map(
|
||||
lambda contact: contact.id.encode('hex'), filter(lambda contact: stored_to[contact], stored_to.keys())
|
||||
)
|
||||
contacted_node_ids = [binascii.hexlify(contact.id) for contact in stored_to.keys() if stored_to[contact]]
|
||||
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(blob_hash),
|
||||
len(contacted_node_ids), len(contacts))
|
||||
defer.returnValue(contacted_node_ids)
|
||||
@@ -375,7 +357,7 @@ class Node(MockKademliaHelper):
|
||||
return self._iterativeFind(key)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def iterativeFindValue(self, key):
|
||||
def iterativeFindValue(self, key, exclude=None):
|
||||
""" The Kademlia search operation (deterministic)
|
||||
|
||||
Call this to retrieve data from the DHT.
|
||||
@@ -396,11 +378,11 @@ class Node(MockKademliaHelper):
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length!")
|
||||
|
||||
# Execute the search
|
||||
find_result = yield self._iterativeFind(key, rpc='findValue')
|
||||
find_result = yield self._iterativeFind(key, rpc='findValue', exclude=exclude)
|
||||
if isinstance(find_result, dict):
|
||||
# We have found the value; now see who was the closest contact without it...
|
||||
# ...and store the key/value pair
|
||||
@@ -418,22 +400,15 @@ class Node(MockKademliaHelper):
|
||||
else:
|
||||
pass
|
||||
|
||||
expanded_peers = []
|
||||
if find_result:
|
||||
if key in find_result:
|
||||
for peer in find_result[key]:
|
||||
expanded = expand_peer(peer)
|
||||
if expanded not in expanded_peers:
|
||||
expanded_peers.append(expanded)
|
||||
# TODO: get this working
|
||||
# if 'closestNodeNoValue' in find_result:
|
||||
# closest_node_without_value = find_result['closestNodeNoValue']
|
||||
# try:
|
||||
# response, address = yield closest_node_without_value.findValue(key, rawResponse=True)
|
||||
# yield closest_node_without_value.store(key, response.response['token'], self.peerPort)
|
||||
# except TimeoutError:
|
||||
# pass
|
||||
defer.returnValue(expanded_peers)
|
||||
defer.returnValue(list(set(find_result.get(key, []) if find_result else [])))
|
||||
# TODO: get this working
|
||||
# if 'closestNodeNoValue' in find_result:
|
||||
# closest_node_without_value = find_result['closestNodeNoValue']
|
||||
# try:
|
||||
# response, address = yield closest_node_without_value.findValue(key, rawResponse=True)
|
||||
# yield closest_node_without_value.store(key, response.response['token'], self.peerPort)
|
||||
# except TimeoutError:
|
||||
# pass
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add/update the given contact; simple wrapper for the same method
|
||||
@@ -466,20 +441,10 @@ class Node(MockKademliaHelper):
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
try:
|
||||
contact = self._routingTable.getContact(contactID)
|
||||
df = defer.Deferred()
|
||||
df.callback(contact)
|
||||
df = defer.succeed(self._routingTable.getContact(contactID))
|
||||
except (ValueError, IndexError):
|
||||
def parseResults(nodes):
|
||||
node_ids = [c.id for c in nodes]
|
||||
if contactID in nodes:
|
||||
contact = nodes[node_ids.index(contactID)]
|
||||
return contact
|
||||
else:
|
||||
return None
|
||||
|
||||
df = self.iterativeFindNode(contactID)
|
||||
df.addCallback(parseResults)
|
||||
df.addCallback(lambda nodes: ([node for node in nodes if node.id == contactID] or (None,))[0])
|
||||
return df
|
||||
|
||||
@rpcmethod
|
||||
@@ -488,7 +453,7 @@ class Node(MockKademliaHelper):
|
||||
|
||||
@rtype: str
|
||||
"""
|
||||
return 'pong'
|
||||
return b'pong'
|
||||
|
||||
@rpcmethod
|
||||
def store(self, rpc_contact, blob_hash, token, port, originalPublisherID, age):
|
||||
@@ -523,15 +488,15 @@ class Node(MockKademliaHelper):
|
||||
elif not self.verify_token(token, compact_ip):
|
||||
raise ValueError("Invalid token")
|
||||
if 0 <= port <= 65536:
|
||||
compact_port = str(struct.pack('>H', port))
|
||||
compact_port = port.to_bytes(2, 'big')
|
||||
else:
|
||||
raise TypeError('Invalid port')
|
||||
raise TypeError(f'Invalid port: {port}')
|
||||
compact_address = compact_ip + compact_port + rpc_contact.id
|
||||
now = int(self.clock.seconds())
|
||||
originallyPublished = now - age
|
||||
self._dataStore.addPeerToBlob(rpc_contact, blob_hash, compact_address, now, originallyPublished,
|
||||
originalPublisherID)
|
||||
return 'OK'
|
||||
return b'OK'
|
||||
|
||||
@rpcmethod
|
||||
def findNode(self, rpc_contact, key):
|
||||
@@ -547,7 +512,7 @@ class Node(MockKademliaHelper):
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid contact id length: %i" % len(key))
|
||||
|
||||
contacts = self._routingTable.findCloseNodes(key, sender_node_id=rpc_contact.id)
|
||||
@@ -569,15 +534,15 @@ class Node(MockKademliaHelper):
|
||||
@rtype: dict or list
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid blob hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
'token': self.make_token(rpc_contact.compact_ip()),
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
}
|
||||
|
||||
if self._protocol._protocolVersion:
|
||||
response['protocolVersion'] = self._protocol._protocolVersion
|
||||
response[b'protocolVersion'] = self._protocol._protocolVersion
|
||||
|
||||
# get peers we have stored for this blob
|
||||
has_other_peers = self._dataStore.hasPeersForBlob(key)
|
||||
@@ -587,17 +552,15 @@ class Node(MockKademliaHelper):
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and key in self._dataStore.completed_blobs:
|
||||
compact_ip = str(
|
||||
reduce(lambda buff, x: buff + bytearray([int(x)]), self.externalIP.split('.'), bytearray())
|
||||
)
|
||||
compact_port = str(struct.pack('>H', self.peerPort))
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), self.externalIP.split('.'), bytearray())
|
||||
compact_port = self.peerPort.to_bytes(2, 'big')
|
||||
compact_address = compact_ip + compact_port + self.node_id
|
||||
peers.append(compact_address)
|
||||
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response['contacts'] = self.findNode(rpc_contact, key)
|
||||
response[b'contacts'] = self.findNode(rpc_contact, key)
|
||||
return response
|
||||
|
||||
def _generateID(self):
|
||||
@@ -608,10 +571,10 @@ class Node(MockKademliaHelper):
|
||||
"""
|
||||
return generate_id()
|
||||
|
||||
# from lbrynet.core.utils import profile_deferred
|
||||
# from lbrynet.p2p.utils import profile_deferred
|
||||
# @profile_deferred()
|
||||
@defer.inlineCallbacks
|
||||
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode'):
|
||||
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode', exclude=None):
|
||||
""" The basic Kademlia iterative lookup operation (for nodes/values)
|
||||
|
||||
This builds a list of k "closest" contacts through iterative use of
|
||||
@@ -640,7 +603,7 @@ class Node(MockKademliaHelper):
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
|
||||
if startupShortlist is None:
|
||||
@@ -649,7 +612,7 @@ class Node(MockKademliaHelper):
|
||||
# # Update the "last accessed" timestamp for the appropriate k-bucket
|
||||
# self._routingTable.touchKBucket(key)
|
||||
if len(shortlist) == 0:
|
||||
log.warning("This node doesnt know any other nodes")
|
||||
log.warning("This node doesn't know any other nodes")
|
||||
# This node doesn't know of any other nodes
|
||||
fakeDf = defer.Deferred()
|
||||
fakeDf.callback([])
|
||||
@@ -659,7 +622,7 @@ class Node(MockKademliaHelper):
|
||||
# This is used during the bootstrap process
|
||||
shortlist = startupShortlist
|
||||
|
||||
result = yield iterativeFind(self, shortlist, key, rpc)
|
||||
result = yield iterativeFind(self, shortlist, key, rpc, exclude=exclude)
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -668,19 +631,14 @@ class Node(MockKademliaHelper):
|
||||
replication/republishing as necessary """
|
||||
yield self._refreshRoutingTable()
|
||||
self._dataStore.removeExpiredPeers()
|
||||
yield self._refreshStoringPeers()
|
||||
self._refreshStoringPeers()
|
||||
defer.returnValue(None)
|
||||
|
||||
def _refreshContacts(self):
|
||||
return defer.DeferredList(
|
||||
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in self.contacts]
|
||||
)
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self.contacts, delay=0)
|
||||
|
||||
def _refreshStoringPeers(self):
|
||||
storing_contacts = self._dataStore.getStoringContacts()
|
||||
return defer.DeferredList(
|
||||
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in storing_contacts]
|
||||
)
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self._dataStore.getStoringContacts(), delay=0)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _refreshRoutingTable(self):
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from lbrynet.interfaces import IPeerFinder
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyPeerFinder(object):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
return defer.succeed([])
|
||||
|
||||
|
||||
class DHTPeerFinder(DummyPeerFinder):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
implements(IPeerFinder)
|
||||
|
||||
def __init__(self, dht_node, peer_manager):
|
||||
"""
|
||||
dht_node - an instance of dht.Node class
|
||||
peer_manager - an instance of PeerManager class
|
||||
"""
|
||||
self.dht_node = dht_node
|
||||
self.peer_manager = peer_manager
|
||||
self.peers = []
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
"""
|
||||
Find peers for blob in the DHT
|
||||
blob_hash (str): blob hash to look for
|
||||
timeout (int): seconds to timeout after
|
||||
filter_self (bool): if True, and if a peer for a blob is itself, filter it
|
||||
from the result
|
||||
|
||||
Returns:
|
||||
list of peers for the blob
|
||||
"""
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = self.dht_node.iterativeFindValue(bin_hash)
|
||||
timeout = timeout or conf.settings['peer_search_timeout']
|
||||
if timeout:
|
||||
finished_deferred.addTimeout(timeout, self.dht_node.clock)
|
||||
try:
|
||||
peer_list = yield finished_deferred
|
||||
except defer.TimeoutError:
|
||||
log.warning("DHT timed out while looking peers for blob"
|
||||
" %s after %s seconds.", blob_hash, timeout)
|
||||
peer_list = []
|
||||
|
||||
peers = set(peer_list)
|
||||
results = []
|
||||
for node_id, host, port in peers:
|
||||
if filter_self and (host, port) == (self.dht_node.externalIP, self.dht_node.peerPort):
|
||||
continue
|
||||
results.append(self.peer_manager.get_peer(host, port))
|
||||
defer.returnValue(results)
|
||||
@@ -1,20 +1,15 @@
|
||||
import logging
|
||||
import socket
|
||||
import errno
|
||||
from collections import deque
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
from error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
import constants
|
||||
import encoding
|
||||
import msgtypes
|
||||
import msgformat
|
||||
from lbrynet.dht import constants, encoding, msgformat, msgtypes
|
||||
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PingQueue(object):
|
||||
class PingQueue:
|
||||
"""
|
||||
Schedules a 15 minute delayed ping after a new node sends us a query. This is so the new node gets added to the
|
||||
routing table after having been given enough time for a pinhole to expire.
|
||||
@@ -22,71 +17,53 @@ class PingQueue(object):
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._get_time = self._node.clock.seconds
|
||||
self._queue = deque()
|
||||
self._enqueued_contacts = {}
|
||||
self._semaphore = defer.DeferredSemaphore(1)
|
||||
self._ping_semaphore = defer.DeferredSemaphore(constants.alpha)
|
||||
self._process_lc = node.get_looping_call(self._semaphore.run, self._process)
|
||||
self._pending_contacts = {}
|
||||
self._process_lc = node.get_looping_call(self._process)
|
||||
|
||||
def _add_contact(self, contact, delay=None):
|
||||
if contact in self._enqueued_contacts:
|
||||
return defer.succeed(None)
|
||||
delay = delay or constants.checkRefreshInterval
|
||||
self._enqueued_contacts[contact] = self._get_time() + delay
|
||||
self._queue.append(contact)
|
||||
return defer.succeed(None)
|
||||
def enqueue_maybe_ping(self, *contacts, **kwargs):
|
||||
delay = kwargs.get('delay', constants.checkRefreshInterval)
|
||||
no_op = (defer.succeed(None), lambda: None)
|
||||
for contact in contacts:
|
||||
if delay and contact not in self._enqueued_contacts:
|
||||
self._pending_contacts.setdefault(contact, self._node.clock.seconds() + delay)
|
||||
else:
|
||||
self._enqueued_contacts.setdefault(contact, no_op)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _process(self):
|
||||
if not len(self._queue):
|
||||
defer.returnValue(None)
|
||||
contact = self._queue.popleft()
|
||||
now = self._get_time()
|
||||
|
||||
# if the oldest contact in the queue isn't old enough to be pinged, add it back to the queue and return
|
||||
if now < self._enqueued_contacts[contact]:
|
||||
self._queue.appendleft(contact)
|
||||
defer.returnValue(None)
|
||||
|
||||
pinged = []
|
||||
checked = []
|
||||
while now > self._enqueued_contacts[contact]:
|
||||
checked.append(contact)
|
||||
if not contact.contact_is_good:
|
||||
pinged.append(contact)
|
||||
if not len(self._queue):
|
||||
break
|
||||
contact = self._queue.popleft()
|
||||
if not now > self._enqueued_contacts[contact]:
|
||||
checked.append(contact)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _ping(contact):
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
|
||||
yield defer.DeferredList([_ping(contact) for contact in pinged])
|
||||
|
||||
for contact in checked:
|
||||
def _ping(self, contact):
|
||||
if contact.contact_is_good:
|
||||
return
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
finally:
|
||||
if contact in self._enqueued_contacts:
|
||||
del self._enqueued_contacts[contact]
|
||||
|
||||
defer.returnValue(None)
|
||||
def _process(self):
|
||||
# move contacts that are scheduled to join the queue
|
||||
if self._pending_contacts:
|
||||
now = self._node.clock.seconds()
|
||||
for contact in [contact for contact, schedule in self._pending_contacts.items() if schedule <= now]:
|
||||
del self._pending_contacts[contact]
|
||||
self._enqueued_contacts.setdefault(contact, (defer.succeed(None), lambda: None))
|
||||
# spread pings across 60 seconds to avoid flood and/or false negatives
|
||||
step = 60.0/float(len(self._enqueued_contacts)) if self._enqueued_contacts else 0
|
||||
for index, (contact, (call, _)) in enumerate(self._enqueued_contacts.items()):
|
||||
if call.called and not contact.contact_is_good:
|
||||
self._enqueued_contacts[contact] = self._node.reactor_callLater(index*step, self._ping, contact)
|
||||
|
||||
def start(self):
|
||||
return self._node.safe_start_looping_call(self._process_lc, 60)
|
||||
|
||||
def stop(self):
|
||||
map(None, (cancel() for _, (call, cancel) in self._enqueued_contacts.items() if not call.called))
|
||||
return self._node.safe_stop_looping_call(self._process_lc)
|
||||
|
||||
def enqueue_maybe_ping(self, contact, delay=None):
|
||||
return self._semaphore.run(self._add_contact, contact, delay)
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
@@ -95,7 +72,6 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._encoder = encoding.Bencode()
|
||||
self._translator = msgformat.DefaultFormat()
|
||||
self._sentMessages = {}
|
||||
self._partialMessages = {}
|
||||
@@ -106,12 +82,12 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
self.started_listening_time = 0
|
||||
|
||||
def _migrate_incoming_rpc_args(self, contact, method, *args):
|
||||
if method == 'store' and contact.protocolVersion == 0:
|
||||
if method == b'store' and contact.protocolVersion == 0:
|
||||
if isinstance(args[1], dict):
|
||||
blob_hash = args[0]
|
||||
token = args[1].pop('token', None)
|
||||
port = args[1].pop('port', -1)
|
||||
originalPublisherID = args[1].pop('lbryid', None)
|
||||
token = args[1].pop(b'token', None)
|
||||
port = args[1].pop(b'port', -1)
|
||||
originalPublisherID = args[1].pop(b'lbryid', None)
|
||||
age = 0
|
||||
return (blob_hash, token, port, originalPublisherID, age), {}
|
||||
return args, {}
|
||||
@@ -122,18 +98,36 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
protocol version keyword argument to calls to contacts who will accept it
|
||||
"""
|
||||
if contact.protocolVersion == 0:
|
||||
if method == 'store':
|
||||
if method == b'store':
|
||||
blob_hash, token, port, originalPublisherID, age = args
|
||||
args = (blob_hash, {'token': token, 'port': port, 'lbryid': originalPublisherID}, originalPublisherID,
|
||||
False)
|
||||
args = (
|
||||
blob_hash, {
|
||||
b'token': token,
|
||||
b'port': port,
|
||||
b'lbryid': originalPublisherID
|
||||
}, originalPublisherID, False
|
||||
)
|
||||
return args
|
||||
return args
|
||||
if args and isinstance(args[-1], dict):
|
||||
args[-1]['protocolVersion'] = self._protocolVersion
|
||||
args[-1][b'protocolVersion'] = self._protocolVersion
|
||||
return args
|
||||
return args + ({'protocolVersion': self._protocolVersion},)
|
||||
return args + ({b'protocolVersion': self._protocolVersion},)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def sendRPC(self, contact, method, args):
|
||||
for _ in range(constants.rpcAttempts):
|
||||
try:
|
||||
response = yield self._sendRPC(contact, method, args)
|
||||
return response
|
||||
except TimeoutError:
|
||||
if contact.contact_is_good:
|
||||
log.debug("RETRY %s ON %s", method, contact)
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
def _sendRPC(self, contact, method, args):
|
||||
"""
|
||||
Sends an RPC to the specified contact
|
||||
|
||||
@@ -156,11 +150,11 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
msg = msgtypes.RequestMessage(self._node.node_id, method, self._migrate_outgoing_rpc_args(contact, method,
|
||||
*args))
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = self._encoder.encode(msgPrimitive)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
|
||||
if args:
|
||||
log.debug("%s:%i SEND CALL %s(%s) TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
args[0].encode('hex'), contact.address, contact.port)
|
||||
hexlify(args[0]), contact.address, contact.port)
|
||||
else:
|
||||
log.debug("%s:%i SEND CALL %s TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
contact.address, contact.port)
|
||||
@@ -168,20 +162,23 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df = defer.Deferred()
|
||||
|
||||
def _remove_contact(failure): # remove the contact from the routing table and track the failure
|
||||
contact.update_last_failed()
|
||||
try:
|
||||
self._node.removeContact(contact)
|
||||
if not contact.contact_is_good:
|
||||
self._node.removeContact(contact)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
contact.update_last_failed()
|
||||
return failure
|
||||
|
||||
def _update_contact(result): # refresh the contact in the routing table
|
||||
contact.update_last_replied()
|
||||
if method == 'findValue':
|
||||
if 'protocolVersion' not in result:
|
||||
if method == b'findValue':
|
||||
if b'token' in result:
|
||||
contact.update_token(result[b'token'])
|
||||
if b'protocolVersion' not in result:
|
||||
contact.update_protocol_version(0)
|
||||
else:
|
||||
contact.update_protocol_version(result.pop('protocolVersion'))
|
||||
contact.update_protocol_version(result.pop(b'protocolVersion'))
|
||||
d = self._node.addContact(contact)
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
@@ -199,7 +196,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
return df
|
||||
|
||||
def startProtocol(self):
|
||||
log.info("DHT listening on UDP %s:%i", self._node.externalIP, self._node.port)
|
||||
log.info("DHT listening on UDP %i (ext port %i)", self._node.port, self._node.externalUDPPort)
|
||||
if self._listening.called:
|
||||
self._listening = defer.Deferred()
|
||||
self._listening.callback(True)
|
||||
@@ -212,18 +209,17 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
@note: This is automatically called by Twisted when the protocol
|
||||
receives a UDP datagram
|
||||
"""
|
||||
|
||||
if datagram[0] == '\x00' and datagram[25] == '\x00':
|
||||
totalPackets = (ord(datagram[1]) << 8) | ord(datagram[2])
|
||||
if chr(datagram[0]) == '\x00' and chr(datagram[25]) == '\x00':
|
||||
totalPackets = (datagram[1] << 8) | datagram[2]
|
||||
msgID = datagram[5:25]
|
||||
seqNumber = (ord(datagram[3]) << 8) | ord(datagram[4])
|
||||
seqNumber = (datagram[3] << 8) | datagram[4]
|
||||
if msgID not in self._partialMessages:
|
||||
self._partialMessages[msgID] = {}
|
||||
self._partialMessages[msgID][seqNumber] = datagram[26:]
|
||||
if len(self._partialMessages[msgID]) == totalPackets:
|
||||
keys = self._partialMessages[msgID].keys()
|
||||
keys.sort()
|
||||
data = ''
|
||||
data = b''
|
||||
for key in keys:
|
||||
data += self._partialMessages[msgID][key]
|
||||
datagram = data
|
||||
@@ -231,11 +227,11 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
else:
|
||||
return
|
||||
try:
|
||||
msgPrimitive = self._encoder.decode(datagram)
|
||||
msgPrimitive = encoding.bdecode(datagram)
|
||||
message = self._translator.fromPrimitive(msgPrimitive)
|
||||
except (encoding.DecodeError, ValueError) as err:
|
||||
# We received some rubbish here
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", datagram.encode('hex'),
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", hexlify(datagram),
|
||||
address[0], address[1], err)
|
||||
return
|
||||
except (IndexError, KeyError):
|
||||
@@ -305,7 +301,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
# the node id of the node we sent a message to (these messages are treated as an error)
|
||||
if remoteContact.id and remoteContact.id != message.nodeID: # sent_to_id will be None for bootstrap
|
||||
log.debug("mismatch: (%s) %s:%i (%s vs %s)", method, remoteContact.address, remoteContact.port,
|
||||
remoteContact.log_id(False), message.nodeID.encode('hex'))
|
||||
remoteContact.log_id(False), hexlify(message.nodeID))
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
return
|
||||
elif not remoteContact.id:
|
||||
@@ -325,7 +321,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
If the data is spread over multiple UDP datagrams, the packets have the
|
||||
following structure::
|
||||
| | | | | |||||||||||| 0x00 |
|
||||
|Transmision|Total number|Sequence number| RPC ID |Header end|
|
||||
|Transmission|Total number|Sequence number| RPC ID |Header end|
|
||||
| type ID | of packets |of this packet | | indicator|
|
||||
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
|
||||
| | | | | |||||||||||| |
|
||||
@@ -343,7 +339,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
# 1st byte is transmission type id, bytes 2 & 3 are the
|
||||
# total number of packets in this transmission, bytes 4 &
|
||||
# 5 are the sequence number for this specific packet
|
||||
totalPackets = len(data) / self.msgSizeLimit
|
||||
totalPackets = len(data) // self.msgSizeLimit
|
||||
if len(data) % self.msgSizeLimit > 0:
|
||||
totalPackets += 1
|
||||
encTotalPackets = chr(totalPackets >> 8) + chr(totalPackets & 0xff)
|
||||
@@ -352,7 +348,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
while seqNumber < totalPackets:
|
||||
packetData = data[startPos:startPos + self.msgSizeLimit]
|
||||
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
|
||||
txData = '\x00%s%s%s\x00%s' % (encTotalPackets, encSeqNumber, rpcID, packetData)
|
||||
txData = f'\x00{encTotalPackets}{encSeqNumber}{rpcID}\x00{packetData}'
|
||||
self._scheduleSendNext(txData, address)
|
||||
|
||||
startPos += self.msgSizeLimit
|
||||
@@ -368,10 +364,10 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
if self.transport:
|
||||
try:
|
||||
self.transport.write(txData, address)
|
||||
except socket.error as err:
|
||||
except OSError as err:
|
||||
if err.errno == errno.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesnt seem to affect anything -grin
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
elif err.errno == errno.ENETUNREACH:
|
||||
# this should probably try to retransmit when the network connection is back
|
||||
@@ -388,15 +384,16 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
"""
|
||||
msg = msgtypes.ResponseMessage(rpcID, self._node.node_id, response)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = self._encoder.encode(msgPrimitive)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _sendError(self, contact, rpcID, exceptionType, exceptionMessage):
|
||||
""" Send an RPC error message to the specified contact
|
||||
"""
|
||||
exceptionMessage = exceptionMessage.encode()
|
||||
msg = msgtypes.ErrorMessage(rpcID, self._node.node_id, exceptionType, exceptionMessage)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = self._encoder.encode(msgPrimitive)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _handleRPC(self, senderContact, rpcID, method, args):
|
||||
@@ -414,7 +411,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df.addErrback(handleError)
|
||||
|
||||
# Execute the RPC
|
||||
func = getattr(self._node, method, None)
|
||||
func = getattr(self._node, method.decode(), None)
|
||||
if callable(func) and hasattr(func, "rpcmethod"):
|
||||
# Call the exposed Node method and return the result to the deferred callback chain
|
||||
# if args:
|
||||
@@ -423,19 +420,19 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
# else:
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self._node.externalIP, self._node.port, method,
|
||||
senderContact.address, senderContact.port)
|
||||
if args and isinstance(args[-1], dict) and 'protocolVersion' in args[-1]: # args don't need reformatting
|
||||
senderContact.update_protocol_version(int(args[-1].pop('protocolVersion')))
|
||||
if args and isinstance(args[-1], dict) and b'protocolVersion' in args[-1]: # args don't need reformatting
|
||||
senderContact.update_protocol_version(int(args[-1].pop(b'protocolVersion')))
|
||||
a, kw = tuple(args[:-1]), args[-1]
|
||||
else:
|
||||
senderContact.update_protocol_version(0)
|
||||
a, kw = self._migrate_incoming_rpc_args(senderContact, method, *args)
|
||||
try:
|
||||
if method != 'ping':
|
||||
if method != b'ping':
|
||||
result = func(senderContact, *a)
|
||||
else:
|
||||
result = func()
|
||||
except Exception, e:
|
||||
log.exception("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
except Exception as e:
|
||||
log.error("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
df.errback(e)
|
||||
else:
|
||||
df.callback(result)
|
||||
@@ -452,7 +449,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
log.error("deferred timed out, but is not present in sent messages list!")
|
||||
return
|
||||
remoteContact, df, timeout_call, timeout_canceller, method, args = self._sentMessages[messageID]
|
||||
if self._partialMessages.has_key(messageID):
|
||||
if messageID in self._partialMessages:
|
||||
# We are still receiving this message
|
||||
self._msgTimeoutInProgress(messageID, timeout_canceller, remoteContact, df, method, args)
|
||||
return
|
||||
@@ -478,7 +475,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
|
||||
def _hasProgressBeenMade(self, messageID):
|
||||
return (
|
||||
self._partialMessagesProgress.has_key(messageID) and
|
||||
messageID in self._partialMessagesProgress and
|
||||
(
|
||||
len(self._partialMessagesProgress[messageID]) !=
|
||||
len(self._partialMessages[messageID])
|
||||
|
||||
@@ -6,19 +6,17 @@
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import random
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
import constants
|
||||
import kbucket
|
||||
from error import TimeoutError
|
||||
from distance import Distance
|
||||
from interface import IRoutingTable
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.dht import constants, kbucket
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht.distance import Distance
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TreeRoutingTable(object):
|
||||
class TreeRoutingTable:
|
||||
""" This class implements a routing table used by a Node class.
|
||||
|
||||
The Kademlia routing table is a binary tree whFose leaves are k-buckets,
|
||||
@@ -33,7 +31,7 @@ class TreeRoutingTable(object):
|
||||
C{PING} RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
implements(IRoutingTable)
|
||||
#implements(IRoutingTable)
|
||||
|
||||
def __init__(self, parentNodeID, getTime=None):
|
||||
"""
|
||||
@@ -48,6 +46,7 @@ class TreeRoutingTable(object):
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self._ongoing_replacements = set()
|
||||
|
||||
def get_contacts(self):
|
||||
contacts = []
|
||||
@@ -134,8 +133,14 @@ class TreeRoutingTable(object):
|
||||
to_replace = not_good_contacts[0]
|
||||
else:
|
||||
to_replace = self._buckets[bucketIndex]._contacts[0]
|
||||
df = to_replace.ping()
|
||||
df.addErrback(replaceContact, to_replace)
|
||||
if to_replace not in self._ongoing_replacements:
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.port)
|
||||
self._ongoing_replacements.add(to_replace)
|
||||
df = to_replace.ping()
|
||||
df.addErrback(replaceContact, to_replace)
|
||||
df.addBoth(lambda _: self._ongoing_replacements.remove(to_replace))
|
||||
else:
|
||||
df = defer.succeed(None)
|
||||
return df
|
||||
else:
|
||||
self.touchKBucketByIndex(bucketIndex)
|
||||
@@ -150,7 +155,7 @@ class TreeRoutingTable(object):
|
||||
@param count: the amount of contacts to return, default of k (8)
|
||||
@type count: int
|
||||
@param sender_node_id: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the paramater will get
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type sender_node_id: str
|
||||
|
||||
@@ -180,12 +185,7 @@ class TreeRoutingTable(object):
|
||||
by this node
|
||||
"""
|
||||
bucketIndex = self._kbucketIndex(contactID)
|
||||
try:
|
||||
contact = self._buckets[bucketIndex].getContact(contactID)
|
||||
except ValueError:
|
||||
raise
|
||||
else:
|
||||
return contact
|
||||
return self._buckets[bucketIndex].getContact(contactID)
|
||||
|
||||
def getRefreshList(self, startIndex=0, force=False):
|
||||
""" Finds all k-buckets that need refreshing, starting at the
|
||||
@@ -213,7 +213,7 @@ class TreeRoutingTable(object):
|
||||
now = int(self._getTime())
|
||||
for bucket in self._buckets[startIndex:]:
|
||||
if force or now - bucket.lastAccessed >= constants.refreshTimeout:
|
||||
searchID = self._randomIDInBucketRange(bucketIndex)
|
||||
searchID = self.midpoint_id_in_bucket_range(bucketIndex)
|
||||
refreshIDs.append(searchID)
|
||||
bucketIndex += 1
|
||||
return refreshIDs
|
||||
@@ -261,22 +261,25 @@ class TreeRoutingTable(object):
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def _randomIDInBucketRange(self, bucketIndex):
|
||||
def random_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns a random ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
idValue = random.randrange(
|
||||
self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax)
|
||||
randomID = hex(idValue)[2:]
|
||||
if randomID[-1] == 'L':
|
||||
randomID = randomID[:-1]
|
||||
if len(randomID) % 2 != 0:
|
||||
randomID = '0' + randomID
|
||||
randomID = randomID.decode('hex')
|
||||
randomID = (constants.key_bits / 8 - len(randomID)) * '\x00' + randomID
|
||||
return randomID
|
||||
|
||||
random_id = int(random.randrange(self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax))
|
||||
return random_id.to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns the middle ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
|
||||
half = int((self._buckets[bucketIndex].rangeMax - self._buckets[bucketIndex].rangeMin) // 2)
|
||||
return int(self._buckets[bucketIndex].rangeMin + half).to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def _splitBucket(self, oldBucketIndex):
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
import typing
|
||||
|
||||
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
|
||||
# This needs to happen before anything else
|
||||
if 'win' in sys.platform:
|
||||
import certifi
|
||||
os.environ['SSL_CERT_FILE'] = certifi.where()
|
||||
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
from twisted.internet import reactor
|
||||
import logging
|
||||
from aiohttp.client_exceptions import ClientConnectorError
|
||||
from requests.exceptions import ConnectionError
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
from lbrynet import conf, log_support, __name__ as lbrynet_name
|
||||
from lbrynet.utils import check_connection, json_dumps_pretty
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from lbrynet.extras.daemon.DaemonConsole import main as daemon_console
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.system_info import get_platform
|
||||
|
||||
log = logging.getLogger(lbrynet_name)
|
||||
|
||||
optional_path_getter_type = typing.Optional[typing.Callable[[], str]]
|
||||
|
||||
|
||||
def start_daemon(settings: typing.Optional[typing.Dict] = None,
|
||||
console_output: typing.Optional[bool] = True, verbose: typing.Optional[typing.List[str]] = None,
|
||||
data_dir: typing.Optional[str] = None, wallet_dir: typing.Optional[str] = None,
|
||||
download_dir: typing.Optional[str] = None):
|
||||
|
||||
settings = settings or {}
|
||||
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
|
||||
for k, v in settings.items():
|
||||
conf.settings.update({k, v}, data_types=(conf.TYPE_CLI,))
|
||||
|
||||
log_support.configure_logging(conf.settings.get_log_filename(), console_output, verbose)
|
||||
log_support.configure_loggly_handler()
|
||||
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if check_connection():
|
||||
daemon = Daemon()
|
||||
daemon.start_listening()
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
|
||||
def start_daemon_with_cli_args(argv=None, data_dir: typing.Optional[str] = None,
|
||||
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=False
|
||||
)
|
||||
parser.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
parser.add_argument(
|
||||
'--version', action="store_true",
|
||||
help='Show daemon version and quit'
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
settings = {}
|
||||
if args.useauth:
|
||||
settings['use_auth_http'] = True
|
||||
|
||||
verbose = None
|
||||
if args.verbose:
|
||||
verbose = args.verbose
|
||||
|
||||
console_output = not args.quiet
|
||||
|
||||
if args.version:
|
||||
print(json_dumps_pretty(get_platform()))
|
||||
return
|
||||
|
||||
return start_daemon(settings, console_output, verbose, data_dir, wallet_dir, download_dir)
|
||||
|
||||
|
||||
async def execute_command(method, params, data_dir: typing.Optional[str] = None,
|
||||
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
|
||||
# this check if the daemon is running or not
|
||||
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
|
||||
api = None
|
||||
try:
|
||||
api = await LBRYAPIClient.get_client()
|
||||
await api.status()
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
if api:
|
||||
await api.session.close()
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
return 1
|
||||
|
||||
# this actually executes the method
|
||||
resp = await api.call(method, params)
|
||||
|
||||
try:
|
||||
await api.session.close()
|
||||
print(json.dumps(resp["result"], indent=2))
|
||||
except KeyError:
|
||||
if resp["error"]["code"] == -32500:
|
||||
print(json.dumps(resp["error"], indent=2))
|
||||
else:
|
||||
print(json.dumps(resp["error"]["message"], indent=2))
|
||||
|
||||
|
||||
def print_help():
|
||||
print(dedent("""
|
||||
NAME
|
||||
lbrynet - LBRY command line client.
|
||||
|
||||
USAGE
|
||||
lbrynet [--data_dir=<blob and database directory>] [--wallet_dir=<wallet directory>]
|
||||
[--download_dir=<downloads directory>] <command> [<args>]
|
||||
|
||||
EXAMPLES
|
||||
lbrynet start # starts the daemon and listens for jsonrpc commands
|
||||
lbrynet help # display this message
|
||||
lbrynet help <command_name> # get help for a command(doesn't need the daemon to be running)
|
||||
lbrynet commands # list available commands
|
||||
lbrynet status # get the running status of the daemon
|
||||
lbrynet resolve what # resolve a name
|
||||
|
||||
lbrynet --wallet_dir=~/wallet2 start # start the daemon using an alternative wallet directory
|
||||
lbrynet --data_dir=~/lbry start # start the daemon using an alternative data directory
|
||||
|
||||
lbrynet --data_dir=~/lbry <command_name> # run a command on a daemon using an alternative data directory,
|
||||
# which can contain a full daemon_settings.yml config file.
|
||||
# Note: since the daemon is what runs the wallet and
|
||||
# downloads files, only the --data_dir setting is needed when
|
||||
# running commands. The wallet_dir and download_dir would only
|
||||
# by used when starting the daemon.
|
||||
"""))
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print(dedent(fn.__doc__))
|
||||
else:
|
||||
print("Invalid command name")
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
k = None
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
if not argv:
|
||||
print_help()
|
||||
return 1
|
||||
|
||||
dir_args = {}
|
||||
if len(argv) > 2:
|
||||
dir_arg_keys = [
|
||||
'data_dir',
|
||||
'wallet_dir',
|
||||
'download_directory'
|
||||
]
|
||||
|
||||
for arg in argv:
|
||||
found_dir_arg = False
|
||||
for key in dir_arg_keys:
|
||||
if arg.startswith(f'--{key}='):
|
||||
if key in dir_args:
|
||||
print(f"Multiple values provided for '{key}' argument")
|
||||
print_help()
|
||||
return 1
|
||||
dir_args[key] = os.path.expanduser(os.path.expandvars(arg.lstrip(f'--{key}=')))
|
||||
found_dir_arg = True
|
||||
if not found_dir_arg:
|
||||
break
|
||||
argv = argv[len(dir_args):]
|
||||
|
||||
data_dir = dir_args.get('data_dir')
|
||||
wallet_dir = dir_args.get('wallet_dir')
|
||||
download_dir = dir_args.get('download_directory')
|
||||
|
||||
for k, v in dir_args.items():
|
||||
if not os.path.isdir(v):
|
||||
print(f"'{data_dir}' is not a directory, cannot use it for {k}")
|
||||
return 1
|
||||
|
||||
method, args = argv[0], argv[1:]
|
||||
|
||||
if method in ['help', '--help', '-h']:
|
||||
if len(args) == 1:
|
||||
print_help_for_command(args[0])
|
||||
else:
|
||||
print_help()
|
||||
return 0
|
||||
|
||||
elif method in ['version', '--version', '-v']:
|
||||
print("{lbrynet_name} {lbrynet_version}".format(
|
||||
lbrynet_name=lbrynet_name, **get_platform()
|
||||
))
|
||||
return 0
|
||||
|
||||
elif method == 'start':
|
||||
sys.exit(start_daemon_with_cli_args(args, data_dir, wallet_dir, download_dir))
|
||||
|
||||
elif method == 'console':
|
||||
sys.exit(daemon_console())
|
||||
|
||||
elif method not in Daemon.callable_methods:
|
||||
if method not in Daemon.deprecated_methods:
|
||||
print(f'{method} is not a valid command.')
|
||||
return 1
|
||||
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
return 0
|
||||
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(method, params, data_dir, wallet_dir, download_dir))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,74 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted._threads import AlreadyQuit
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ComponentType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
if name != "Component":
|
||||
ComponentManager.default_component_classes[klass.component_name] = klass
|
||||
return klass
|
||||
|
||||
|
||||
class Component(metaclass=ComponentType):
|
||||
"""
|
||||
lbrynet-daemon component helper
|
||||
|
||||
Inheriting classes will be automatically registered with the ComponentManager and must implement setup and stop
|
||||
methods
|
||||
"""
|
||||
|
||||
depends_on = []
|
||||
component_name = None
|
||||
|
||||
def __init__(self, component_manager):
|
||||
self.component_manager = component_manager
|
||||
self._running = False
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.component_name < other.component_name
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def get_status(self):
|
||||
return
|
||||
|
||||
def start(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.start)
|
||||
self._running = True
|
||||
defer.returnValue(result)
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.stop)
|
||||
self._running = False
|
||||
defer.returnValue(result)
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error stopping %s", self.__class__.__name__)
|
||||
raise err
|
||||
@@ -0,0 +1,178 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.p2p.Error import ComponentStartConditionNotMet
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RegisteredConditions:
|
||||
conditions = {}
|
||||
|
||||
|
||||
class RequiredConditionType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
if name != "RequiredCondition":
|
||||
if klass.name in RegisteredConditions.conditions:
|
||||
raise SyntaxError("already have a component registered for \"%s\"" % klass.name)
|
||||
RegisteredConditions.conditions[klass.name] = klass
|
||||
return klass
|
||||
|
||||
|
||||
class RequiredCondition(metaclass=RequiredConditionType):
|
||||
name = ""
|
||||
component = ""
|
||||
message = ""
|
||||
|
||||
@staticmethod
|
||||
def evaluate(component):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class ComponentManager:
|
||||
default_component_classes = {}
|
||||
|
||||
def __init__(self, reactor=None, analytics_manager=None, skip_components=None,
|
||||
peer_manager=None, peer_finder=None, **override_components):
|
||||
self.skip_components = skip_components or []
|
||||
self.reactor = reactor
|
||||
self.component_classes = {}
|
||||
self.components = set()
|
||||
self.analytics_manager = analytics_manager
|
||||
self.peer_manager = peer_manager or PeerManager()
|
||||
self.peer_finder = peer_finder or DHTPeerFinder(self)
|
||||
|
||||
for component_name, component_class in self.default_component_classes.items():
|
||||
if component_name in override_components:
|
||||
component_class = override_components.pop(component_name)
|
||||
if component_name not in self.skip_components:
|
||||
self.component_classes[component_name] = component_class
|
||||
|
||||
if override_components:
|
||||
raise SyntaxError("unexpected components: %s" % override_components)
|
||||
|
||||
for component_class in self.component_classes.values():
|
||||
self.components.add(component_class(self))
|
||||
|
||||
def evaluate_condition(self, condition_name):
|
||||
if condition_name not in RegisteredConditions.conditions:
|
||||
raise NameError(condition_name)
|
||||
condition = RegisteredConditions.conditions[condition_name]
|
||||
try:
|
||||
component = self.get_component(condition.component)
|
||||
result = condition.evaluate(component)
|
||||
except Exception:
|
||||
log.exception('failed to evaluate condition:')
|
||||
result = False
|
||||
return result, "" if result else condition.message
|
||||
|
||||
def sort_components(self, reverse=False):
|
||||
"""
|
||||
Sort components by requirements
|
||||
"""
|
||||
steps = []
|
||||
staged = set()
|
||||
components = set(self.components)
|
||||
|
||||
# components with no requirements
|
||||
step = []
|
||||
for component in set(components):
|
||||
if not component.depends_on:
|
||||
step.append(component)
|
||||
staged.add(component.component_name)
|
||||
components.remove(component)
|
||||
|
||||
if step:
|
||||
step.sort()
|
||||
steps.append(step)
|
||||
|
||||
while components:
|
||||
step = []
|
||||
to_stage = set()
|
||||
for component in set(components):
|
||||
reqs_met = 0
|
||||
for needed in component.depends_on:
|
||||
if needed in staged:
|
||||
reqs_met += 1
|
||||
if reqs_met == len(component.depends_on):
|
||||
step.append(component)
|
||||
to_stage.add(component.component_name)
|
||||
components.remove(component)
|
||||
if step:
|
||||
step.sort()
|
||||
staged.update(to_stage)
|
||||
steps.append(step)
|
||||
elif components:
|
||||
raise ComponentStartConditionNotMet("Unresolved dependencies for: %s" % components)
|
||||
if reverse:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self, **callbacks):
|
||||
"""
|
||||
Start Components in sequence sorted by requirements
|
||||
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
for component_name, cb in callbacks.items():
|
||||
if component_name not in self.component_classes:
|
||||
if component_name not in self.skip_components:
|
||||
raise NameError("unknown component: %s" % component_name)
|
||||
if not callable(cb):
|
||||
raise ValueError("%s is not callable" % cb)
|
||||
|
||||
def _setup(component):
|
||||
if component.component_name in callbacks:
|
||||
d = component._setup()
|
||||
d.addCallback(callbacks[component.component_name], component)
|
||||
return d
|
||||
return component._setup()
|
||||
|
||||
stages = self.sort_components()
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([_setup(component) for component in stage if not component.running])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
"""
|
||||
Stop Components in reversed startup order
|
||||
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
stages = self.sort_components(reverse=True)
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([component._stop() for component in stage if component.running])
|
||||
|
||||
def all_components_running(self, *component_names):
|
||||
"""
|
||||
Check if components are running
|
||||
|
||||
:return: (bool) True if all specified components are running
|
||||
"""
|
||||
components = {component.component_name: component for component in self.components}
|
||||
for component in component_names:
|
||||
if component not in components:
|
||||
raise NameError("%s is not a known Component" % component)
|
||||
if not components[component].running:
|
||||
return False
|
||||
return True
|
||||
|
||||
def get_components_status(self):
|
||||
"""
|
||||
List status of all the components, whether they are running or not
|
||||
|
||||
:return: (dict) {(str) component_name: (bool) True is running else False}
|
||||
"""
|
||||
return {
|
||||
component.component_name: component.running
|
||||
for component in self.components
|
||||
}
|
||||
|
||||
def get_component(self, component_name):
|
||||
for component in self.components:
|
||||
if component.component_name == component_name:
|
||||
return component.component
|
||||
raise NameError(component_name)
|
||||
@@ -0,0 +1,775 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import treq
|
||||
import json
|
||||
import math
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
from twisted.internet import defer, threads, reactor, error, task
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
from lbrynet import conf
|
||||
|
||||
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.utils import DeferredDict, generate_id
|
||||
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.p2p.RateLimiter import RateLimiter
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
|
||||
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# settings must be initialized before this file is imported
|
||||
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
HEADERS_COMPONENT = "blockchain_headers"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
REFLECTOR_COMPONENT = "reflector"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
RATE_LIMITER_COMPONENT = "rate_limiter"
|
||||
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
|
||||
|
||||
|
||||
def from_future(coroutine: asyncio.coroutine) -> defer.Deferred:
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(coroutine))
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
buf = []
|
||||
response = yield treq.get("https://api.lbry.io/ip")
|
||||
yield treq.collect(response, buf.append)
|
||||
parsed = json.loads(b"".join(buf).decode())
|
||||
if parsed['success']:
|
||||
return parsed['data']['ip']
|
||||
return
|
||||
except Exception as err:
|
||||
return
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.storage = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.storage
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 10
|
||||
|
||||
@staticmethod
|
||||
def get_revision_filename():
|
||||
return conf.settings.get_db_revision_filename()
|
||||
|
||||
@staticmethod
|
||||
def _write_db_revision_file(version_num):
|
||||
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
|
||||
db_revision.write(str(version_num))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
# check directories exist, create them if they don't
|
||||
log.info("Loading databases")
|
||||
|
||||
if not os.path.exists(self.get_revision_filename()):
|
||||
log.warning("db_revision file not found. Creating it")
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
|
||||
# check the db migration and run any needed migrations
|
||||
with open(self.get_revision_filename(), "r") as revision_read_handle:
|
||||
old_revision = int(revision_read_handle.read().strip())
|
||||
|
||||
if old_revision > self.get_current_db_revision():
|
||||
raise Exception('This version of lbrynet is not compatible with the database\n'
|
||||
'Your database is revision %i, expected %i' %
|
||||
(old_revision, self.get_current_db_revision()))
|
||||
if old_revision < self.get_current_db_revision():
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
yield threads.deferToThread(
|
||||
dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
|
||||
# start SQLiteStorage
|
||||
self.storage = SQLiteStorage(conf.settings.data_dir)
|
||||
yield self.storage.setup()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.storage.stop()
|
||||
self.storage = None
|
||||
|
||||
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
HEADER_SIZE = 112
|
||||
|
||||
|
||||
class HeadersComponent(Component):
|
||||
component_name = HEADERS_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.headers_dir = os.path.join(conf.settings.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_file = os.path.join(self.headers_dir, 'headers')
|
||||
self.old_file = os.path.join(conf.settings.wallet_dir, 'blockchain_headers')
|
||||
self._downloading_headers = None
|
||||
self._headers_progress_percent = 0
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def get_status(self):
|
||||
return {} if not self._downloading_headers else {
|
||||
'downloading_headers': self._downloading_headers,
|
||||
'download_progress': self._headers_progress_percent
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def fetch_headers_from_s3(self):
|
||||
def collector(data, h_file):
|
||||
h_file.write(data)
|
||||
local_size = float(h_file.tell())
|
||||
final_size = float(final_size_after_download)
|
||||
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
|
||||
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
response = yield treq.get(HEADERS_URL, headers=resume_header)
|
||||
got_406 = response.code == 406 # our file is bigger
|
||||
final_size_after_download = response.length + local_header_size
|
||||
if got_406:
|
||||
log.warning("s3 is more out of date than we are")
|
||||
# should have something to download and a final length divisible by the header size
|
||||
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
|
||||
s3_height = (final_size_after_download / HEADER_SIZE) - 1
|
||||
local_height = self.local_header_file_height()
|
||||
if s3_height > local_height:
|
||||
if local_header_size:
|
||||
log.info("Resuming download of %i bytes from s3", response.length)
|
||||
with open(self.headers_file, "a+b") as headers_file:
|
||||
yield treq.collect(response, lambda d: collector(d, headers_file))
|
||||
else:
|
||||
with open(self.headers_file, "wb") as headers_file:
|
||||
yield treq.collect(response, lambda d: collector(d, headers_file))
|
||||
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
|
||||
self._check_header_file_integrity()
|
||||
else:
|
||||
log.warning("s3 is more out of date than we are")
|
||||
else:
|
||||
log.error("invalid size for headers from s3")
|
||||
|
||||
def local_header_file_height(self):
|
||||
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
|
||||
|
||||
def local_header_file_size(self):
|
||||
if os.path.isfile(self.headers_file):
|
||||
return os.stat(self.headers_file).st_size
|
||||
return 0
|
||||
|
||||
async def get_remote_height(self):
|
||||
ledger = SimpleNamespace()
|
||||
ledger.config = {
|
||||
'default_servers': conf.settings['lbryum_servers'],
|
||||
'data_path': conf.settings.wallet_dir
|
||||
}
|
||||
net = Network(ledger)
|
||||
first_connection = net.on_connected.first
|
||||
asyncio.ensure_future(net.start())
|
||||
await first_connection
|
||||
remote_height = await net.get_server_height()
|
||||
await net.stop()
|
||||
return remote_height
|
||||
|
||||
async def should_download_headers_from_s3(self):
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
return False
|
||||
self._check_header_file_integrity()
|
||||
s3_headers_depth = conf.settings['s3_headers_depth']
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
local_height = self.local_header_file_height()
|
||||
remote_height = await self.get_remote_height()
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
if remote_height > (local_height + s3_headers_depth):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_header_file_integrity(self):
|
||||
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
return
|
||||
hashsum = sha256()
|
||||
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
|
||||
checksum_length_in_bytes = checksum_height * HEADER_SIZE
|
||||
if self.local_header_file_size() < checksum_length_in_bytes:
|
||||
return
|
||||
with open(self.headers_file, "rb") as headers_file:
|
||||
hashsum.update(headers_file.read(checksum_length_in_bytes))
|
||||
current_checksum = hashsum.hexdigest()
|
||||
if current_checksum != checksum:
|
||||
msg = f"Expected checksum {checksum}, got {current_checksum}"
|
||||
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
|
||||
log.warning("Deleting header file so it can be downloaded again.")
|
||||
os.unlink(self.headers_file)
|
||||
elif (self.local_header_file_size() % HEADER_SIZE) != 0:
|
||||
log.warning("Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint.")
|
||||
with open(self.headers_file, "rb+") as headers_file:
|
||||
headers_file.truncate(checksum_length_in_bytes)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
self._downloading_headers = yield f2d(self.should_download_headers_from_s3())
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
yield self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self._downloading_headers = False
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
component_name = WALLET_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, HEADERS_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.wallet_manager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.wallet_manager
|
||||
|
||||
def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet_manager.wallet.use_encryption,
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
|
||||
self.wallet_manager = yield f2d(LbryWalletManager.from_lbrynet_config(conf.settings, storage))
|
||||
self.wallet_manager.old_db = storage
|
||||
yield f2d(self.wallet_manager.start())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield f2d(self.wallet_manager.stop())
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.blob_manager
|
||||
|
||||
def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
datastore = None
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
datastore = dht_node._dataStore
|
||||
self.blob_manager = DiskBlobManager(os.path.join(conf.settings.data_dir, "blobfiles"), storage, datastore)
|
||||
return self.blob_manager.setup()
|
||||
|
||||
def stop(self):
|
||||
return self.blob_manager.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = yield self.blob_manager.storage.count_finished_blobs()
|
||||
defer.returnValue({
|
||||
'finished_blobs': count
|
||||
})
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
component_name = DHT_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.dht_node = None
|
||||
self.upnp_component = None
|
||||
self.external_udp_port = None
|
||||
self.external_peer_port = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.dht_node
|
||||
|
||||
def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(conf.settings.get_node_id()),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"])
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"])
|
||||
node_id = conf.settings.get_node_id()
|
||||
if node_id is None:
|
||||
node_id = generate_id()
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = yield get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
self.dht_node = Node(
|
||||
node_id=node_id,
|
||||
udpPort=conf.settings['dht_node_port'],
|
||||
externalUDPPort=self.external_udp_port,
|
||||
externalIP=external_ip,
|
||||
peerPort=self.external_peer_port
|
||||
)
|
||||
|
||||
yield self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False)
|
||||
log.info("Started the dht")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.dht_node.stop()
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
component_name = HASH_ANNOUNCER_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.hash_announcer = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.hash_announcer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = DHTHashAnnouncer(dht_node, storage)
|
||||
yield self.hash_announcer.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.hash_announcer.stop()
|
||||
|
||||
def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
|
||||
}
|
||||
|
||||
|
||||
class RateLimiterComponent(Component):
|
||||
component_name = RATE_LIMITER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.rate_limiter = RateLimiter()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.rate_limiter
|
||||
|
||||
def start(self):
|
||||
self.rate_limiter.start()
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
self.rate_limiter.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class PaymentRateComponent(Component):
|
||||
component_name = PAYMENT_RATE_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.payment_rate_manager = OnlyFreePaymentsManager()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.payment_rate_manager
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
|
||||
PAYMENT_RATE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.file_manager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.file_manager
|
||||
|
||||
def get_status(self):
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.lbry_files)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
|
||||
sd_identifier = StreamDescriptorIdentifier()
|
||||
add_lbry_file_to_sd_identifier(sd_identifier)
|
||||
file_saver_factory = EncryptedFileSaverFactory(
|
||||
self.component_manager.peer_finder,
|
||||
rate_limiter,
|
||||
blob_manager,
|
||||
storage,
|
||||
wallet,
|
||||
conf.settings.download_dir
|
||||
)
|
||||
yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
|
||||
log.info('Starting the file manager')
|
||||
self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier)
|
||||
yield self.file_manager.setup()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.file_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
component_name = PEER_PROTOCOL_SERVER_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
|
||||
PAYMENT_RATE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.lbry_server_port = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.lbry_server_port
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
peer_port = conf.settings['peer_port']
|
||||
query_handlers = {
|
||||
handler.get_primary_query_identifier(): handler for handler in [
|
||||
BlobRequestHandlerFactory(
|
||||
self.component_manager.get_component(BLOB_COMPONENT),
|
||||
wallet,
|
||||
self.component_manager.get_component(PAYMENT_RATE_COMPONENT),
|
||||
self.component_manager.analytics_manager
|
||||
),
|
||||
wallet.get_wallet_info_query_handler_factory(),
|
||||
]
|
||||
}
|
||||
server_factory = ServerProtocolFactory(
|
||||
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
|
||||
self.component_manager.peer_manager
|
||||
)
|
||||
|
||||
try:
|
||||
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
|
||||
upnp.upnp_redirects.get("TCP", peer_port))
|
||||
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
|
||||
" more details.", peer_port)
|
||||
log.error("%s", traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, old_port = None, self.lbry_server_port
|
||||
log.info('Stop listening on port %s', old_port.port)
|
||||
yield old_port.stopListening()
|
||||
|
||||
|
||||
class ReflectorComponent(Component):
|
||||
component_name = REFLECTOR_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.reflector_server_port = conf.settings['reflector_port']
|
||||
self.reflector_server = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.reflector_server
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
log.info("Starting reflector server")
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
|
||||
try:
|
||||
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
|
||||
log.info('Started reflector on port %s', self.reflector_server_port)
|
||||
except error.CannotListenError as e:
|
||||
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
|
||||
raise ValueError(f"{e} lbrynet may already be running on your computer.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
if self.reflector_server is not None:
|
||||
log.info("Stopping reflector server")
|
||||
self.reflector_server, p = None, self.reflector_server
|
||||
yield p.stopListening
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
component_name = UPNP_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = conf.settings['peer_port']
|
||||
self._int_dht_node_port = conf.settings['dht_node_port']
|
||||
self.use_upnp = conf.settings['use_upnp']
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
self.external_ip = None
|
||||
self._maintain_redirects_lc = task.LoopingCall(self._maintain_redirects)
|
||||
self._maintain_redirects_lc.clock = self.component_manager.reactor
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup_redirects(self):
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
if not self.upnp:
|
||||
try:
|
||||
self.upnp = yield from_future(UPnP.discover())
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
self.upnp = None
|
||||
|
||||
# update the external ip
|
||||
external_ip = None
|
||||
if self.upnp:
|
||||
try:
|
||||
external_ip = yield from_future(self.upnp.get_external_ip())
|
||||
if external_ip != "0.0.0.0" and not self.external_ip:
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
pass
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = yield get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
assert self.external_ip is not None # TODO: handle going/starting offline
|
||||
|
||||
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
||||
try:
|
||||
log.info("add UPnP port mappings")
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
log.info("set up redirects: %s", upnp_redirects)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
self.upnp = None
|
||||
return self._maintain_redirects()
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = yield from_future(self.upnp.get_redirects())
|
||||
for mapping in mappings:
|
||||
proto = mapping['NewProtocol']
|
||||
if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
|
||||
if mapping['NewInternalClient'] == self.upnp.lan_address:
|
||||
found.add(proto)
|
||||
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
udp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
)
|
||||
self.upnp_redirects['UDP'] = udp_port
|
||||
log.info("refreshed upnp redirect for dht port: %i", udp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
del self.upnp_redirects['UDP']
|
||||
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
tcp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
)
|
||||
self.upnp_redirects['TCP'] = tcp_port
|
||||
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
del self.upnp_redirects['TCP']
|
||||
if ('TCP' in self.upnp_redirects
|
||||
and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and (
|
||||
'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
|
||||
if self.upnp_redirects:
|
||||
log.debug("upnp redirects are still active")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = yield get_external_ip()
|
||||
return
|
||||
success = False
|
||||
yield self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info)
|
||||
else:
|
||||
success = True
|
||||
if self.upnp_redirects:
|
||||
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status())
|
||||
self._maintain_redirects_lc.start(360, now=False)
|
||||
|
||||
def stop(self):
|
||||
if self._maintain_redirects_lc.running:
|
||||
self._maintain_redirects_lc.stop()
|
||||
return defer.DeferredList(
|
||||
[from_future(self.upnp.delete_port_mapping(port, protocol))
|
||||
for protocol, port in self.upnp_redirects.items()]
|
||||
)
|
||||
|
||||
def get_status(self):
|
||||
return {
|
||||
'aioupnp_version': aioupnp_version,
|
||||
'redirects': self.upnp_redirects,
|
||||
'gateway': 'No gateway found' if not self.upnp else self.upnp.gateway.manufacturer_string,
|
||||
'dht_redirect_set': 'UDP' in self.upnp_redirects,
|
||||
'peer_redirect_set': 'TCP' in self.upnp_redirects,
|
||||
'external_ip': self.external_ip
|
||||
}
|
||||
|
||||
|
||||
class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.exchange_rate_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
yield self.exchange_rate_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.exchange_rate_manager.stop()
|
||||
@@ -1,20 +1,15 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import code
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging.handlers
|
||||
from exceptions import SystemExit
|
||||
from twisted.internet import defer, reactor, threads
|
||||
from lbrynet import analytics
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core import log_support
|
||||
from lbrynet.daemon.DaemonServer import DaemonServer
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
from aiohttp import client_exceptions
|
||||
|
||||
get_client = LBRYAPIClient.get_client
|
||||
from lbrynet import utils, conf, log_support
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -108,7 +103,7 @@ To exit:
|
||||
"""
|
||||
|
||||
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
|
||||
welcome += "\n".join(["{:<60}".format(w) for w in welcometext.splitlines()])
|
||||
welcome += "\n".join([f"{w:<60}" for w in welcometext.splitlines()])
|
||||
welcome += "\n%s" % ("*" * 60)
|
||||
welcome = color(welcome, "grey")
|
||||
banner = color(logo, "green") + color(welcome, "grey")
|
||||
@@ -118,12 +113,12 @@ def get_methods(daemon):
|
||||
locs = {}
|
||||
|
||||
def wrapped(name, fn):
|
||||
client = get_client()
|
||||
client = LBRYAPIClient.get_client()
|
||||
_fn = getattr(client, name)
|
||||
_fn.__doc__ = fn.__doc__
|
||||
return {name: _fn}
|
||||
|
||||
for method_name, method in daemon.callable_methods.iteritems():
|
||||
for method_name, method in daemon.callable_methods.items():
|
||||
locs.update(wrapped(method_name, method))
|
||||
return locs
|
||||
|
||||
@@ -134,14 +129,14 @@ def run_terminal(callable_methods, started_daemon, quiet=False):
|
||||
|
||||
def help(method_name=None):
|
||||
if not method_name:
|
||||
print "Available api functions: "
|
||||
print("Available api functions: ")
|
||||
for name in callable_methods:
|
||||
print "\t%s" % name
|
||||
print("\t%s" % name)
|
||||
return
|
||||
if method_name not in callable_methods:
|
||||
print "\"%s\" is not a recognized api function"
|
||||
print("\"%s\" is not a recognized api function")
|
||||
return
|
||||
print callable_methods[method_name].__doc__
|
||||
print(callable_methods[method_name].__doc__)
|
||||
return
|
||||
|
||||
locs.update({'help': help})
|
||||
@@ -149,7 +144,7 @@ def run_terminal(callable_methods, started_daemon, quiet=False):
|
||||
if started_daemon:
|
||||
def exit(status=None):
|
||||
if not quiet:
|
||||
print "Stopping lbrynet-daemon..."
|
||||
print("Stopping lbrynet-daemon...")
|
||||
callable_methods['daemon_stop']()
|
||||
return sys.exit(status)
|
||||
|
||||
@@ -159,7 +154,7 @@ def run_terminal(callable_methods, started_daemon, quiet=False):
|
||||
try:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
except Exception as err:
|
||||
print "error stopping reactor: ", err
|
||||
print(f"error stopping reactor: {err}")
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
@@ -175,18 +170,7 @@ def start_server_and_listen(use_auth, analytics_manager, quiet):
|
||||
logging.getLogger("requests").setLevel(logging.CRITICAL)
|
||||
|
||||
analytics_manager.send_server_startup()
|
||||
daemon_server = DaemonServer(analytics_manager)
|
||||
try:
|
||||
yield daemon_server.start(use_auth)
|
||||
analytics_manager.send_server_startup_success()
|
||||
if not quiet:
|
||||
print "Started lbrynet-daemon!"
|
||||
defer.returnValue(True)
|
||||
except Exception as e:
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
analytics_manager.send_server_startup_error(str(e))
|
||||
daemon_server.stop()
|
||||
raise
|
||||
yield Daemon().start_listening()
|
||||
|
||||
|
||||
def threaded_terminal(started_daemon, quiet):
|
||||
@@ -196,21 +180,21 @@ def threaded_terminal(started_daemon, quiet):
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
def start_lbrynet_console(quiet, use_existing_daemon, useauth):
|
||||
async def start_lbrynet_console(quiet, use_existing_daemon, useauth):
|
||||
if not utils.check_connection():
|
||||
print "Not connected to internet, unable to start"
|
||||
print("Not connected to internet, unable to start")
|
||||
raise Exception("Not connected to internet, unable to start")
|
||||
if not quiet:
|
||||
print "Starting lbrynet-console..."
|
||||
print("Starting lbrynet-console...")
|
||||
try:
|
||||
get_client().status()
|
||||
await LBRYAPIClient.get_client().status()
|
||||
d = defer.succeed(False)
|
||||
if not quiet:
|
||||
print "lbrynet-daemon is already running, connecting to it..."
|
||||
except:
|
||||
print("lbrynet-daemon is already running, connecting to it...")
|
||||
except client_exceptions.ClientConnectorError:
|
||||
if not use_existing_daemon:
|
||||
if not quiet:
|
||||
print "Starting lbrynet-daemon..."
|
||||
print("Starting lbrynet-daemon...")
|
||||
analytics_manager = analytics.Manager.new_instance()
|
||||
d = start_server_and_listen(useauth, analytics_manager, quiet)
|
||||
else:
|
||||
@@ -237,7 +221,8 @@ def main():
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
args = parser.parse_args()
|
||||
start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
|
||||
reactor.run()
|
||||
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbryschema.fee import Fee
|
||||
|
||||
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
|
||||
from lbrynet.core.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout
|
||||
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
from lbrynet.core.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet import conf
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
|
||||
from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError
|
||||
from lbrynet.p2p.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from torba.client.constants import COIN
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.daemon.Components import f2d
|
||||
|
||||
INITIALIZING_CODE = 'initializing'
|
||||
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
|
||||
@@ -29,62 +30,41 @@ STREAM_STAGES = [
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GetStream(object):
|
||||
def __init__(self, sd_identifier, session, exchange_rate_manager,
|
||||
max_key_fee, disable_max_key_fee, data_rate=None, timeout=None):
|
||||
|
||||
class GetStream:
|
||||
def __init__(self, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, rate_limiter,
|
||||
payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None, timeout=None,
|
||||
reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.reactor = reactor
|
||||
self.timeout = timeout or conf.settings['download_timeout']
|
||||
self.data_rate = data_rate or conf.settings['data_rate']
|
||||
self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
|
||||
self.download_directory = conf.settings['download_directory']
|
||||
self.download_directory = conf.settings.download_dir
|
||||
self.timeout_counter = 0
|
||||
self.code = None
|
||||
self.sd_hash = None
|
||||
self.session = session
|
||||
self.wallet = self.session.wallet
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.wallet = wallet
|
||||
self.exchange_rate_manager = exchange_rate_manager
|
||||
self.payment_rate_manager = self.session.payment_rate_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.storage = storage
|
||||
self.downloader = None
|
||||
self.checker = LoopingCall(self.check_status)
|
||||
|
||||
# fired when the download is complete
|
||||
self.finished_deferred = None
|
||||
# fired after the metadata and the first data blob have been downloaded
|
||||
self.data_downloading_deferred = defer.Deferred(None)
|
||||
self.wrote_data = False
|
||||
|
||||
@property
|
||||
def download_path(self):
|
||||
return os.path.join(self.download_directory, self.downloader.file_name)
|
||||
|
||||
def _check_status(self, status):
|
||||
if status.num_completed > 0 and not self.data_downloading_deferred.called:
|
||||
self.data_downloading_deferred.callback(True)
|
||||
if self.data_downloading_deferred.called:
|
||||
safe_stop_looping_call(self.checker)
|
||||
else:
|
||||
log.debug("Waiting for stream data (%i seconds)", self.timeout_counter)
|
||||
|
||||
def check_status(self):
|
||||
"""
|
||||
Check if we've got the first data blob in the stream yet
|
||||
"""
|
||||
self.timeout_counter += 1
|
||||
if self.timeout_counter > self.timeout:
|
||||
if not self.data_downloading_deferred.called:
|
||||
if self.downloader:
|
||||
err = DownloadDataTimeout(self.sd_hash)
|
||||
else:
|
||||
err = DownloadSDTimeout(self.sd_hash)
|
||||
self.data_downloading_deferred.errback(err)
|
||||
safe_stop_looping_call(self.checker)
|
||||
elif self.downloader:
|
||||
d = self.downloader.status()
|
||||
d.addCallback(self._check_status)
|
||||
else:
|
||||
log.debug("Waiting for stream descriptor (%i seconds)", self.timeout_counter)
|
||||
|
||||
def convert_max_fee(self):
|
||||
currency, amount = self.max_key_fee['currency'], self.max_key_fee['amount']
|
||||
return self.exchange_rate_manager.convert_currency(currency, "LBC", amount)
|
||||
@@ -93,15 +73,16 @@ class GetStream(object):
|
||||
log.info("Download lbry://%s status changed to %s" % (name, status))
|
||||
self.code = next(s for s in STREAM_STAGES if s[0] == status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def check_fee_and_convert(self, fee):
|
||||
max_key_fee_amount = self.convert_max_fee()
|
||||
converted_fee_amount = self.exchange_rate_manager.convert_currency(fee.currency, "LBC",
|
||||
fee.amount)
|
||||
if converted_fee_amount > self.wallet.get_balance():
|
||||
if converted_fee_amount > (yield f2d(self.wallet.default_account.get_balance())):
|
||||
raise InsufficientFundsError('Unable to pay the key fee of %s' % converted_fee_amount)
|
||||
if converted_fee_amount > max_key_fee_amount and not self.disable_max_key_fee:
|
||||
raise KeyFeeAboveMaxAllowed('Key fee %s above max allowed %s' % (converted_fee_amount,
|
||||
max_key_fee_amount))
|
||||
raise KeyFeeAboveMaxAllowed('Key fee {} above max allowed {}'.format(converted_fee_amount,
|
||||
max_key_fee_amount))
|
||||
converted_fee = {
|
||||
'currency': 'LBC',
|
||||
'amount': converted_fee_amount,
|
||||
@@ -113,7 +94,7 @@ class GetStream(object):
|
||||
for factory in factories:
|
||||
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
|
||||
return factory
|
||||
raise Exception('No suitable factory was found in {}'.format(factories))
|
||||
raise Exception(f'No suitable factory was found in {factories}')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_downloader(self, factory, stream_metadata, file_name=None):
|
||||
@@ -129,37 +110,34 @@ class GetStream(object):
|
||||
defer.returnValue(downloader)
|
||||
|
||||
def _pay_key_fee(self, address, fee_lbc, name):
|
||||
log.info("Pay key fee %f --> %s", fee_lbc, address)
|
||||
log.info("Pay key fee %s --> %s", dewies_to_lbc(fee_lbc), address)
|
||||
reserved_points = self.wallet.reserve_points(address, fee_lbc)
|
||||
if reserved_points is None:
|
||||
raise InsufficientFundsError('Unable to pay the key fee of %s for %s' % (fee_lbc, name))
|
||||
return self.wallet.send_points_to_address(reserved_points, fee_lbc)
|
||||
raise InsufficientFundsError(
|
||||
'Unable to pay the key fee of {} for {}'.format(dewies_to_lbc(fee_lbc), name)
|
||||
)
|
||||
return f2d(self.wallet.send_points_to_address(reserved_points, fee_lbc))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def pay_key_fee(self, fee, name):
|
||||
if fee is not None:
|
||||
yield self._pay_key_fee(fee.address, fee.amount, name)
|
||||
yield self._pay_key_fee(fee.address.decode(), int(fee.amount * COIN), name)
|
||||
else:
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def finish(self, results, name):
|
||||
self.set_status(DOWNLOAD_STOPPED_CODE, name)
|
||||
log.info("Finished downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
|
||||
self.download_path)
|
||||
safe_stop_looping_call(self.checker)
|
||||
status = yield self.downloader.status()
|
||||
self._check_status(status)
|
||||
defer.returnValue(self.download_path)
|
||||
return defer.succeed(self.download_path)
|
||||
|
||||
def fail(self, err):
|
||||
safe_stop_looping_call(self.checker)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _initialize(self, stream_info):
|
||||
# Set sd_hash and return key_fee from stream_info
|
||||
self.sd_hash = stream_info.source_hash
|
||||
self.sd_hash = stream_info.source_hash.decode()
|
||||
key_fee = None
|
||||
if stream_info.has_fee:
|
||||
key_fee = yield self.check_fee_and_convert(stream_info.source_fee)
|
||||
@@ -174,17 +152,21 @@ class GetStream(object):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _download_sd_blob(self):
|
||||
sd_blob = yield download_sd_blob(self.session, self.sd_hash,
|
||||
self.payment_rate_manager, self.timeout)
|
||||
sd_blob = yield download_sd_blob(
|
||||
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, conf.settings['download_mirrors']
|
||||
)
|
||||
defer.returnValue(sd_blob)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
|
||||
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
|
||||
yield self.pay_key_fee(key_fee, name)
|
||||
yield self.session.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
|
||||
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
|
||||
yield self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
|
||||
self.finished_deferred = self.downloader.start()
|
||||
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
|
||||
self.data_downloading_deferred.callback
|
||||
)
|
||||
self.finished_deferred.addCallbacks(lambda result: self.finish(result, name), self.fail)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -200,16 +182,19 @@ class GetStream(object):
|
||||
"""
|
||||
self.set_status(INITIALIZING_CODE, name)
|
||||
key_fee = yield self._initialize(stream_info)
|
||||
|
||||
safe_start_looping_call(self.checker, 1)
|
||||
self.set_status(DOWNLOAD_METADATA_CODE, name)
|
||||
try:
|
||||
sd_blob = yield self._download_sd_blob()
|
||||
yield self._download(sd_blob, name, key_fee, txid, nout, file_name)
|
||||
self.set_status(DOWNLOAD_RUNNING_CODE, name)
|
||||
yield self.data_downloading_deferred
|
||||
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
|
||||
self.data_downloading_deferred.addTimeout(self.timeout, self.reactor)
|
||||
try:
|
||||
yield self.data_downloading_deferred
|
||||
self.wrote_data = True
|
||||
except defer.TimeoutError:
|
||||
raise DownloadDataTimeout("data download timed out")
|
||||
except (DownloadDataTimeout, InvalidStreamDescriptorError) as err:
|
||||
safe_stop_looping_call(self.checker)
|
||||
raise err
|
||||
|
||||
defer.returnValue((self.downloader, self.finished_deferred))
|
||||
@@ -6,16 +6,16 @@ import treq
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbrynet.core.Error import InvalidExchangeRateResponse
|
||||
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CURRENCY_PAIRS = ["USDBTC", "BTCLBC"]
|
||||
BITTREX_FEE = 0.0025
|
||||
COINBASE_FEE = 0.0 #add fee
|
||||
COINBASE_FEE = 0.0 # add fee
|
||||
|
||||
|
||||
class ExchangeRate(object):
|
||||
class ExchangeRate:
|
||||
def __init__(self, market, spot, ts):
|
||||
if not int(time.time()) - ts < 600:
|
||||
raise ValueError('The timestamp is too dated.')
|
||||
@@ -34,9 +34,10 @@ class ExchangeRate(object):
|
||||
return {'spot': self.spot, 'ts': self.ts}
|
||||
|
||||
|
||||
class MarketFeed(object):
|
||||
class MarketFeed:
|
||||
REQUESTS_TIMEOUT = 20
|
||||
EXCHANGE_RATE_UPDATE_RATE_SEC = 300
|
||||
|
||||
def __init__(self, market, name, url, params, fee):
|
||||
self.market = market
|
||||
self.name = name
|
||||
@@ -95,8 +96,7 @@ class MarketFeed(object):
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
super().__init__(
|
||||
"BTCLBC",
|
||||
"Bittrex",
|
||||
"https://bittrex.com/api/v1.1/public/getmarkethistory",
|
||||
@@ -115,14 +115,13 @@ class BittrexFeed(MarketFeed):
|
||||
qtys = sum([i['Quantity'] for i in trades])
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponse(self.market, 'quantities were not positive')
|
||||
vwap = totals/qtys
|
||||
vwap = totals / qtys
|
||||
return defer.succeed(float(1.0 / vwap))
|
||||
|
||||
|
||||
class LBRYioFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
super().__init__(
|
||||
"BTCLBC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
@@ -139,8 +138,7 @@ class LBRYioFeed(MarketFeed):
|
||||
|
||||
class LBRYioBTCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
super().__init__(
|
||||
"USDBTC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
@@ -160,8 +158,7 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
|
||||
class CryptonatorBTCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
super().__init__(
|
||||
"USDBTC",
|
||||
"cryptonator.com",
|
||||
"https://api.cryptonator.com/api/ticker/usd-btc",
|
||||
@@ -175,16 +172,14 @@ class CryptonatorBTCFeed(MarketFeed):
|
||||
except ValueError:
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response")
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
super().__init__(
|
||||
"BTCLBC",
|
||||
"cryptonator.com",
|
||||
"https://api.cryptonator.com/api/ticker/btc-lbc",
|
||||
@@ -198,12 +193,12 @@ class CryptonatorFeed(MarketFeed):
|
||||
except ValueError:
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response")
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
|
||||
|
||||
class ExchangeRateManager(object):
|
||||
class ExchangeRateManager:
|
||||
def __init__(self):
|
||||
self.market_feeds = [
|
||||
LBRYioBTCFeed(),
|
||||
@@ -231,15 +226,15 @@ class ExchangeRateManager(object):
|
||||
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return amount * market.rate.spot
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
raise Exception(
|
||||
'Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency))
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
def fee_dict(self):
|
||||
return {market: market.rate.as_dict() for market in self.market_feeds}
|
||||