Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
574c2ce794 | ||
|
|
7f5541eb5c | ||
|
|
7b49241c38 | ||
|
|
da716b2e3d | ||
|
|
a144fae8b2 | ||
|
|
0d90b82287 | ||
|
|
567c23e10f | ||
|
|
16970b7d37 | ||
|
|
8593cd362a | ||
|
|
2dbf96da81 | ||
|
|
83c7797187 | ||
|
|
6606deafa5 | ||
|
|
19b0d59159 | ||
|
|
9ccb3fa2a3 | ||
|
|
48533df523 | ||
|
|
1ae1137005 | ||
|
|
d5f2832b2f | ||
|
|
d0230b4893 | ||
|
|
a6cd53b97c | ||
|
|
7a28171a72 | ||
|
|
150bcb1116 | ||
|
|
6fe9c5bf00 | ||
|
|
3a86fc6680 | ||
|
|
2805cbb331 | ||
|
|
68d7a7014c | ||
|
|
883fa267a5 | ||
|
|
5f2c3ac965 | ||
|
|
bf8e32ebd2 | ||
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 | ||
|
|
6aa709e34e | ||
|
|
e73f121c08 | ||
|
|
3b9d6c4f8e | ||
|
|
69b60c19c3 | ||
|
|
04278cb1ab | ||
|
|
e8dec5b440 | ||
|
|
75f82b107e | ||
|
|
0e90c4b611 | ||
|
|
fb66b68bc8 | ||
|
|
c601677386 | ||
|
|
1fbd1cd04e | ||
|
|
4a71b91fbf | ||
|
|
b8b0d8b1ac | ||
|
|
8ef73d3985 | ||
|
|
05e556a5e3 | ||
|
|
d1da42bea8 | ||
|
|
90685e8357 | ||
|
|
a471d5d167 | ||
|
|
16122299e4 | ||
|
|
9f18da4bae | ||
|
|
1abce52867 | ||
|
|
072e2b65d3 | ||
|
|
f6d11e5467 | ||
|
|
458bddcb65 | ||
|
|
5fad43ff69 | ||
|
|
6992d0d947 | ||
|
|
d1b0180437 | ||
|
|
ba59e126cd | ||
|
|
4e3467bb7e | ||
|
|
4d4c705eb9 | ||
|
|
ce7b3184c4 | ||
|
|
b71faddaac | ||
|
|
1328cb4cce | ||
|
|
946488bce9 | ||
|
|
7713881e71 | ||
|
|
26fcc48c5a | ||
|
|
bf12fb68a2 | ||
|
|
3a4c9ff4f9 | ||
|
|
6180a2917d | ||
|
|
014cbb4623 | ||
|
|
ee98251d79 | ||
|
|
e3d08ec773 | ||
|
|
f03e68e63e | ||
|
|
72090b45dd | ||
|
|
93f9622798 | ||
|
|
16ff4b4e2c | ||
|
|
c90cca4b20 | ||
|
|
98d83ea74f | ||
|
|
a234f92d2c | ||
|
|
8ba3575901 | ||
|
|
a857c83c59 | ||
|
|
a6128ac8a2 | ||
|
|
2693301ad0 | ||
|
|
f7c7cb9535 | ||
|
|
b36a5f9b6b | ||
|
|
4826eb9969 | ||
|
|
c34b150794 | ||
|
|
c4c7670a7c | ||
|
|
0918bac86e | ||
|
|
a60a921706 | ||
|
|
b3ee29b66d | ||
|
|
3a1a9b988c | ||
|
|
12f2bebc46 | ||
|
|
31a3812c8a | ||
|
|
70ec08e30b | ||
|
|
23e8f0d849 | ||
|
|
1cdfcd5b4e | ||
|
|
dc2786fbc0 | ||
|
|
7e39b0c4fe | ||
|
|
4af830c405 | ||
|
|
fe844581fd | ||
|
|
443ce6352b | ||
|
|
158c354a27 | ||
|
|
92d1230a90 | ||
|
|
d64916a061 | ||
|
|
20a0568253 | ||
|
|
5a1f42ee54 | ||
|
|
ab95f14e21 | ||
|
|
bae2676b69 | ||
|
|
ea6869a63e | ||
|
|
953327c150 | ||
|
|
e01ad6bc85 | ||
|
|
47a33009ff | ||
|
|
bc844013a1 | ||
|
|
2fd2319cdb | ||
|
|
a50657c472 | ||
|
|
ef5f6e06ba | ||
|
|
977cf9f506 | ||
|
|
ffae10d11f | ||
|
|
221107e76e | ||
|
|
e74c78b607 | ||
|
|
434c7b57b1 | ||
|
|
9dc4608a62 | ||
|
|
fc69366092 | ||
|
|
9f0b1f3e25 | ||
|
|
3f03a845ec | ||
|
|
bc28976e12 | ||
|
|
8f826098f9 | ||
|
|
0a765e6207 | ||
|
|
f83aa61923 | ||
|
|
f9ff3e5d14 | ||
|
|
b3fde9d78d | ||
|
|
0c999083cc | ||
|
|
561215db8a | ||
|
|
8dbb7f2c78 | ||
|
|
6f7f3bb316 | ||
|
|
8d507c4b73 | ||
|
|
2beb0eda94 | ||
|
|
b7e07b72ca | ||
|
|
fde8718c8c | ||
|
|
b2d1bddcfe | ||
|
|
dd1d1be994 | ||
|
|
a93957178a | ||
|
|
5026d0b2f3 | ||
|
|
cc8b47d882 | ||
|
|
db0b5961b3 | ||
|
|
cb6f7ada42 | ||
|
|
59464099b4 | ||
|
|
aed00b53fd | ||
|
|
cdb0f97cb4 | ||
|
|
33bcc8256c | ||
|
|
0a8d44f12f | ||
|
|
1b74594f5b | ||
|
|
b0718d295e | ||
|
|
9bb91b126d | ||
|
|
6dbb00d8d5 | ||
|
|
3a27a4509a | ||
|
|
1d438f5d23 | ||
|
|
3840f19869 | ||
|
|
ec0483b975 | ||
|
|
2fba1f417f | ||
|
|
5b419b55c5 | ||
|
|
b31379e51c | ||
|
|
8394dcb663 | ||
|
|
457a56ce04 | ||
|
|
7c8016d807 | ||
|
|
d704336f20 | ||
|
|
1b4230b06e | ||
|
|
88095af1cd | ||
|
|
e9efd9f68a | ||
|
|
7b32e2ff7c | ||
|
|
572777305e | ||
|
|
7e27a6e37d | ||
|
|
b98621d7ea | ||
|
|
bc5824b8bc | ||
|
|
a6b643bb34 | ||
|
|
488b976734 | ||
|
|
253912b52e | ||
|
|
0bf8416d9f | ||
|
|
2e921437d8 | ||
|
|
b6f9b60683 | ||
|
|
57430e9f5f | ||
|
|
1e13fbee8d | ||
|
|
3d8b9e2b66 | ||
|
|
64898d43f1 | ||
|
|
b2117fddab | ||
|
|
5658623f29 | ||
|
|
934682bd0f | ||
|
|
daecc582f3 | ||
|
|
3cb4012dbf | ||
|
|
f7bd44a4f5 | ||
|
|
52317b05bb | ||
|
|
83c0baa3cb | ||
|
|
340c5addab | ||
|
|
e3fd94dbf0 | ||
|
|
923fcfb9e9 | ||
|
|
0ae5f1a4c6 | ||
|
|
90e06abba2 | ||
|
|
6ecfc94ff1 | ||
|
|
60c4a727d3 | ||
|
|
9fbb9f2e98 | ||
|
|
f1feec66c0 | ||
|
|
73b9b983b1 | ||
|
|
588779e3a4 | ||
|
|
b7ae446059 | ||
|
|
db898307b5 | ||
|
|
7e3512cbf2 | ||
|
|
cdd65c6783 | ||
|
|
320ad717cd | ||
|
|
264e6744e1 | ||
|
|
64e2b79c07 | ||
|
|
9255bb5abe | ||
|
|
d7525dacc3 | ||
|
|
5cb07100f8 | ||
|
|
171c080d33 | ||
|
|
cfc3ec2a05 | ||
|
|
1392b8f83b | ||
|
|
8b265801df | ||
|
|
6c4fb8dab1 | ||
|
|
c1a9cd93ee | ||
|
|
0d4e83e167 | ||
|
|
1292eb345d | ||
|
|
f4c6dc90c9 | ||
|
|
8c8f5d8f58 | ||
|
|
b6d986cb40 | ||
|
|
6cc134c7c8 | ||
|
|
eee964cfa0 | ||
|
|
a03c94bfad | ||
|
|
f1f0917063 | ||
|
|
fe2476b5aa | ||
|
|
9c5fa8bf76 | ||
|
|
cd43a71172 | ||
|
|
e2c6de76f4 | ||
|
|
05aeadc9fe | ||
|
|
7153417372 | ||
|
|
fceb2ae0a2 | ||
|
|
0cef2fdd82 | ||
|
|
9c91b135bd | ||
|
|
e3b127192c | ||
|
|
ee1738da6e | ||
|
|
8ca2b75c80 | ||
|
|
dc81b6def1 | ||
|
|
22f5aaee70 | ||
|
|
4bd6569c9c | ||
|
|
dbf39baeb1 | ||
|
|
3adf3c26f2 | ||
|
|
fa0d13e630 | ||
|
|
227a82d6d3 | ||
|
|
65cfc1f63c | ||
|
|
e46f0e2444 | ||
|
|
00bc7995c4 | ||
|
|
2dcebe48b7 | ||
|
|
d12b6cddb3 | ||
|
|
699c057f2d | ||
|
|
3ce9f31f4b | ||
|
|
485d5d55e8 | ||
|
|
6ee9f2513b | ||
|
|
81a6e14a5c | ||
|
|
496ffe0096 | ||
|
|
216c760179 | ||
|
|
01eca3004f | ||
|
|
2f47a3758e | ||
|
|
73bcc1796f | ||
|
|
e03e41ad31 | ||
|
|
6cb30f8e05 | ||
|
|
d9b1be6214 | ||
|
|
5f2f06c269 | ||
|
|
e1dcdcf4db | ||
|
|
416472c5d8 | ||
|
|
196f3ba92d | ||
|
|
6feae93c58 | ||
|
|
203d30d4f7 | ||
|
|
d0a7ca841b | ||
|
|
2e6c848c39 | ||
|
|
18a0204105 | ||
|
|
01baa193d5 | ||
|
|
ba57af5502 | ||
|
|
8139f93ff1 | ||
|
|
4596d5c26f | ||
|
|
810a63782d | ||
|
|
ea83827d57 | ||
|
|
e19f8f3d95 | ||
|
|
7af501e0bc | ||
|
|
a692b6ff7d | ||
|
|
3364ac0976 | ||
|
|
cf1157f02a | ||
|
|
0ed56bbe1a | ||
|
|
213bdac55f | ||
|
|
6d8ab97ca9 | ||
|
|
fb34476cd7 | ||
|
|
07d353a947 | ||
|
|
3c5fb6490d | ||
|
|
a5d21081b9 | ||
|
|
ed98a3cc21 | ||
|
|
45c2cd2986 | ||
|
|
75f21950da | ||
|
|
3f704be85c | ||
|
|
67a15ea788 | ||
|
|
55bcd2dfee | ||
|
|
d9757fdae8 | ||
|
|
a4ae8d7c53 | ||
|
|
78078b4c9f | ||
|
|
1331e2bbe6 | ||
|
|
4c4d652f1b | ||
|
|
73bd76b631 | ||
|
|
59a0f9f602 | ||
|
|
1b1847bc31 | ||
|
|
21a569edcc | ||
|
|
2ddf1a08f6 | ||
|
|
8754c6fabe | ||
|
|
f049109a38 | ||
|
|
8db7c15aa1 | ||
|
|
90e0694d2e | ||
|
|
4748cbf654 | ||
|
|
74eb991983 | ||
|
|
e11bae2dda | ||
|
|
8462f56b13 | ||
|
|
0bec3c7294 | ||
|
|
2bb4052193 | ||
|
|
101a3f2b22 | ||
|
|
04a2bde8dd | ||
|
|
a1368b05da | ||
|
|
99ea7a26b0 | ||
|
|
8b3a457925 | ||
|
|
e02e2a249a | ||
|
|
3019a1c222 | ||
|
|
1f3a19c100 | ||
|
|
5eac4247e1 | ||
|
|
1ca82b9d52 | ||
|
|
21702ec43c | ||
|
|
d2083bc17e | ||
|
|
52edf880e0 | ||
|
|
1f45af991f | ||
|
|
8afdd0b2e9 | ||
|
|
99efc70ba3 | ||
|
|
64e306801d | ||
|
|
330db61b36 | ||
|
|
357763ff61 | ||
|
|
8a6e6b4d77 | ||
|
|
64913c4288 | ||
|
|
f01ad0e4bd | ||
|
|
d9fcc458dc | ||
|
|
24a11d3f21 | ||
|
|
234eaf8890 | ||
|
|
9e2e53147e | ||
|
|
fc41af5889 | ||
|
|
72ce0cd0b0 | ||
|
|
e4497c1327 | ||
|
|
a1bf832ea6 | ||
|
|
d4e76500fb | ||
|
|
f898bb3d0e | ||
|
|
ad13282494 | ||
|
|
73519f8c6a | ||
|
|
b2c95f35c7 | ||
|
|
0bfccb86ff | ||
|
|
5118240897 | ||
|
|
427fc179ac | ||
|
|
55bcc520e4 | ||
|
|
bc8251696e | ||
|
|
677ba25ccd | ||
|
|
95436b233c | ||
|
|
82871b8f6b | ||
|
|
9525b87224 | ||
|
|
f06098978c | ||
|
|
141491f2da | ||
|
|
38006f7d29 | ||
|
|
3664c25d98 | ||
|
|
1747c9d866 | ||
|
|
1066fbaa7c | ||
|
|
7f7961f4cc | ||
|
|
9d9916548b | ||
|
|
ada5b55f22 | ||
|
|
a2f28e7788 | ||
|
|
c6092a4c79 | ||
|
|
315a557019 | ||
|
|
20221e1186 | ||
|
|
50003d2600 | ||
|
|
ea6b2b98fb | ||
|
|
4890fdfb50 | ||
|
|
2f95c3a9d1 | ||
|
|
6039159ecc | ||
|
|
8e4d8345bd | ||
|
|
22df26eeb6 | ||
|
|
7b9ff3e8b5 | ||
|
|
1a5b2c08ee | ||
|
|
b6babb1c9d | ||
|
|
a88485c5a0 | ||
|
|
0cefa07e97 | ||
|
|
ca9d48d574 | ||
|
|
e10e1de92f | ||
|
|
bdf6887791 | ||
|
|
295f2c1115 | ||
|
|
78fcf6a44a | ||
|
|
7dcebf99e0 | ||
|
|
463cbde4b1 | ||
|
|
1adce8b227 | ||
|
|
673d259d7a | ||
|
|
d0b739a0b9 | ||
|
|
b4aa51444c | ||
|
|
9178ca701c | ||
|
|
7eb9f344f4 | ||
|
|
9f8d3e69ae | ||
|
|
e1130147b5 | ||
|
|
0fb6f50aba | ||
|
|
91fc772e3c | ||
|
|
30b3d31af0 | ||
|
|
1a9f81ea14 | ||
|
|
5813df74a5 | ||
|
|
dc6d803fed | ||
|
|
f9cba69d03 | ||
|
|
17f9d7cfb2 | ||
|
|
42c43e86ef | ||
|
|
8cd230eb6e | ||
|
|
dcc71713c3 | ||
|
|
3d72b2adf3 | ||
|
|
655ece18de | ||
|
|
cda14f7c69 | ||
|
|
3d36e36173 | ||
|
|
2dd0221711 | ||
|
|
1fec9f52ef | ||
|
|
ca9ca09a00 | ||
|
|
b16c36b55f | ||
|
|
1c2849235c | ||
|
|
202a957a1a | ||
|
|
66f33202c7 | ||
|
|
f310bb6825 | ||
|
|
1fb7ffd759 | ||
|
|
3fcca4bc47 | ||
|
|
12c73cf50c | ||
|
|
02d398613d | ||
|
|
65e13898c2 | ||
|
|
559afa465a | ||
|
|
c753a5825a | ||
|
|
4185a3b9e3 | ||
|
|
2f4e6911d6 | ||
|
|
bcf7218b66 | ||
|
|
e21ade7360 | ||
|
|
53d4c149bc | ||
|
|
0adcaa6a5d | ||
|
|
ce4d734250 | ||
|
|
04e0933d36 | ||
|
|
bcf52c3f24 | ||
|
|
a702afd010 | ||
|
|
ad492a09f7 | ||
|
|
790991ecad | ||
|
|
f5789775a2 | ||
|
|
5cc0d365c2 | ||
|
|
5430a0f289 | ||
|
|
cc7656faa4 | ||
|
|
03415415ed | ||
|
|
1e47ff5a83 | ||
|
|
88bad1f868 | ||
|
|
91405ef232 | ||
|
|
570c1163c1 | ||
|
|
17827036ce | ||
|
|
338e6c2116 | ||
|
|
cdf6155102 | ||
|
|
e47ef741f7 | ||
|
|
2878716381 | ||
|
|
6d9947779e | ||
|
|
c7e5bdd3ae | ||
|
|
46dcf8a899 | ||
|
|
c050231ac5 | ||
|
|
ee5be5adc8 | ||
|
|
d99ce5f3d6 | ||
|
|
e7634b74c8 | ||
|
|
26af01a5cc | ||
|
|
36989f1974 | ||
|
|
1d40ec34a2 | ||
|
|
f0f30fd75c | ||
|
|
be2ef747dc | ||
|
|
268a01dc23 | ||
|
|
8399f2ab33 | ||
|
|
7698ab7053 | ||
|
|
eff66bdf80 | ||
|
|
70e68c6361 | ||
|
|
2a53afbfd8 | ||
|
|
a42bf67800 | ||
|
|
b783b4daed | ||
|
|
c38b463bf6 | ||
|
|
8696cd3c27 | ||
|
|
0abbc20899 | ||
|
|
2fca29144e | ||
|
|
1675cc4580 | ||
|
|
93d134f0ce | ||
|
|
1c6ce49b7b | ||
|
|
a7f34e2913 | ||
|
|
2a285d139a | ||
|
|
5e371e8e6a | ||
|
|
fdc98fc746 | ||
|
|
0d8e55bc95 | ||
|
|
69851a6588 | ||
|
|
b101fafd39 | ||
|
|
d2aaf7bd09 | ||
|
|
5a1f551ce5 | ||
|
|
584433e16c | ||
|
|
0c9c6c2705 | ||
|
|
afae649a9e | ||
|
|
2238e1f802 | ||
|
|
518c447fef | ||
|
|
cb9b348828 | ||
|
|
eab95a6246 | ||
|
|
593d0046bd | ||
|
|
c5d1478c02 | ||
|
|
1c57293e9c | ||
|
|
14a067a4c4 | ||
|
|
707ab831fd | ||
|
|
ceb25f917a | ||
|
|
002faf7a58 | ||
|
|
d1bbb7af74 | ||
|
|
ea603dce13 | ||
|
|
f2a768c9c4 | ||
|
|
12384a89a1 | ||
|
|
c21956b4f3 | ||
|
|
4a8776be10 | ||
|
|
d6d69a46d4 | ||
|
|
4384a5f022 | ||
|
|
aa83bbd637 | ||
|
|
daae0b2836 | ||
|
|
5afd446bb3 | ||
|
|
2b5e3204c0 | ||
|
|
84c91c480f | ||
|
|
cbc98e72a7 | ||
|
|
d1f00255c1 | ||
|
|
434d98c6a6 | ||
|
|
e51c98ca8c | ||
|
|
f719aa1db0 | ||
|
|
3686b1d970 | ||
|
|
e6fd8cc0f2 | ||
|
|
8739dc5d2c | ||
|
|
d6c04d3586 | ||
|
|
7236203f1e | ||
|
|
8c6c442fdd | ||
|
|
49c659b832 | ||
|
|
2590523fa1 | ||
|
|
08b4fd6942 | ||
|
|
6a8963d807 | ||
|
|
083bf4f61d | ||
|
|
ab3b0134b5 | ||
|
|
3b88d2465d | ||
|
|
0841c90e6c | ||
|
|
3f6e928cc2 | ||
|
|
a32a0c6401 | ||
|
|
95cb29d3c8 | ||
|
|
e85f8b245b | ||
|
|
04836ea0d9 | ||
|
|
451823f33e | ||
|
|
8c3552dd34 | ||
|
|
a204f0d3e6 | ||
|
|
892758be65 | ||
|
|
3594c8976d | ||
|
|
23ede44bff | ||
|
|
0006a68bf2 | ||
|
|
0349e68201 | ||
|
|
be248dc448 | ||
|
|
d0d5d0340d | ||
|
|
f79a49bbf4 | ||
|
|
6a5d88a0d5 | ||
|
|
4165881d5f | ||
|
|
8dc4e3be43 | ||
|
|
0badea874d | ||
|
|
0dd6193eaa | ||
|
|
d10778430f | ||
|
|
da8e09846d | ||
|
|
310fe4a42c | ||
|
|
a7ef8889dd | ||
|
|
f41229cb5b | ||
|
|
8ab4e3ca49 | ||
|
|
ff8d37443e | ||
|
|
fcd46629c4 | ||
|
|
9ad9eb083b | ||
|
|
4669507880 | ||
|
|
9348f4f366 | ||
|
|
47bb634035 | ||
|
|
10b34d6b33 | ||
|
|
f061ca2b15 | ||
|
|
bc24dbea29 | ||
|
|
2d4bf73632 | ||
|
|
c21d8cc8dc | ||
|
|
cc6a91a477 | ||
|
|
21a2725215 | ||
|
|
411b8c74cc | ||
|
|
91bf312e4d | ||
|
|
277b8b122c | ||
|
|
59c49d2d31 | ||
|
|
16024c95aa | ||
|
|
923ed80fcf | ||
|
|
2ee2916f78 | ||
|
|
79bf1b427f | ||
|
|
c21c329aa0 | ||
|
|
6b3af4bc21 | ||
|
|
ffbdf03847 | ||
|
|
8d08bfb3f9 | ||
|
|
1af6326b97 | ||
|
|
31630a84be | ||
|
|
0601bf3197 | ||
|
|
826d2866fa | ||
|
|
ba00498d83 | ||
|
|
4da40e8d52 | ||
|
|
d54d989c40 | ||
|
|
18ecbba87c | ||
|
|
9b417486a9 | ||
|
|
6d208ac496 | ||
|
|
8990c145fb | ||
|
|
e5b1c3c829 | ||
|
|
6c7128c0f6 | ||
|
|
f31f1fa40b | ||
|
|
2ceba79b9f | ||
|
|
1c8a59c1a9 | ||
|
|
298bf91293 | ||
|
|
b25d592d99 | ||
|
|
875edb5f76 | ||
|
|
84d97ab323 | ||
|
|
aeaffd620e | ||
|
|
3f7091cbf2 | ||
|
|
a95f791215 | ||
|
|
af2aeaa66b | ||
|
|
855fd8bf9a | ||
|
|
a937aff80f | ||
|
|
5597d45aed | ||
|
|
473d2eabfa | ||
|
|
78c8c8e64d | ||
|
|
7335d012ef | ||
|
|
99be38604a | ||
|
|
d35d340613 | ||
|
|
f05ca137be | ||
|
|
e718caca77 | ||
|
|
693a3346d2 | ||
|
|
1f75b6a49e | ||
|
|
4ece422f48 | ||
|
|
607677f98f | ||
|
|
e1e7be63b8 | ||
|
|
e1314a9d1e | ||
|
|
1ee682f06f | ||
|
|
ba80c0e594 | ||
|
|
fbdbcc8070 | ||
|
|
24a872885a | ||
|
|
005a131440 | ||
|
|
0b1cb16353 | ||
|
|
1a3eaf7e6a | ||
|
|
971252d5d1 | ||
|
|
c5ba9b263e | ||
|
|
c01716a6c0 | ||
|
|
0fbf131df3 | ||
|
|
4bd0806e58 | ||
|
|
4cbb71705e | ||
|
|
8ab4dd6b2c | ||
|
|
318c369752 | ||
|
|
cea3b7630c | ||
|
|
19211d4417 | ||
|
|
c312d1b3a6 | ||
|
|
e4ea1ccbfb | ||
|
|
5b35c4e8f0 | ||
|
|
9967857a57 | ||
|
|
78c560a3be | ||
|
|
07a12b66e8 | ||
|
|
a28c9d09c8 | ||
|
|
b2ab13187f | ||
|
|
5ec696fbfd | ||
|
|
44bd0ae847 | ||
|
|
5809ba093d | ||
|
|
17aee92dc9 | ||
|
|
6f1d2e7690 | ||
|
|
75f57fc517 | ||
|
|
f589410496 | ||
|
|
aecc7c6643 | ||
|
|
c5c1b62939 | ||
|
|
adc92a2b52 | ||
|
|
9636ca22e7 | ||
|
|
0442770c9d | ||
|
|
3b6a879fc9 | ||
|
|
a1b4c9acd6 | ||
|
|
0f90dee224 | ||
|
|
159e84468c | ||
|
|
59e4ac30c2 | ||
|
|
b5ce948bc1 | ||
|
|
076af7ef43 | ||
|
|
43bef9447c | ||
|
|
b1c5fe0b4d | ||
|
|
39d7f2e46e | ||
|
|
315661208d | ||
|
|
e175e43082 | ||
|
|
cb9b320ed7 | ||
|
|
1d906fb0c7 | ||
|
|
c544e26206 | ||
|
|
1eaac35a3e | ||
|
|
cc7962fc2e | ||
|
|
3638984d67 | ||
|
|
62e77c69f5 | ||
|
|
774d813576 | ||
|
|
688caf4453 | ||
|
|
eeb5b3c139 | ||
|
|
b5e6dd1060 | ||
|
|
272d818050 | ||
|
|
0fc1bd7a8c | ||
|
|
2b234e0713 | ||
|
|
9e8cb17ecb | ||
|
|
e0230864ce | ||
|
|
9e89b3e40e | ||
|
|
036663ae62 | ||
|
|
b62321689d | ||
|
|
bdd271e78f | ||
|
|
4a56d38782 | ||
|
|
70a7ca95fe | ||
|
|
f85e61d8ed | ||
|
|
ac8c641d64 | ||
|
|
d81502e191 | ||
|
|
8c35d355e5 | ||
|
|
72b004664f | ||
|
|
49a7408968 | ||
|
|
17738ad248 | ||
|
|
425c952293 | ||
|
|
03d2d0e237 | ||
|
|
edfc2c6124 | ||
|
|
545451a829 | ||
|
|
12ff7015cd | ||
|
|
f2f212edba | ||
|
|
69446491b8 | ||
|
|
d5beaa0937 | ||
|
|
cc970c499b | ||
|
|
0087b16912 | ||
|
|
5520d518b5 | ||
|
|
b3e9240aa8 | ||
|
|
137a270488 | ||
|
|
e36610d63c | ||
|
|
c4273fbef0 | ||
|
|
a4e9f3100f | ||
|
|
2668ceac0e | ||
|
|
fb2fb5c38c | ||
|
|
56175df121 | ||
|
|
7d3daa9fe6 | ||
|
|
1d2de27536 | ||
|
|
aab42e56bd | ||
|
|
1368a6e074 | ||
|
|
80990b959c | ||
|
|
053b6d1c80 | ||
|
|
692d8e8e1a | ||
|
|
5e71dcbaf0 | ||
|
|
83958604d5 | ||
|
|
dcd8a6bb0e | ||
|
|
7161a0edb9 | ||
|
|
0fd160e6e6 | ||
|
|
ca8b2dd83e | ||
|
|
79d1da5ff8 | ||
|
|
d73252b11a | ||
|
|
1f5c56cac3 | ||
|
|
18db2aa970 | ||
|
|
a7a53f9381 | ||
|
|
f4b2a05fff | ||
|
|
cdd2b6bfc1 | ||
|
|
26bd27cd32 | ||
|
|
613feb0808 | ||
|
|
d4849443e5 | ||
|
|
db70f7e98a | ||
|
|
37ddaeca12 | ||
|
|
2c546b2cde | ||
|
|
c48d195951 | ||
|
|
149fa625e0 | ||
|
|
16cda60f31 |
No files matched your search
@@ -1,48 +0,0 @@
|
||||
version: 1.0.{build}
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
|
||||
environment:
|
||||
GH_TOKEN:
|
||||
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
|
||||
key_pass:
|
||||
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
|
||||
pfx_key:
|
||||
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
|
||||
AWS_ACCESS_KEY_ID:
|
||||
secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
|
||||
|
||||
#notifications:
|
||||
# - provider: Slack
|
||||
# incoming_webhook:
|
||||
# secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
|
||||
|
||||
|
||||
clone_folder: c:\projects\lbry
|
||||
|
||||
test_script:
|
||||
- cd C:\projects\lbry\
|
||||
- pip install cython
|
||||
- pip install mock pylint unqlite Faker
|
||||
- pip install .
|
||||
- pylint lbrynet
|
||||
# disable tests for now so that appveyor can build the app
|
||||
- set PYTHONPATH=.
|
||||
- trial lbrynet.tests
|
||||
# TODO: integration tests do not work
|
||||
#- python -m unittest discover tests/integration
|
||||
#- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
|
||||
|
||||
build_script:
|
||||
- cd C:\projects\lbry\build\
|
||||
- ps: .\build.ps1
|
||||
|
||||
|
||||
artifacts:
|
||||
- path: build\dist\*.zip
|
||||
name: lbrynet-daemon
|
||||
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
@@ -51,3 +51,24 @@ Tell us what happens instead
|
||||
|
||||
## Screenshots
|
||||
<!-- If a screenshot would help explain the bug, please include one or two here -->
|
||||
|
||||
|
||||
|
||||
## Internal Use
|
||||
|
||||
### Acceptance Criteria
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
### Definition of Done
|
||||
- [ ] Tested against acceptance criteria
|
||||
- [ ] Tested against the assumptions of user story
|
||||
- [ ] The project builds without errors
|
||||
- [ ] Unit tests are written and passing
|
||||
- [ ] Tests on devices/browsers listed in the issue have passed
|
||||
- [ ] QA performed & issues resolved
|
||||
- [ ] Refactoring completed
|
||||
- [ ] Any configuration or build changes documented
|
||||
- [ ] Documentation updated
|
||||
- [ ] Peer Code Review performed
|
||||
@@ -0,0 +1,38 @@
|
||||
## PR Checklist
|
||||
Please check all that apply to this PR using "x":
|
||||
|
||||
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
|
||||
- [ ] I have checked that this PR does not introduce a breaking change
|
||||
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
|
||||
|
||||
|
||||
## PR Type
|
||||
What kind of change does this PR introduce?
|
||||
|
||||
Why is this change necessary?
|
||||
|
||||
<!-- Please check all that apply to this PR using "x". -->
|
||||
|
||||
- [ ] Bugfix
|
||||
- [ ] Feature
|
||||
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
|
||||
- [ ] Code style update (formatting)
|
||||
- [ ] Refactoring (no functional changes)
|
||||
- [ ] Documentation changes
|
||||
- [ ] Other - Please describe:
|
||||
|
||||
## Fixes
|
||||
|
||||
Issue Number: N/A
|
||||
|
||||
|
||||
## What is the current behavior?
|
||||
|
||||
|
||||
## What is the new behavior?
|
||||
|
||||
|
||||
## Other information
|
||||
|
||||
|
||||
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
|
||||
@@ -1,25 +1,9 @@
|
||||
*.pyc
|
||||
*.egg
|
||||
*.so
|
||||
*.xml
|
||||
*.iml
|
||||
*.log
|
||||
*.pem
|
||||
*.decTest
|
||||
*.prof
|
||||
.#*
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
|
||||
/build/build
|
||||
/build/dist
|
||||
/bulid/requirements_base.txt
|
||||
|
||||
/lbrynet.egg-info
|
||||
/docs_build
|
||||
/lbry-venv
|
||||
|
||||
.idea/
|
||||
.coverage
|
||||
.DS_Store
|
||||
|
||||
# temporary files from the twisted.trial test runner
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
ignore=CVS,schema
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
@@ -62,7 +62,7 @@ confidence=
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# disable everything first and then re-enable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
@@ -121,7 +121,11 @@ disable=
|
||||
unidiomatic-typecheck,
|
||||
global-at-module-level,
|
||||
inconsistent-return-statements,
|
||||
keyword-arg-before-vararg
|
||||
keyword-arg-before-vararg,
|
||||
assignment-from-no-return,
|
||||
useless-return,
|
||||
assignment-from-none,
|
||||
stop-iteration-return
|
||||
|
||||
|
||||
[REPORTS]
|
||||
@@ -386,7 +390,7 @@ int-import-graph=
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
max-args=10
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
@@ -405,7 +409,7 @@ max-branches=12
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
max-parents=8
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
@@ -1,42 +1,130 @@
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
dist: xenial
|
||||
language: python
|
||||
python: 2.7
|
||||
python: "3.7"
|
||||
|
||||
branches:
|
||||
except:
|
||||
- gh-pages
|
||||
jobs:
|
||||
include:
|
||||
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
- &tests
|
||||
stage: test
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .[test]
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
|
||||
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
|
||||
script: tox
|
||||
after_success:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
services:
|
||||
- docker
|
||||
install:
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-windows.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-windows.zip
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- &build
|
||||
name: "Linux"
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- python scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
|
||||
- ./dist/lbrynet --version
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- <<: *build
|
||||
name: "Mac"
|
||||
os: osx
|
||||
osx_image: xcode8.3
|
||||
language: generic
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
addons:
|
||||
#srcclr:
|
||||
# debug: true
|
||||
apt:
|
||||
packages:
|
||||
- libgmp3-dev
|
||||
- build-essential
|
||||
- git
|
||||
- libssl-dev
|
||||
- libffi-dev
|
||||
|
||||
before_install:
|
||||
- virtualenv venv
|
||||
- source venv/bin/activate
|
||||
|
||||
install:
|
||||
- pip install -U pip==9.0.3
|
||||
- pip install -r requirements.txt
|
||||
- pip install -r requirements_testing.txt
|
||||
- pip install .
|
||||
|
||||
script:
|
||||
- pip install mock pylint
|
||||
- pylint lbrynet
|
||||
- PYTHONPATH=. trial lbrynet.tests
|
||||
- rvm install ruby-2.3.1
|
||||
- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
- $TRAVIS_BUILD_DIR/.tox
|
||||
@@ -4,33 +4,131 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/) with
|
||||
regard to the json-rpc api. As we're currently pre-1.0 release, we
|
||||
can and probably will change functionality and break backwards compatability
|
||||
can and probably will change functionality and break backwards compatibility
|
||||
at anytime.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.30.1] - 2018-12-09
|
||||
|
||||
### API
|
||||
* add `aioupnp_version` to `upnp` field in the response to `status`
|
||||
* fixed uncaught claim signature errors
|
||||
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
|
||||
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
|
||||
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
|
||||
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
|
||||
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
|
||||
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
|
||||
|
||||
### Schema
|
||||
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
|
||||
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
|
||||
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
|
||||
|
||||
### Wallet
|
||||
* fixed resolving names with invalid channel signatures
|
||||
* fixed wallet sync issues
|
||||
* fixed wallet locking/encryption issues
|
||||
|
||||
### DHT
|
||||
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
|
||||
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
|
||||
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
|
||||
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
|
||||
|
||||
### Database
|
||||
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
|
||||
|
||||
### Misc
|
||||
* switch to Python 3.7
|
||||
* brought back icon for windows binary
|
||||
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
|
||||
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
|
||||
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
|
||||
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
|
||||
|
||||
## [0.30.0] - 2018-10-22
|
||||
This release is the most ambitious upgrade to lbrynet yet; including a brand
|
||||
new wallet implementation, switch to Python 3 of the entire code base and
|
||||
many changes to the API such as pagination and a more consistent output from
|
||||
most commands.
|
||||
|
||||
|
||||
### Security
|
||||
*
|
||||
*
|
||||
* upgraded `cryptography` package.
|
||||
* switch to `coincurve` for private / public key generation (addresses)
|
||||
* support for connecting to daemon via SSL
|
||||
* improved workflow for encrypt/decrypt of wallet
|
||||
|
||||
### API
|
||||
* unified all command line executables into a single `lbrynet` executable.
|
||||
* added `use_https` configuration setting (off by default), if true the api
|
||||
server will generate a self signed ssl certificate when started and only
|
||||
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
|
||||
* deprecated `daemon_stop` command, use `stop` instead.
|
||||
* deprecated `wallet_balance` command, use `account_balance` instead.
|
||||
* deprecated `wallet_unlock` command, use `account_unlock` instead.
|
||||
* deprecated `wallet_decrypt` command, use `account_decrypt` instead.
|
||||
* deprecated `wallet_encrypt` command, use `account_encrypt` instead.
|
||||
* deprecated `wallet_prefill_addresses` command, use `account_fund` instead.
|
||||
* deprecated `wallet_list` command, use `address_list` instead.
|
||||
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
|
||||
* deprecated `wallet_public_key` command.
|
||||
* deprecated `wallet_new_address` command.
|
||||
* deprecated `wallet_unused_address` command, use `address_unused` instead.
|
||||
* deprecated `claim_renew` command.
|
||||
* added `account_list` command to list accounts including their balance.
|
||||
* added `account_add` command to add a previously created account from seed or private key.
|
||||
* added `account_create` command to generate a new account.
|
||||
* added `account_remove` command to remove an account from wallet.
|
||||
* added `account_set` command to change a setting on an account.
|
||||
* added `account_balance` command to get just the account balance.
|
||||
* added `account_unlock` command to unlock an encrypted account.
|
||||
* added `account_lock` command to lock an encrypted account.
|
||||
* added `account_encrypt` command to encrypt an account.
|
||||
* added `account_decrypt` command to decrypt an account.
|
||||
* added `account_fund` command to move funds between or within an account in various ways.
|
||||
* added `account_max_address_gap` command to find large gaps of unused addresses.
|
||||
* added `address_list` command to list addresses.
|
||||
* added `address_is_mine` command to check if an address is one of your addresses.
|
||||
* added `address_unused` command to get existing or generate a new unused address.
|
||||
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
|
||||
`transaction_list` and `utxo_list`.
|
||||
* added `upnp` field to `status` response
|
||||
* removed `send_amount_to_address` command previously marked as deprecated
|
||||
* removed `channel_list_mine` command previously marked as deprecated
|
||||
* removed `get_availability` command previously marked as deprecated
|
||||
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
|
||||
|
||||
### Wallet
|
||||
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
|
||||
* changed wallet file format to support multiple accounts in one wallet.
|
||||
* moved transaction data from wallet file into an sqlite database.
|
||||
* changed channel certificates to be keyed by txid:nout instead of claim_id which
|
||||
makes it possible to recover old certificates.
|
||||
|
||||
### File Manager
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### DHT
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### P2P
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### Reflector
|
||||
* No major changes to reflector.
|
||||
|
||||
### Database
|
||||
* No changes to existing storage module.
|
||||
|
||||
### Known Issues
|
||||
* Channel certificate import/export is currently not working.
|
||||
|
||||
## [0.21.2] - 2018-08-23
|
||||
### Fixed
|
||||
*
|
||||
*
|
||||
|
||||
### Deprecated
|
||||
*
|
||||
*
|
||||
|
||||
### Changed
|
||||
*
|
||||
*
|
||||
|
||||
### Added
|
||||
*
|
||||
*
|
||||
|
||||
### Removed
|
||||
*
|
||||
*
|
||||
* issue in dht ping queue where enqueued pings that aren't yet due wouldn't be rescheduled
|
||||
* blob mirror downloader not finishing streams that were partially uploaded at the time of the download attempt (https://github.com/lbryio/lbry/issues/1376)
|
||||
|
||||
|
||||
## [0.21.1] - 2018-08-13
|
||||
@@ -388,7 +486,7 @@ at anytime.
|
||||
* Fixed https://github.com/lbryio/lbry/issues/923
|
||||
* Fixed concurrent reflects opening too many files
|
||||
* Fixed cases when reflecting would fail on error conditions
|
||||
* Fixed deadlocks from occuring during blob writes
|
||||
* Fixed deadlocks from occurring during blob writes
|
||||
* Fixed and updated`lbrynet.tests.dht`
|
||||
* Fixed redundant dht id
|
||||
* Fixed dht `ping` method
|
||||
@@ -775,7 +873,7 @@ at anytime.
|
||||
* Removed check_pending logic from Daemon
|
||||
* Switched to txrequests so requests can use twisted event loop
|
||||
* Renamed API command file_seed to file_set_status
|
||||
* Dont add expected payment to wallet when payment rate is 0
|
||||
* Don't add expected payment to wallet when payment rate is 0
|
||||
### Fixed
|
||||
* Fix restart procedure in DaemonControl
|
||||
* Create download directory if it doesn't exist
|
||||
@@ -806,7 +904,7 @@ at anytime.
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
* Made dictionary key names in API commmand outputs to be more consistent
|
||||
* Made dictionary key names in API command outputs to be more consistent
|
||||
### Added
|
||||
* Add file filters: `claim_id`, `outpoint`, and `rowid`
|
||||
* Make loggly logs less verbose
|
||||
@@ -858,7 +956,7 @@ at anytime.
|
||||
* Include download url in version check
|
||||
|
||||
### Fixed
|
||||
* add misssing traceback to logging
|
||||
* add missing traceback to logging
|
||||
|
||||
## [0.8.3] - 2017-02-15
|
||||
### Fixed
|
||||
|
||||
@@ -1,70 +1,48 @@
|
||||
# Installing LBRY
|
||||
|
||||
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
|
||||
[](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, you'll need to install some additional libraries depending on your operating system.
|
||||
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
|
||||
### Python
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
LBRY requires Python 2.7. Python 3 is not supported.
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### OS X Prerequisites
|
||||
### macOS
|
||||
|
||||
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install mpfr libmpc protobuf
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
brew install python3 protobuf
|
||||
```
|
||||
|
||||
### Linux Prerequisites
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 16.04), install the following:
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux flavor, install the equivalent of the above packages for your system.
|
||||
|
||||
### Windows Prerequisites
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
|
||||
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- requests
|
||||
|
||||
Install the above binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -73,35 +51,27 @@ To install:
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
python --version # Python 3 is not supported. Make sure you're on Python 2.7
|
||||
|
||||
pip install -U -r requirements.txt
|
||||
pip install -U -r requirements_testing.txt
|
||||
pip install --editable .
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
## Run the tests
|
||||
To run the unit tests from the repo directory:
|
||||
```
|
||||
trial --reactor=asyncio tests.unit
|
||||
```
|
||||
|
||||
## Making a Change
|
||||
## Usage
|
||||
|
||||
If you are not already inside the virtual environment, run:
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
|
||||
```
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
This will install `lbrynet-daemon` in such a way that the changes you make to the code will be used as soon as you restart the daemon.
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,36 +1,35 @@
|
||||
# LBRY
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
[](https://coveralls.io/github/lbryio/lbry)
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules. LBRY also provides a data network that consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
This project aims to provide a daemon that can interact with the network via a json-rpc interface over HTTP.
|
||||
|
||||
The project is written in python2.7 and extensively uses Twisted framework.
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
| | Windows | macOS | Linux |
|
||||
| --------------------- | -------------------------------------------- | -------------------------------------------- | -------------------------------------------- |
|
||||
| Latest Stable Release | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.0/lbrynet-daemon-v0.20.0-windows.zip) | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.0/lbrynet-daemon-v0.20.0-macos.zip) | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.0/lbrynet-daemon-v0.20.0-linux.zip) |
|
||||
| Latest Pre-release | [Download](https://github.com/lbryio/lbry/releases/download/untagged-99114fa31abbfe3a5ef4/lbrynet-daemon-v0.20.1rc3-windows.zip) | [Download](https://github.com/lbryio/lbry/releases/download/untagged-99114fa31abbfe3a5ef4/lbrynet-daemon-v0.20.1rc3-macos.zip) | [Download](https://github.com/lbryio/lbry/releases/download/v0.20.1rc4/lbrynet-daemon-v0.20.1rc4-linux.zip) |
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows.
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
|
||||
## Usage
|
||||
|
||||
Run `lbrynet-daemon` to launch the daemon.
|
||||
Run `lbrynet start` to launch the API server.
|
||||
|
||||
By default, `lbrynet-daemon` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
|
||||
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
|
||||
|
||||
The full API is documented [here](https://lbryio.github.io/lbry/cli).
|
||||
With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
|
||||
## Running from source
|
||||
|
||||
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -42,12 +41,16 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. Our PGP key is here if you need it.
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is @jackrobison(jack@lbry.io)
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
The documentation for the api can be found [here](https://lbry.io/api).
|
||||
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
|
||||
@@ -1,33 +0,0 @@
|
||||
$env:Path += ";C:\MinGW\bin\"
|
||||
|
||||
$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
|
||||
gcc --version
|
||||
mingw32-make --version
|
||||
|
||||
# build/install miniupnpc manually
|
||||
tar zxf miniupnpc-1.9.tar.gz
|
||||
cd miniupnpc-1.9
|
||||
mingw32-make -f Makefile.mingw
|
||||
python setupmingw32.py build --compiler=mingw32
|
||||
python setupmingw32.py install
|
||||
cd ..\
|
||||
Remove-Item -Recurse -Force miniupnpc-1.9
|
||||
|
||||
# copy requirements from lbry, but remove miniupnpc (installed manually)
|
||||
Get-Content ..\requirements.txt | Select-String -Pattern 'miniupnpc' -NotMatch | Out-File requirements_base.txt
|
||||
|
||||
python set_build.py
|
||||
|
||||
pip install -r requirements.txt
|
||||
pip install ..\.
|
||||
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
pyinstaller -y console.onefile.spec
|
||||
|
||||
nuget install secure-file -ExcludeVersion
|
||||
secure-file\tools\secure-file -decrypt .\lbry2.pfx.enc -secret "$env:pfx_key"
|
||||
signtool.exe sign /f .\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
|
||||
|
||||
python zip_daemon.py
|
||||
python upload_assets.py
|
||||
@@ -1,59 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
|
||||
cd "$ROOT"
|
||||
BUILD_DIR="$ROOT/build"
|
||||
|
||||
FULL_BUILD="${FULL_BUILD:-false}"
|
||||
if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then
|
||||
FULL_BUILD="true"
|
||||
fi
|
||||
|
||||
[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build"
|
||||
[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# install dependencies
|
||||
$BUILD_DIR/prebuild.sh
|
||||
|
||||
VENV="$BUILD_DIR/venv"
|
||||
if [ -d "$VENV" ]; then
|
||||
rm -rf "$VENV"
|
||||
fi
|
||||
virtualenv "$VENV"
|
||||
set +u
|
||||
source "$VENV/bin/activate"
|
||||
set -u
|
||||
|
||||
# must set build before installing lbrynet. otherwise it has no effect
|
||||
python "$BUILD_DIR/set_build.py"
|
||||
fi
|
||||
|
||||
cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt"
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pip install -r requirements.txt
|
||||
)
|
||||
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
pyinstaller -y console.onefile.spec
|
||||
)
|
||||
|
||||
python "$BUILD_DIR/zip_daemon.py"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# electron-build has a publish feature, but I had a hard time getting
|
||||
# it to reliably work and it also seemed difficult to configure. Not proud of
|
||||
# this, but it seemed better to write my own.
|
||||
python "$BUILD_DIR/upload_assets.py"
|
||||
|
||||
deactivate
|
||||
fi
|
||||
|
||||
echo 'Build complete.'
|
||||
@@ -1,42 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-cli', pathex=[cwd])
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-cli',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
import lbryum
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
datas = [
|
||||
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
|
||||
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
|
||||
]
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-console', pathex=[cwd], datas=datas)
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-console',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
import lbryum
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
datas = [
|
||||
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
|
||||
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
|
||||
]
|
||||
|
||||
|
||||
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-daemon', pathex=[cwd], datas=datas)
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-daemon',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -1,47 +0,0 @@
|
||||
# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point
|
||||
def Entrypoint(dist, group, name,
|
||||
scripts=None, pathex=None, binaries=None, datas=None,
|
||||
hiddenimports=None, hookspath=None, excludes=None, runtime_hooks=None,
|
||||
cipher=None, win_no_prefer_redirects=False, win_private_assemblies=False):
|
||||
import pkg_resources
|
||||
|
||||
# get toplevel packages of distribution from metadata
|
||||
def get_toplevel(dist):
|
||||
distribution = pkg_resources.get_distribution(dist)
|
||||
if distribution.has_metadata('top_level.txt'):
|
||||
return list(distribution.get_metadata('top_level.txt').split())
|
||||
else:
|
||||
return []
|
||||
|
||||
hiddenimports = hiddenimports or []
|
||||
packages = []
|
||||
for distribution in hiddenimports:
|
||||
packages += get_toplevel(distribution)
|
||||
|
||||
scripts = scripts or []
|
||||
pathex = pathex or []
|
||||
# get the entry point
|
||||
ep = pkg_resources.get_entry_info(dist, group, name)
|
||||
# insert path of the egg at the verify front of the search path
|
||||
pathex = [ep.dist.location] + pathex
|
||||
# script name must not be a valid module name to avoid name clashes on import
|
||||
script_path = os.path.join(workpath, name + '-script.py')
|
||||
print "creating script for entry point", dist, group, name
|
||||
with open(script_path, 'w') as fh:
|
||||
fh.write("import {0}\n".format(ep.module_name))
|
||||
fh.write("{0}.{1}()\n".format(ep.module_name, '.'.join(ep.attrs)))
|
||||
for package in packages:
|
||||
fh.write("import {0}\n".format(package))
|
||||
|
||||
return Analysis([script_path] + scripts,
|
||||
pathex=pathex,
|
||||
binaries=binaries,
|
||||
datas=datas,
|
||||
hiddenimports=hiddenimports,
|
||||
hookspath=hookspath,
|
||||
excludes=excludes,
|
||||
runtime_hooks=runtime_hooks,
|
||||
cipher=cipher,
|
||||
win_no_prefer_redirects=win_no_prefer_redirects,
|
||||
win_private_assemblies=win_private_assemblies
|
||||
)
|
||||
@@ -1,82 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
|
||||
LINUX=false
|
||||
OSX=false
|
||||
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
OSX=true
|
||||
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
|
||||
LINUX=true
|
||||
else
|
||||
echo "Platform detection failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
SUDO=''
|
||||
if $LINUX && (( $EUID != 0 )); then
|
||||
SUDO='sudo'
|
||||
fi
|
||||
|
||||
cmd_exists() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
return $?
|
||||
}
|
||||
|
||||
set +eu
|
||||
GITUSERNAME=$(git config --global --get user.name)
|
||||
if [ -z "$GITUSERNAME" ]; then
|
||||
git config --global user.name "$(whoami)"
|
||||
fi
|
||||
GITEMAIL=$(git config --global --get user.email)
|
||||
if [ -z "$GITEMAIL" ]; then
|
||||
git config --global user.email "$(whoami)@lbry.io"
|
||||
fi
|
||||
set -eu
|
||||
|
||||
|
||||
if $LINUX; then
|
||||
INSTALL="$SUDO apt-get install --no-install-recommends -y"
|
||||
$INSTALL build-essential libssl-dev libffi-dev python2.7-dev wget
|
||||
elif $OSX && ! cmd_exists brew ; then
|
||||
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
|
||||
fi
|
||||
|
||||
|
||||
if ! cmd_exists python; then
|
||||
if $LINUX; then
|
||||
$INSTALL python2.7
|
||||
elif $OSX; then
|
||||
brew install python
|
||||
curl https://bootstrap.pypa.io/get-pip.py | python
|
||||
fi
|
||||
fi
|
||||
|
||||
PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
|
||||
if [ "$PYTHON_VERSION" != "2.7" ]; then
|
||||
echo "Python 2.7 required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! cmd_exists pip; then
|
||||
if $LINUX; then
|
||||
$INSTALL python-pip
|
||||
$SUDO pip install --upgrade pip
|
||||
else
|
||||
echo "Pip required"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
|
||||
#$INSTALL python-setuptools
|
||||
$SUDO pip install setuptools
|
||||
fi
|
||||
|
||||
if ! cmd_exists virtualenv; then
|
||||
$SUDO pip install virtualenv
|
||||
fi
|
||||
@@ -1,11 +0,0 @@
|
||||
# install daemon requirements (created by build script. see build.sh, build.ps1)
|
||||
-r requirements_base.txt
|
||||
|
||||
# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
|
||||
file:../.
|
||||
|
||||
# install other build requirements
|
||||
PyInstaller==3.2.1
|
||||
requests[security]==2.13.0
|
||||
uritemplate==3.0.0
|
||||
boto3==1.4.4
|
||||
@@ -1,143 +0,0 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import github
|
||||
import uritemplate
|
||||
import boto3
|
||||
|
||||
|
||||
def main():
|
||||
upload_to_github_if_tagged('lbryio/lbry')
|
||||
upload_to_s3('daemon')
|
||||
|
||||
|
||||
def get_asset_filename():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
return glob.glob(this_dir + '/dist/*.zip')[0]
|
||||
|
||||
|
||||
def upload_to_s3(folder):
|
||||
tag = subprocess.check_output(['git', 'describe', '--always', '--abbrev=8', 'HEAD']).strip()
|
||||
commit_date = subprocess.check_output([
|
||||
'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
bucket = 'releases.lbry.io'
|
||||
key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
|
||||
|
||||
print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
|
||||
|
||||
if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
|
||||
print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
|
||||
return 1
|
||||
|
||||
s3 = boto3.resource(
|
||||
's3',
|
||||
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
|
||||
config=boto3.session.Config(signature_version='s3v4')
|
||||
)
|
||||
s3.meta.client.upload_file(asset_path, bucket, key)
|
||||
|
||||
|
||||
def upload_to_github_if_tagged(repo_name):
|
||||
try:
|
||||
current_tag = subprocess.check_output(
|
||||
['git', 'describe', '--exact-match', 'HEAD']).strip()
|
||||
except subprocess.CalledProcessError:
|
||||
print 'Not uploading to GitHub as we are not currently on a tag'
|
||||
return 1
|
||||
|
||||
print "Current tag: " + current_tag
|
||||
|
||||
if 'GH_TOKEN' not in os.environ:
|
||||
print 'Must set GH_TOKEN in order to publish assets to a release'
|
||||
return 1
|
||||
|
||||
gh_token = os.environ['GH_TOKEN']
|
||||
auth = github.Github(gh_token)
|
||||
repo = auth.get_repo(repo_name)
|
||||
|
||||
if not check_repo_has_tag(repo, current_tag):
|
||||
print 'Tag {} is not in repo {}'.format(current_tag, repo)
|
||||
# TODO: maybe this should be an error
|
||||
return 1
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
print "Uploading " + asset_path + " to Github tag " + current_tag
|
||||
release = get_github_release(repo, current_tag)
|
||||
upload_asset_to_github(release, asset_path, gh_token)
|
||||
|
||||
|
||||
def check_repo_has_tag(repo, target_tag):
|
||||
tags = repo.get_tags().get_page(0)
|
||||
for tag in tags:
|
||||
if tag.name == target_tag:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_github_release(repo, current_tag):
|
||||
for release in repo.get_releases():
|
||||
if release.tag_name == current_tag:
|
||||
return release
|
||||
raise Exception('No release for {} was found'.format(current_tag))
|
||||
|
||||
|
||||
def upload_asset_to_github(release, asset_to_upload, token):
|
||||
basename = os.path.basename(asset_to_upload)
|
||||
for asset in release.raw_data['assets']:
|
||||
if asset['name'] == basename:
|
||||
print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
|
||||
return
|
||||
|
||||
upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
|
||||
count = 0
|
||||
while count < 10:
|
||||
try:
|
||||
output = _curl_uploader(upload_uri, asset_to_upload, token)
|
||||
if 'errors' in output:
|
||||
raise Exception(output)
|
||||
else:
|
||||
print 'Successfully uploaded to {}'.format(output['browser_download_url'])
|
||||
except Exception:
|
||||
print 'Failed uploading on attempt {}'.format(count + 1)
|
||||
count += 1
|
||||
|
||||
|
||||
def _curl_uploader(upload_uri, asset_to_upload, token):
|
||||
# using requests.post fails miserably with SSL EPIPE errors. I spent
|
||||
# half a day trying to debug before deciding to switch to curl.
|
||||
#
|
||||
# TODO: actually set the content type
|
||||
print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
|
||||
cmd = [
|
||||
'curl',
|
||||
'-sS',
|
||||
'-X', 'POST',
|
||||
'-u', ':{}'.format(os.environ['GH_TOKEN']),
|
||||
'--header', 'Content-Type: application/octet-stream',
|
||||
'--data-binary', '@-',
|
||||
upload_uri
|
||||
]
|
||||
# '-d', '{"some_key": "some_value"}',
|
||||
print 'Calling curl:'
|
||||
print cmd
|
||||
print
|
||||
with open(asset_to_upload, 'rb') as fp:
|
||||
p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||
stdout, stderr = p.communicate()
|
||||
print 'curl return code:', p.returncode
|
||||
if stderr:
|
||||
print 'stderr output from curl:'
|
||||
print stderr
|
||||
print 'stdout from curl:'
|
||||
print stdout
|
||||
return json.loads(stdout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,29 +0,0 @@
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
|
||||
def main():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
tag = subprocess.check_output(['git', 'describe']).strip()
|
||||
zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
|
||||
full_filename = os.path.join(this_dir, 'dist', zipfilename)
|
||||
executables = ['lbrynet-daemon', 'lbrynet-cli', 'lbrynet-console']
|
||||
ext = '.exe' if platform.system() == 'Windows' else ''
|
||||
with zipfile.ZipFile(full_filename, 'w') as myzip:
|
||||
for executable in executables:
|
||||
myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext)
|
||||
|
||||
|
||||
def get_system_label():
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
return 'macos'
|
||||
else:
|
||||
return system.lower()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -100,7 +100,7 @@
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
|
||||
|
||||
</a>
|
||||
</div>
|
||||
@@ -196,7 +196,7 @@
|
||||
<label class="md-nav__title md-nav__title--site" for="drawer">
|
||||
<span class="md-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
|
||||
|
||||
</span>
|
||||
LBRY
|
||||
@@ -304,4 +304,4 @@
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -1903,7 +1903,7 @@ Options:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
--claim_address=<claim_address> : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
|
||||
@@ -1743,7 +1743,7 @@ Args:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
'claim_address' : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"docs": [
|
||||
{
|
||||
"location": "/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"title": "API"
|
||||
},
|
||||
{
|
||||
@@ -167,7 +167,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
@@ -292,7 +292,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"title": "CLI"
|
||||
},
|
||||
{
|
||||
@@ -457,7 +457,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# This is an example daemon_settings.yml file.
|
||||
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
|
||||
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.io:50001
|
||||
- lbryumx2.lbry.io:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - dht
|
||||
|
Before Width: | Height: | Size: 7.4 KiB After Width: | Height: | Size: 7.4 KiB |
|
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 19 KiB |
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 2.6 KiB |
|
Before Width: | Height: | Size: 6.1 KiB After Width: | Height: | Size: 6.1 KiB |
|
Before Width: | Height: | Size: 97 KiB After Width: | Height: | Size: 97 KiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 361 KiB After Width: | Height: | Size: 361 KiB |
|
Before Width: | Height: | Size: 5.3 KiB After Width: | Height: | Size: 5.3 KiB |
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 31 KiB After Width: | Height: | Size: 31 KiB |
@@ -1,6 +1,4 @@
|
||||
import logging
|
||||
|
||||
__version__ = "0.21.1"
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.5"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -1,274 +0,0 @@
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer, task
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import looping_call_manager, utils, system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
SERVER_STARTUP_ERROR = 'Server Startup Error'
|
||||
DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
self.session_id = session_id or conf.settings.get_session_id()
|
||||
self.is_started = False
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
api = Api.new_instance(enabled)
|
||||
return cls(api)
|
||||
|
||||
# Things We Track
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 60),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 300),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self._tracked_data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash,
|
||||
'error': error_name(error),
|
||||
'reason': error.message,
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
context = {
|
||||
'app': {
|
||||
'name': 'lbrynet',
|
||||
'version': platform['lbrynet_version'],
|
||||
'python_version': platform['python_version'],
|
||||
'build': platform['build'],
|
||||
'wallet': {
|
||||
'name': wallet,
|
||||
'version': platform['lbryum_version'] if wallet == conf.LBRYUM_WALLET else None
|
||||
},
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
'library': {
|
||||
'name': 'lbrynet-analytics',
|
||||
'version': '1.0.0'
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api(object):
|
||||
def __init__(self, cookies, url, write_key, enabled):
|
||||
self.cookies = cookies
|
||||
self.url = url
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
def update_cookies(response):
|
||||
self.cookies.update(response.cookies())
|
||||
return response
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
headers = {b"Connection": b"close"}
|
||||
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
|
||||
headers=headers, cookies=self.cookies)
|
||||
d.addCallback(update_cookies)
|
||||
return d
|
||||
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('Analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if enabled is None:
|
||||
enabled = conf.settings['share_usage_data']
|
||||
return cls(
|
||||
{},
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
|
||||
enabled,
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
import paths
|
||||
@@ -1,26 +0,0 @@
|
||||
# App root files dir
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
|
||||
def android_files_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base internal storage path
|
||||
def android_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base external (SD card, if present) storage path
|
||||
def android_external_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Internal device storage (private app folder)
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# External (app folder on SD card, if present) storage
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_external_storage_dir():
|
||||
return None
|
||||
@@ -7,7 +7,7 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.p2p.BlobInfo import BlobInfo
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -16,21 +16,21 @@ backend = default_backend()
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
super().__init__(blob_hash, blob_num, length)
|
||||
self.iv = iv
|
||||
|
||||
def get_dict(self):
|
||||
info = {
|
||||
"blob_num": self.blob_num,
|
||||
"length": self.length,
|
||||
"iv": self.iv
|
||||
"iv": self.iv.decode()
|
||||
}
|
||||
if self.blob_hash:
|
||||
info['blob_hash'] = self.blob_hash
|
||||
return info
|
||||
|
||||
|
||||
class StreamBlobDecryptor(object):
|
||||
class StreamBlobDecryptor:
|
||||
def __init__(self, blob, key, iv, length):
|
||||
"""
|
||||
This class decrypts blob
|
||||
@@ -53,10 +53,10 @@ class StreamBlobDecryptor(object):
|
||||
|
||||
def decrypt(self, write_func):
|
||||
"""
|
||||
Decrypt blob and write its content useing write_func
|
||||
Decrypt blob and write its content using write_func
|
||||
|
||||
write_func - function that takes decrypted string as
|
||||
arugment and writes it somewhere
|
||||
argument and writes it somewhere
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -68,14 +68,14 @@ class StreamBlobDecryptor(object):
|
||||
|
||||
def write_bytes():
|
||||
if self.len_read < self.length:
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size / 8))
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size // 8))
|
||||
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
|
||||
write_func(self.cipher.update(data_to_decrypt))
|
||||
|
||||
def finish_decrypt():
|
||||
bytes_left = len(self.buff) % (AES.block_size / 8)
|
||||
bytes_left = len(self.buff) % (AES.block_size // 8)
|
||||
if bytes_left != 0:
|
||||
log.warning(self.buff[-1 * (AES.block_size / 8):].encode('hex'))
|
||||
log.warning(binascii.hexlify(self.buff[-1 * (AES.block_size // 8):]).decode())
|
||||
raise Exception("blob %s has incorrect padding: %i bytes left" %
|
||||
(self.blob.blob_hash, bytes_left))
|
||||
data_to_decrypt, self.buff = self.buff, b''
|
||||
@@ -99,7 +99,7 @@ class StreamBlobDecryptor(object):
|
||||
return d
|
||||
|
||||
|
||||
class CryptStreamBlobMaker(object):
|
||||
class CryptStreamBlobMaker:
|
||||
def __init__(self, key, iv, blob_num, blob):
|
||||
"""
|
||||
This class encrypts data and writes it to a new blob
|
||||
@@ -146,7 +146,7 @@ class CryptStreamBlobMaker(object):
|
||||
def close(self):
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self.length += (AES.block_size / 8) - (self.length % (AES.block_size / 8))
|
||||
self.length += (AES.block_size // 8) - (self.length % (AES.block_size // 8))
|
||||
padded_data = self.padder.finalize()
|
||||
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
|
||||
self.blob.write(encrypted_data)
|
||||
@@ -5,15 +5,14 @@ import os
|
||||
import logging
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
from twisted.internet import defer
|
||||
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(object):
|
||||
class CryptStreamCreator:
|
||||
"""
|
||||
Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
@@ -22,7 +21,7 @@ class CryptStreamCreator(object):
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
|
||||
implements(interfaces.IConsumer)
|
||||
#implements(interfaces.IConsumer)
|
||||
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@@ -101,13 +100,13 @@ class CryptStreamCreator(object):
|
||||
@staticmethod
|
||||
def random_iv_generator():
|
||||
while 1:
|
||||
yield os.urandom(AES.block_size / 8)
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
def setup(self):
|
||||
"""Create the symmetric key if it wasn't provided"""
|
||||
|
||||
if self.key is None:
|
||||
self.key = os.urandom(AES.block_size / 8)
|
||||
self.key = os.urandom(AES.block_size // 8)
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
@@ -122,7 +121,7 @@ class CryptStreamCreator(object):
|
||||
|
||||
yield defer.DeferredList(self.finished_deferreds)
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
iv = next(self.iv_generator)
|
||||
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
|
||||
stream_terminator = yield final_blob.close()
|
||||
terminator_info = yield self._blob_finished(stream_terminator)
|
||||
@@ -133,7 +132,7 @@ class CryptStreamCreator(object):
|
||||
if self.current_blob is None:
|
||||
self.next_blob_creator = self.blob_manager.get_blob_creator()
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
iv = next(self.iv_generator)
|
||||
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
|
||||
done, num_bytes_written = self.current_blob.write(data)
|
||||
data = data[num_bytes_written:]
|
||||
@@ -2,16 +2,17 @@
|
||||
Utilities for turning plain files into LBRY Files.
|
||||
"""
|
||||
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.core.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,7 +24,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
|
||||
def __init__(self, blob_manager, lbry_file_manager, stream_name=None,
|
||||
key=None, iv_generator=None):
|
||||
CryptStreamCreator.__init__(self, blob_manager, stream_name, key, iv_generator)
|
||||
super().__init__(blob_manager, stream_name, key, iv_generator)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.stream_hash = None
|
||||
self.blob_infos = []
|
||||
@@ -37,14 +38,14 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
def _finished(self):
|
||||
# calculate the stream hash
|
||||
self.stream_hash = get_stream_hash(
|
||||
hexlify(self.name), hexlify(self.key), hexlify(self.name),
|
||||
hexlify(self.name.encode()).decode(), hexlify(self.key).decode(), hexlify(self.name.encode()).decode(),
|
||||
self.blob_infos
|
||||
)
|
||||
|
||||
# generate the sd info
|
||||
self.sd_info = format_sd_info(
|
||||
EncryptedFileStreamType, hexlify(self.name), hexlify(self.key),
|
||||
hexlify(self.name), self.stream_hash, self.blob_infos
|
||||
EncryptedFileStreamType, hexlify(self.name.encode()).decode(), hexlify(self.key).decode(),
|
||||
hexlify(self.name.encode()).decode(), self.stream_hash, self.blob_infos
|
||||
)
|
||||
|
||||
# sanity check
|
||||
@@ -119,21 +120,13 @@ def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_mana
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield storage.store_stream(
|
||||
yield f2d(storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
)
|
||||
))
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, binascii.hexlify(file_directory), payment_rate_manager,
|
||||
lbry_file = yield f2d(lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
|
||||
def hexlify(str_or_unicode):
|
||||
if isinstance(str_or_unicode, unicode):
|
||||
strng = str_or_unicode.encode('utf-8')
|
||||
else:
|
||||
strng = str_or_unicode
|
||||
return binascii.hexlify(strng)
|
||||
@@ -2,19 +2,17 @@
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
import logging
|
||||
import binascii
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.core.utils import short_hash
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileDownloader
|
||||
from lbrynet.file_manager.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.core.StreamDescriptor import save_sd_info
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,16 +34,16 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
|
||||
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
|
||||
suggested_file_name, download_mirrors=None):
|
||||
EncryptedFileSaver.__init__(
|
||||
self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name
|
||||
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
|
||||
sd_hash, key, suggested_file_name, download_mirrors=None):
|
||||
super().__init__(
|
||||
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
|
||||
wallet, download_directory, key, stream_name, file_name
|
||||
)
|
||||
self.sd_hash = sd_hash
|
||||
self.rowid = rowid
|
||||
self.suggested_file_name = binascii.unhexlify(suggested_file_name)
|
||||
self.suggested_file_name = unhexlify(suggested_file_name).decode()
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self._saving_status = False
|
||||
self.claim_id = None
|
||||
@@ -57,9 +55,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = None
|
||||
self.metadata = None
|
||||
self.mirror = None
|
||||
if download_mirrors or conf.settings['download_mirrors']:
|
||||
if download_mirrors or conf.download_mirrors:
|
||||
self.mirror = HTTPBlobDownloader(
|
||||
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
|
||||
self.blob_manager, servers=download_mirrors or conf.download_mirrors
|
||||
)
|
||||
|
||||
def set_claim_info(self, claim_info):
|
||||
@@ -72,13 +70,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = claim_info['channel_name']
|
||||
self.metadata = claim_info['value']['stream']['metadata']
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim_info(self, include_supports=True):
|
||||
claim_info = yield self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
async def get_claim_info(self, include_supports=True):
|
||||
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
if claim_info:
|
||||
self.set_claim_info(claim_info)
|
||||
|
||||
defer.returnValue(claim_info)
|
||||
return claim_info
|
||||
|
||||
@property
|
||||
def saving_status(self):
|
||||
@@ -95,7 +91,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
self.completed = True
|
||||
else:
|
||||
raise Exception("Unknown status for stream %s: %s" % (self.stream_hash, status))
|
||||
raise Exception(f"Unknown status for stream {self.stream_hash}: {status}")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None, change_status=True):
|
||||
@@ -103,16 +99,15 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
if self.mirror:
|
||||
self.mirror.stop()
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
yield EncryptedFileDownloader.stop(self, err=err)
|
||||
yield super().stop(err)
|
||||
if change_status is True:
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def status(self):
|
||||
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
async def status(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
|
||||
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
|
||||
num_blobs_completed = len(completed_blobs)
|
||||
num_blobs_known = len(blob_hashes)
|
||||
|
||||
@@ -122,9 +117,10 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = "stopped"
|
||||
else:
|
||||
status = "running"
|
||||
defer.returnValue(EncryptedFileStatusReport(
|
||||
|
||||
return EncryptedFileStatusReport(
|
||||
self.file_name, num_blobs_completed, num_blobs_known, status
|
||||
))
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
@@ -152,7 +148,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
|
||||
self._saving_status = False
|
||||
defer.returnValue(status)
|
||||
return status
|
||||
|
||||
def save_status(self):
|
||||
return self._save_status()
|
||||
@@ -162,8 +158,8 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
class ManagedEncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, lbry_file_manager, blob_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
@@ -180,11 +176,12 @@ class ManagedEncryptedFileDownloaderFactory(object):
|
||||
metadata.source_blob_hash,
|
||||
metadata.validator.raw_info)
|
||||
if file_name:
|
||||
file_name = binascii.hexlify(file_name)
|
||||
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, binascii.hexlify(download_directory), payment_rate_manager,
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@staticmethod
|
||||
@@ -3,24 +3,25 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.reflector.reupload import reflect_file
|
||||
# from lbrynet.core.PaymentRateManager import NegotiatedPaymentRateManager
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
from lbrynet import conf
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileManager(object):
|
||||
class EncryptedFileManager:
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and
|
||||
their LBRY File specific metadata.
|
||||
@@ -28,9 +29,11 @@ class EncryptedFileManager(object):
|
||||
# when reflecting files, reflect up to this many files at a time
|
||||
CONCURRENT_REFLECTS = 5
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
|
||||
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
|
||||
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier):
|
||||
self.conf = conf
|
||||
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
|
||||
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -42,11 +45,9 @@ class EncryptedFileManager(object):
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
yield self._add_to_sd_identifier()
|
||||
yield self._start_lbry_files()
|
||||
log.info("Started file manager")
|
||||
self._add_to_sd_identifier()
|
||||
return self._start_lbry_files()
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
@@ -56,7 +57,7 @@ class EncryptedFileManager(object):
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self.storage.change_file_status(lbry_file.rowid, status)
|
||||
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -80,6 +81,7 @@ class EncryptedFileManager(object):
|
||||
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
|
||||
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
|
||||
return ManagedEncryptedFileDownloader(
|
||||
self.conf,
|
||||
rowid,
|
||||
stream_hash,
|
||||
self.peer_finder,
|
||||
@@ -116,10 +118,9 @@ class EncryptedFileManager(object):
|
||||
except Exception:
|
||||
log.warning("Failed to start %i", file_info.get('rowid'))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_lbry_files(self):
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
async def _start_lbry_files(self):
|
||||
files = await self.storage.get_all_lbry_files()
|
||||
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
@@ -152,14 +153,13 @@ class EncryptedFileManager(object):
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
rowid = yield self.storage.save_published_file(
|
||||
rowid = await self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate, status
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
@@ -167,37 +167,36 @@ class EncryptedFileManager(object):
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info()
|
||||
await lbry_file.get_claim_info()
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = yield self.storage.save_downloaded_file(
|
||||
stream_hash, os.path.basename(file_name.decode('hex')).encode('hex'), download_directory, blob_data_rate
|
||||
rowid = await self.storage.save_downloaded_file(
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = yield self.storage.get_filename_for_rowid(rowid)
|
||||
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info(include_supports=False)
|
||||
await lbry_file.get_claim_info(include_supports=False)
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_lbry_file(self, lbry_file, delete_file=False):
|
||||
@@ -223,7 +222,7 @@ class EncryptedFileManager(object):
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield self.storage.delete_stream(lbry_file.stream_hash)
|
||||
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
@@ -241,10 +240,10 @@ class EncryptedFileManager(object):
|
||||
def reflect_lbry_files(self):
|
||||
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
|
||||
ds = []
|
||||
sd_hashes_to_reflect = yield self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
|
||||
for lbry_file in self.lbry_files:
|
||||
if lbry_file.sd_hash in sd_hashes_to_reflect:
|
||||
ds.append(sem.run(reflect_file, lbry_file))
|
||||
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
|
||||
yield defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -1,4 +1,4 @@
|
||||
class EncryptedFileStatusReport(object):
|
||||
class EncryptedFileStatusReport:
|
||||
def __init__(self, name, num_completed, num_known, running_status):
|
||||
self.name = name
|
||||
self.num_completed = num_completed
|
||||
@@ -1,4 +0,0 @@
|
||||
from blob_file import BlobFile
|
||||
from creator import BlobFileCreator
|
||||
from writer import HashBlobWriter
|
||||
from reader import HashBlobReader
|
||||
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import defer, threads
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.blob.reader import HashBlobReader
|
||||
|
||||
@@ -12,8 +12,26 @@ log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
class BlobFile(object):
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
class BlobFile:
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
|
||||
@@ -52,7 +70,7 @@ class BlobFile(object):
|
||||
def open_for_writing(self, peer):
|
||||
"""
|
||||
open a blob file to be written by peer, supports concurrent
|
||||
writers, as long as they are from differnt peers.
|
||||
writers, as long as they are from different peers.
|
||||
|
||||
returns tuple of (writer, finished_deferred)
|
||||
|
||||
@@ -60,12 +78,12 @@ class BlobFile(object):
|
||||
finished_deferred - deferred that is fired when write is finished and returns
|
||||
a instance of itself as HashBlob
|
||||
"""
|
||||
if not peer in self.writers:
|
||||
if peer not in self.writers:
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(self.get_length, self.writer_finished)
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return (writer, finished_deferred)
|
||||
return writer, finished_deferred
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None
|
||||
|
||||
@@ -93,29 +111,19 @@ class BlobFile(object):
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
self.saved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
try:
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
except Exception as e:
|
||||
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
raise ValueError("File is currently being read or written and cannot be deleted")
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
"""
|
||||
Protect verified from being modified by other classes.
|
||||
verified is True if a write to a blob has completed succesfully,
|
||||
verified is True if a write to a blob has completed successfully,
|
||||
or a blob has been read to have the same length as specified
|
||||
in init
|
||||
"""
|
||||
@@ -149,7 +157,7 @@ class BlobFile(object):
|
||||
def writer_finished(self, writer, err=None):
|
||||
def fire_finished_deferred():
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.callback(self)
|
||||
@@ -160,7 +168,7 @@ class BlobFile(object):
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.errback(err)
|
||||
@@ -182,7 +190,7 @@ class BlobFile(object):
|
||||
if writer.len_so_far != self.length:
|
||||
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
|
||||
else:
|
||||
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
|
||||
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(None)
|
||||
else:
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import binascii
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from lbrynet.cryptstream.CryptBlob import StreamBlobDecryptor
|
||||
from lbrynet.interfaces import IBlobHandler
|
||||
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
|
||||
|
||||
|
||||
class CryptBlobHandler(object):
|
||||
implements(IBlobHandler)
|
||||
class CryptBlobHandler:
|
||||
#implements(IBlobHandler)
|
||||
|
||||
def __init__(self, key, write_func):
|
||||
self.key = key
|
||||
@@ -1,15 +1,15 @@
|
||||
import binascii
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IStreamDownloader
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.cryptstream.client.CryptBlobHandler import CryptBlobHandler
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.p2p.client.DownloadManager import DownloadManager
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -34,11 +34,11 @@ class CurrentlyStartingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CryptStreamDownloader(object):
|
||||
class CryptStreamDownloader:
|
||||
|
||||
implements(IStreamDownloader)
|
||||
#implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
key, stream_name):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@@ -56,14 +56,14 @@ class CryptStreamDownloader(object):
|
||||
@return:
|
||||
|
||||
"""
|
||||
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.key = binascii.unhexlify(key)
|
||||
self.stream_name = binascii.unhexlify(stream_name)
|
||||
self.key = unhexlify(key)
|
||||
self.stream_name = unhexlify(stream_name).decode()
|
||||
self.completed = False
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
@@ -1,16 +1,16 @@
|
||||
import binascii
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.StreamDescriptor import save_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbry_file.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
import os
|
||||
from twisted.internet import defer, threads
|
||||
import logging
|
||||
import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.Error import FileOpenError
|
||||
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -19,35 +19,45 @@ log = logging.getLogger(__name__)
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
storage, payment_rate_manager, wallet, key, stream_name, file_name):
|
||||
CryptStreamDownloader.__init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
self.file_name = binascii.unhexlify(os.path.basename(file_name))
|
||||
self.file_name = os.path.basename(unhexlify(file_name).decode())
|
||||
self._calculated_total_bytes = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_data(self):
|
||||
crypt_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(self.stream_hash)
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
|
||||
blob_hashes.append(sd_hash)
|
||||
yield self.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
self._close_output()
|
||||
return super().stop(err=err)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _start(self):
|
||||
def check_start_succeeded(success):
|
||||
if success:
|
||||
self.starting = False
|
||||
self.stopped = False
|
||||
self.completed = False
|
||||
return True
|
||||
else:
|
||||
return self._start_failed()
|
||||
|
||||
self.download_manager = self._get_download_manager()
|
||||
d = self._setup_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader._start(self))
|
||||
d.addCallback(lambda _: self.download_manager.start_downloading())
|
||||
d.addCallbacks(check_start_succeeded)
|
||||
return d
|
||||
|
||||
def _setup_output(self):
|
||||
@@ -56,14 +66,9 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
def _close_output(self):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
async def get_total_bytes(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
@@ -90,10 +95,11 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
self.storage, download_manager)
|
||||
|
||||
|
||||
class EncryptedFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
class EncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -126,22 +132,18 @@ class EncryptedFileDownloaderFactory(object):
|
||||
|
||||
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name):
|
||||
EncryptedFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = binascii.unhexlify(download_directory)
|
||||
self.file_written_to = os.path.join(self.download_directory, binascii.unhexlify(file_name))
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
|
||||
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = unhexlify(download_directory).decode()
|
||||
self.file_written_to = os.path.join(self.download_directory, unhexlify(file_name).decode())
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.file_written_to)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
@@ -156,7 +158,7 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
self.file_written_to = file_written_to
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise ValueError(
|
||||
raise FileOpenError(
|
||||
"Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." % file_written_to
|
||||
)
|
||||
@@ -164,15 +166,11 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
def _close_output(self):
|
||||
self.file_handle, file_handle = None, self.file_handle
|
||||
|
||||
def close_file():
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
return threads.deferToThread(close_file)
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
def _get_write_func(self):
|
||||
def write_func(data):
|
||||
@@ -182,17 +180,18 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
EncryptedFileDownloaderFactory.__init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = binascii.hexlify(download_directory)
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = hexlify(download_directory.encode())
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
stream_name = stream_info.raw_info['stream_name']
|
||||
key = stream_info.raw_info['key']
|
||||
suggested_file_name = stream_info.raw_info['suggested_file_name']
|
||||
return EncryptedFileSaver(
|
||||
stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage, payment_rate_manager,
|
||||
self.wallet, self.download_directory, key=key, stream_name=stream_name, file_name=suggested_file_name
|
||||
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
|
||||
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
|
||||
file_name=suggested_file_name
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1,14 +1,12 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from lbrynet.interfaces import IMetadataHandler
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileMetadataHandler(object):
|
||||
implements(IMetadataHandler)
|
||||
class EncryptedFileMetadataHandler:
|
||||
|
||||
def __init__(self, stream_hash, storage, download_manager):
|
||||
self.stream_hash = stream_hash
|
||||
@@ -20,9 +18,8 @@ class EncryptedFileMetadataHandler(object):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_initial_blobs(self):
|
||||
blob_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
formatted_infos = self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
defer.returnValue(formatted_infos)
|
||||
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
return self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
|
||||
def final_blob_num(self):
|
||||
return self._final_blob_num
|
||||
@@ -1,6 +1,6 @@
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.p2p.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
@@ -8,7 +8,7 @@ def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
EncryptedFileOptions())
|
||||
|
||||
|
||||
class EncryptedFileOptions(object):
|
||||
class EncryptedFileOptions:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@@ -3,12 +3,12 @@ import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileCreator(object):
|
||||
class BlobFileCreator:
|
||||
"""
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we do not know the blob hash beforehand (i.e, when creating
|
||||
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
class HashBlobReader:
|
||||
"""
|
||||
This is a file like reader class that supports
|
||||
read(size) and close()
|
||||
@@ -15,7 +15,7 @@ class HashBlobReader(object):
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warn("Garbage collection was called, but reader for %s was not closed yet",
|
||||
log.warning("Garbage collection was called, but reader for %s was not closed yet",
|
||||
self.read_handle.name)
|
||||
self.close()
|
||||
|
||||
@@ -28,5 +28,3 @@ class HashBlobReader(object):
|
||||
return
|
||||
self.read_handle.close()
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobWriter(object):
|
||||
class HashBlobWriter:
|
||||
def __init__(self, length_getter, finished_cb):
|
||||
self.write_handle = BytesIO()
|
||||
self.length_getter = length_getter
|
||||
@@ -18,7 +18,7 @@ class HashBlobWriter(object):
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warn("Garbage collection was called, but writer was not closed yet")
|
||||
log.warning("Garbage collection was called, but writer was not closed yet")
|
||||
self.close()
|
||||
|
||||
@property
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
# dont touch this. CI server changes this during build/deployment
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
@@ -1,120 +0,0 @@
|
||||
from random import choice
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
import treq
|
||||
|
||||
from lbrynet.core.Error import DownloadCanceledError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HTTPBlobDownloader(object):
|
||||
'''
|
||||
A downloader that is able to get blobs from HTTP mirrors.
|
||||
Note that when a blob gets downloaded from a mirror or from a peer, BlobManager will mark it as completed
|
||||
and cause any other type of downloader to progress to the next missing blob. Also, BlobFile is naturally able
|
||||
to cancel other writers when a writer finishes first. That's why there is no call to cancel/resume/stop between
|
||||
different types of downloaders.
|
||||
'''
|
||||
def __init__(self, blob_manager, blob_hashes=None, servers=None, client=None, sd_hashes=None):
|
||||
self.blob_manager = blob_manager
|
||||
self.servers = servers or []
|
||||
self.client = client or treq
|
||||
self.blob_hashes = blob_hashes or []
|
||||
self.sd_hashes = sd_hashes or []
|
||||
self.head_blob_hashes = []
|
||||
self.max_failures = 3
|
||||
self.running = False
|
||||
self.semaphore = defer.DeferredSemaphore(2)
|
||||
self.deferreds = []
|
||||
self.writers = []
|
||||
|
||||
def start(self):
|
||||
if not self.running and self.blob_hashes and self.servers:
|
||||
return self._start()
|
||||
defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
if self.running:
|
||||
for d in reversed(self.deferreds):
|
||||
d.cancel()
|
||||
for writer in self.writers:
|
||||
writer.close(DownloadCanceledError())
|
||||
self.running = False
|
||||
self.blob_hashes = []
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
self.running = True
|
||||
dl = []
|
||||
for blob_hash in self.blob_hashes:
|
||||
blob = yield self.blob_manager.get_blob(blob_hash)
|
||||
if not blob.verified:
|
||||
d = self.semaphore.run(self.download_blob, blob)
|
||||
d.addErrback(lambda err: err.check(defer.TimeoutError, defer.CancelledError))
|
||||
dl.append(d)
|
||||
self.deferreds = dl
|
||||
yield defer.DeferredList(dl)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_blob(self, blob):
|
||||
for _ in range(self.max_failures):
|
||||
writer, finished_deferred = blob.open_for_writing('mirror')
|
||||
self.writers.append(writer)
|
||||
try:
|
||||
downloaded = yield self._write_blob(writer, blob)
|
||||
if downloaded:
|
||||
yield finished_deferred # yield for verification errors, so we log them
|
||||
if blob.verified:
|
||||
log.info('Mirror completed download for %s', blob.blob_hash)
|
||||
b_h = blob.blob_hash
|
||||
if b_h in self.sd_hashes or b_h in self.head_blob_hashes:
|
||||
should_announce = True
|
||||
else:
|
||||
should_announce = False
|
||||
yield self.blob_manager.blob_completed(blob, should_announce=should_announce)
|
||||
break
|
||||
except (IOError, Exception) as e:
|
||||
if isinstance(e, DownloadCanceledError) or 'closed file' in str(e):
|
||||
# some other downloader finished first or it was simply cancelled
|
||||
log.info("Mirror download cancelled: %s", blob.blob_hash)
|
||||
break
|
||||
else:
|
||||
log.exception('Mirror failed downloading')
|
||||
finally:
|
||||
finished_deferred.addBoth(lambda _: None) # suppress echoed errors
|
||||
if 'mirror' in blob.writers:
|
||||
writer.close()
|
||||
self.writers.remove(writer)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _write_blob(self, writer, blob):
|
||||
response = yield self.client.get(url_for(choice(self.servers), blob.blob_hash))
|
||||
if response.code != 200:
|
||||
log.debug('Missing a blob: %s', blob.blob_hash)
|
||||
if blob.blob_hash in self.blob_hashes:
|
||||
self.blob_hashes.remove(blob.blob_hash)
|
||||
defer.returnValue(False)
|
||||
|
||||
log.debug('Download started: %s', blob.blob_hash)
|
||||
blob.set_length(response.length)
|
||||
yield self.client.collect(response, writer.write)
|
||||
defer.returnValue(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_stream(self, stream_hash, sd_hash):
|
||||
blobs = yield self.blob_manager.storage.get_blobs_for_stream(stream_hash)
|
||||
blob_hashes = [
|
||||
b.blob_hash for b in blobs if b.blob_hash is not None and b.blob_hash not in self.blob_hashes
|
||||
]
|
||||
self.blob_hashes.extend(blob_hashes)
|
||||
if sd_hash not in self.sd_hashes:
|
||||
self.sd_hashes.append(sd_hash)
|
||||
if blob_hashes[0] not in self.head_blob_hashes:
|
||||
self.head_blob_hashes.append(blob_hashes[0])
|
||||
yield self.start()
|
||||
|
||||
|
||||
def url_for(server, blob_hash=''):
|
||||
return 'http://{}/{}'.format(server, blob_hash)
|
||||
@@ -1,17 +0,0 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_read_handle(path):
|
||||
"""
|
||||
Get os independent read handle for a file
|
||||
"""
|
||||
|
||||
if os.name == "nt":
|
||||
file_mode = 'rb'
|
||||
else:
|
||||
file_mode = 'r'
|
||||
read_handle = open(path, file_mode)
|
||||
yield read_handle
|
||||
read_handle.close()
|
||||
@@ -1,54 +0,0 @@
|
||||
import platform
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
from urllib2 import urlopen, URLError
|
||||
from lbryschema import __version__ as lbryschema_version
|
||||
from lbryum import __version__ as LBRYUM_VERSION
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from lbrynet.conf import ROOT_DIR
|
||||
|
||||
|
||||
def get_lbrynet_version():
|
||||
if build_type.BUILD == "dev":
|
||||
try:
|
||||
with open(os.devnull, 'w') as devnull:
|
||||
git_dir = ROOT_DIR + '/.git'
|
||||
return subprocess.check_output(
|
||||
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
|
||||
stderr=devnull
|
||||
).strip().lstrip('v')
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
print "failed to get version from git"
|
||||
return lbrynet_version
|
||||
|
||||
|
||||
def get_platform(get_ip=True):
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": get_lbrynet_version(),
|
||||
"lbryum_version": LBRYUM_VERSION,
|
||||
"lbryschema_version": lbryschema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
|
||||
# TODO: remove this from get_platform and add a get_external_ip function using treq
|
||||
if get_ip:
|
||||
try:
|
||||
response = json.loads(urlopen("https://api.lbry.io/ip").read())
|
||||
if not response['success']:
|
||||
raise URLError("failed to get external ip")
|
||||
p['ip'] = response['data']['ip']
|
||||
except (URLError, AssertionError):
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
@@ -1,8 +0,0 @@
|
||||
"""
|
||||
Classes and functions for dealing with Crypt Streams.
|
||||
|
||||
Crypt Streams are encrypted blobs and metadata tying those blobs together. At least some of the
|
||||
metadata is generally stored in a Stream Descriptor File, for example containing a public key
|
||||
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
|
||||
may or may not be present.
|
||||
"""
|
||||
@@ -1,106 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import inspect
|
||||
import logging
|
||||
TRACE = 5
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
|
||||
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to subtitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
@@ -1,726 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
from hashlib import sha256
|
||||
import treq
|
||||
import math
|
||||
import binascii
|
||||
from twisted.internet import defer, threads, reactor, error
|
||||
from txupnp.upnp import UPnP
|
||||
from lbryum.simple_config import SimpleConfig
|
||||
from lbryum.constants import HEADERS_URL, HEADER_SIZE
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.utils import DeferredDict
|
||||
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.core.RateLimiter import RateLimiter
|
||||
from lbrynet.core.BlobManager import DiskBlobManager
|
||||
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
|
||||
from lbrynet.core.Wallet import LBRYumWallet
|
||||
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
|
||||
from lbrynet.daemon.Component import Component
|
||||
from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager
|
||||
from lbrynet.database.storage import SQLiteStorage
|
||||
from lbrynet.dht import node, hashannouncer
|
||||
from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.reflector import ServerFactory as reflector_server_factory
|
||||
from lbrynet.txlbryum.factory import StratumClient
|
||||
from lbrynet.core.utils import generate_id
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# settings must be initialized before this file is imported
|
||||
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
HEADERS_COMPONENT = "blockchain_headers"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_IDENTIFIER_COMPONENT = "stream_identifier"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
REFLECTOR_COMPONENT = "reflector"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
RATE_LIMITER_COMPONENT = "rate_limiter"
|
||||
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
|
||||
|
||||
|
||||
def get_wallet_config():
|
||||
wallet_type = GCS('wallet')
|
||||
if wallet_type == conf.LBRYCRD_WALLET:
|
||||
raise ValueError('LBRYcrd Wallet is no longer supported')
|
||||
elif wallet_type != conf.LBRYUM_WALLET:
|
||||
raise ValueError('Wallet Type {} is not valid'.format(wallet_type))
|
||||
lbryum_servers = {address: {'t': str(port)}
|
||||
for address, port in GCS('lbryum_servers')}
|
||||
config = {
|
||||
'auto_connect': True,
|
||||
'chain': GCS('blockchain_name'),
|
||||
'default_servers': lbryum_servers
|
||||
}
|
||||
if 'use_keyring' in conf.settings:
|
||||
config['use_keyring'] = GCS('use_keyring')
|
||||
if conf.settings['lbryum_wallet_dir']:
|
||||
config['lbryum_path'] = GCS('lbryum_wallet_dir')
|
||||
return config
|
||||
|
||||
|
||||
class ConfigSettings(object):
|
||||
@staticmethod
|
||||
def get_conf_setting(setting_name):
|
||||
return conf.settings[setting_name]
|
||||
|
||||
@staticmethod
|
||||
def get_blobfiles_dir():
|
||||
if conf.settings['BLOBFILES_DIR'] == "blobfiles":
|
||||
return os.path.join(GCS("data_dir"), "blobfiles")
|
||||
else:
|
||||
log.info("Using non-default blobfiles directory: %s", conf.settings['BLOBFILES_DIR'])
|
||||
return conf.settings['BLOBFILES_DIR']
|
||||
|
||||
@staticmethod
|
||||
def get_node_id():
|
||||
return conf.settings.node_id
|
||||
|
||||
@staticmethod
|
||||
def get_external_ip():
|
||||
from lbrynet.core.system_info import get_platform
|
||||
platform = get_platform(get_ip=True)
|
||||
return platform['ip']
|
||||
|
||||
|
||||
# Shorthand for common ConfigSettings methods
|
||||
CS = ConfigSettings
|
||||
GCS = ConfigSettings.get_conf_setting
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.storage = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.storage
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 9
|
||||
|
||||
@staticmethod
|
||||
def get_revision_filename():
|
||||
return conf.settings.get_db_revision_filename()
|
||||
|
||||
@staticmethod
|
||||
def _write_db_revision_file(version_num):
|
||||
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
|
||||
db_revision.write(str(version_num))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
# check directories exist, create them if they don't
|
||||
log.info("Loading databases")
|
||||
|
||||
if not os.path.exists(GCS('download_directory')):
|
||||
os.mkdir(GCS('download_directory'))
|
||||
|
||||
if not os.path.exists(GCS('data_dir')):
|
||||
os.mkdir(GCS('data_dir'))
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.debug("Created the db revision file: %s", self.get_revision_filename())
|
||||
|
||||
if not os.path.exists(CS.get_blobfiles_dir()):
|
||||
os.mkdir(CS.get_blobfiles_dir())
|
||||
log.debug("Created the blobfile directory: %s", str(CS.get_blobfiles_dir()))
|
||||
|
||||
if not os.path.exists(self.get_revision_filename()):
|
||||
log.warning("db_revision file not found. Creating it")
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
|
||||
# check the db migration and run any needed migrations
|
||||
with open(self.get_revision_filename(), "r") as revision_read_handle:
|
||||
old_revision = int(revision_read_handle.read().strip())
|
||||
|
||||
if old_revision > self.get_current_db_revision():
|
||||
raise Exception('This version of lbrynet is not compatible with the database\n'
|
||||
'Your database is revision %i, expected %i' %
|
||||
(old_revision, self.get_current_db_revision()))
|
||||
if old_revision < self.get_current_db_revision():
|
||||
from lbrynet.database.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
yield threads.deferToThread(
|
||||
dbmigrator.migrate_db, GCS('data_dir'), old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
|
||||
# start SQLiteStorage
|
||||
self.storage = SQLiteStorage(GCS('data_dir'))
|
||||
yield self.storage.setup()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.storage.stop()
|
||||
self.storage = None
|
||||
|
||||
|
||||
class HeadersComponent(Component):
|
||||
component_name = HEADERS_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.config = SimpleConfig(get_wallet_config())
|
||||
self._downloading_headers = None
|
||||
self._headers_progress_percent = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def get_status(self):
|
||||
return {} if not self._downloading_headers else {
|
||||
'downloading_headers': self._downloading_headers,
|
||||
'download_progress': self._headers_progress_percent
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def fetch_headers_from_s3(self):
|
||||
local_header_size = self.local_header_file_size()
|
||||
self._headers_progress_percent = 0.0
|
||||
resume_header = {"Range": "bytes={}-".format(local_header_size)}
|
||||
response = yield treq.get(HEADERS_URL, headers=resume_header)
|
||||
final_size_after_download = response.length + local_header_size
|
||||
|
||||
def collector(data, h_file, start_size):
|
||||
h_file.write(data)
|
||||
local_size = float(h_file.tell())
|
||||
final_size = float(final_size_after_download)
|
||||
self._headers_progress_percent = math.ceil((local_size - start_size) / (final_size - start_size) * 100)
|
||||
|
||||
if response.code == 406: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
# should have something to download and a final length divisible by the header size
|
||||
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
|
||||
s3_height = (final_size_after_download / HEADER_SIZE) - 1
|
||||
local_height = self.local_header_file_height()
|
||||
if s3_height > local_height:
|
||||
if local_header_size:
|
||||
log.info("Resuming download of %i bytes from s3", response.length)
|
||||
with open(os.path.join(self.config.path, "blockchain_headers"), "a+b") as headers_file:
|
||||
yield treq.collect(response, lambda d: collector(d, headers_file, local_header_size))
|
||||
else:
|
||||
with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file:
|
||||
yield treq.collect(response, lambda d: collector(d, headers_file, 0))
|
||||
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
|
||||
self._check_header_file_integrity()
|
||||
else:
|
||||
log.warning("s3 is more out of date than we are")
|
||||
else:
|
||||
log.error("invalid size for headers from s3")
|
||||
|
||||
def local_header_file_height(self):
|
||||
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
|
||||
|
||||
def local_header_file_size(self):
|
||||
headers_path = os.path.join(self.config.path, "blockchain_headers")
|
||||
if os.path.isfile(headers_path):
|
||||
return os.stat(headers_path).st_size
|
||||
return 0
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_remote_height(self, server, port):
|
||||
connected = defer.Deferred()
|
||||
connected.addTimeout(3, reactor, lambda *_: None)
|
||||
client = StratumClient(connected)
|
||||
reactor.connectTCP(server, port, client)
|
||||
yield connected
|
||||
remote_height = yield client.blockchain_block_get_server_height()
|
||||
client.client.transport.loseConnection()
|
||||
defer.returnValue(remote_height)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def should_download_headers_from_s3(self):
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
defer.returnValue(False)
|
||||
self._check_header_file_integrity()
|
||||
s3_headers_depth = conf.settings['s3_headers_depth']
|
||||
if not s3_headers_depth:
|
||||
defer.returnValue(False)
|
||||
local_height = self.local_header_file_height()
|
||||
for server_url in self.config.get('default_servers'):
|
||||
port = int(self.config.get('default_servers')[server_url]['t'])
|
||||
try:
|
||||
remote_height = yield self.get_remote_height(server_url, port)
|
||||
log.info("%s:%i height: %i, local height: %s", server_url, port, remote_height, local_height)
|
||||
if remote_height > (local_height + s3_headers_depth):
|
||||
defer.returnValue(True)
|
||||
except Exception as err:
|
||||
log.warning("error requesting remote height from %s:%i - %s", server_url, port, err)
|
||||
defer.returnValue(False)
|
||||
|
||||
def _check_header_file_integrity(self):
|
||||
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
return
|
||||
hashsum = sha256()
|
||||
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
|
||||
checksum_length_in_bytes = checksum_height * HEADER_SIZE
|
||||
if self.local_header_file_size() < checksum_length_in_bytes:
|
||||
return
|
||||
headers_path = os.path.join(self.config.path, "blockchain_headers")
|
||||
with open(headers_path, "rb") as headers_file:
|
||||
hashsum.update(headers_file.read(checksum_length_in_bytes))
|
||||
current_checksum = hashsum.hexdigest()
|
||||
if current_checksum != checksum:
|
||||
msg = "Expected checksum {}, got {}".format(checksum, current_checksum)
|
||||
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
|
||||
log.warning("Deleting header file so it can be downloaded again.")
|
||||
os.unlink(headers_path)
|
||||
elif (self.local_header_file_size() % HEADER_SIZE) != 0:
|
||||
log.warning("Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint.")
|
||||
with open(headers_path, "rb+") as headers_file:
|
||||
headers_file.truncate(checksum_length_in_bytes)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
self._downloading_headers = yield self.should_download_headers_from_s3()
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
yield self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self._downloading_headers = False
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
component_name = WALLET_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, HEADERS_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.wallet = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.wallet
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
if self.wallet:
|
||||
local_height = self.wallet.network.get_local_height()
|
||||
remote_height = self.wallet.network.get_server_height()
|
||||
best_hash = yield self.wallet.get_best_blockhash()
|
||||
defer.returnValue({
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet.wallet.use_encryption,
|
||||
'is_locked': not self.wallet.is_wallet_unlocked,
|
||||
})
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
config = get_wallet_config()
|
||||
self.wallet = LBRYumWallet(storage, config)
|
||||
yield self.wallet.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.wallet.stop()
|
||||
self.wallet = None
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, DHT_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.blob_manager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.blob_manager
|
||||
|
||||
def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore)
|
||||
return self.blob_manager.setup()
|
||||
|
||||
def stop(self):
|
||||
return self.blob_manager.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = yield self.blob_manager.storage.count_finished_blobs()
|
||||
defer.returnValue({
|
||||
'finished_blobs': count
|
||||
})
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
component_name = DHT_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.dht_node = None
|
||||
self.upnp_component = None
|
||||
self.external_udp_port = None
|
||||
self.external_peer_port = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.dht_node
|
||||
|
||||
def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(CS.get_node_id()),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", GCS("peer_port"))
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", GCS("dht_node_port"))
|
||||
node_id = CS.get_node_id()
|
||||
if node_id is None:
|
||||
node_id = generate_id()
|
||||
|
||||
self.dht_node = node.Node(
|
||||
node_id=node_id,
|
||||
udpPort=GCS('dht_node_port'),
|
||||
externalUDPPort=self.external_udp_port,
|
||||
externalIP=self.upnp_component.external_ip,
|
||||
peerPort=self.external_peer_port
|
||||
)
|
||||
|
||||
self.dht_node.start_listening()
|
||||
yield self.dht_node._protocol._listening
|
||||
d = self.dht_node.joinNetwork(GCS('known_dht_nodes'))
|
||||
d.addCallback(lambda _: self.dht_node.start_looping_calls())
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
log.info("Started the dht")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.dht_node.stop()
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
component_name = HASH_ANNOUNCER_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.hash_announcer = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.hash_announcer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage)
|
||||
yield self.hash_announcer.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.hash_announcer.stop()
|
||||
|
||||
def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
|
||||
}
|
||||
|
||||
|
||||
class RateLimiterComponent(Component):
|
||||
component_name = RATE_LIMITER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.rate_limiter = RateLimiter()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.rate_limiter
|
||||
|
||||
def start(self):
|
||||
self.rate_limiter.start()
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
self.rate_limiter.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class StreamIdentifierComponent(Component):
|
||||
component_name = STREAM_IDENTIFIER_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.sd_identifier
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
|
||||
add_lbry_file_to_sd_identifier(self.sd_identifier)
|
||||
file_saver_factory = EncryptedFileSaverFactory(
|
||||
dht_node.peer_finder,
|
||||
rate_limiter,
|
||||
blob_manager,
|
||||
storage,
|
||||
wallet,
|
||||
GCS('download_directory')
|
||||
)
|
||||
yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class PaymentRateComponent(Component):
|
||||
component_name = PAYMENT_RATE_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.payment_rate_manager = OnlyFreePaymentsManager()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.payment_rate_manager
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
|
||||
STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.file_manager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.file_manager
|
||||
|
||||
def get_status(self):
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.lbry_files)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT)
|
||||
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
|
||||
log.info('Starting the file manager')
|
||||
self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier)
|
||||
yield self.file_manager.setup()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.file_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
component_name = PEER_PROTOCOL_SERVER_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
|
||||
PAYMENT_RATE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.lbry_server_port = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.lbry_server_port
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
peer_port = GCS('peer_port')
|
||||
query_handlers = {
|
||||
handler.get_primary_query_identifier(): handler for handler in [
|
||||
BlobRequestHandlerFactory(
|
||||
self.component_manager.get_component(BLOB_COMPONENT),
|
||||
wallet,
|
||||
self.component_manager.get_component(PAYMENT_RATE_COMPONENT),
|
||||
self.component_manager.analytics_manager
|
||||
),
|
||||
wallet.get_wallet_info_query_handler_factory(),
|
||||
]
|
||||
}
|
||||
server_factory = ServerProtocolFactory(
|
||||
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
|
||||
self.component_manager.get_component(DHT_COMPONENT).peer_manager
|
||||
)
|
||||
|
||||
try:
|
||||
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
|
||||
upnp.upnp_redirects.get("TCP", peer_port))
|
||||
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
|
||||
" more details.", peer_port)
|
||||
log.error("%s", traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, old_port = None, self.lbry_server_port
|
||||
log.info('Stop listening on port %s', old_port.port)
|
||||
yield old_port.stopListening()
|
||||
|
||||
|
||||
class ReflectorComponent(Component):
|
||||
component_name = REFLECTOR_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.reflector_server_port = GCS('reflector_port')
|
||||
self.reflector_server = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.reflector_server
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
log.info("Starting reflector server")
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager)
|
||||
try:
|
||||
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
|
||||
log.info('Started reflector on port %s', self.reflector_server_port)
|
||||
except error.CannotListenError as e:
|
||||
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
|
||||
raise ValueError("{} lbrynet may already be running on your computer.".format(e))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
if self.reflector_server is not None:
|
||||
log.info("Stopping reflector server")
|
||||
self.reflector_server, p = None, self.reflector_server
|
||||
yield p.stopListening
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
component_name = UPNP_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self._int_peer_port = GCS('peer_port')
|
||||
self._int_dht_node_port = GCS('dht_node_port')
|
||||
self.use_upnp = GCS('use_upnp')
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
self.external_ip = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup_redirects(self):
|
||||
upnp_redirects = yield DeferredDict({
|
||||
"UDP": self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"),
|
||||
"TCP": self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
})
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
if not self.use_upnp:
|
||||
self.external_ip = CS.get_external_ip()
|
||||
return
|
||||
self.upnp = UPnP(self.component_manager.reactor, try_miniupnpc_fallback=True)
|
||||
try:
|
||||
found = yield self.upnp.discover()
|
||||
except Exception as err:
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
found = False
|
||||
if found and not self.upnp.miniupnpc_runner:
|
||||
log.info("set up redirects using txupnp")
|
||||
elif found and self.upnp.miniupnpc_runner:
|
||||
log.warning("failed to set up redirect with txupnp, miniupnpc fallback was successful")
|
||||
if found:
|
||||
try:
|
||||
self.external_ip = yield self.upnp.get_external_ip()
|
||||
yield self._setup_redirects()
|
||||
except Exception as err:
|
||||
log.warning("error trying to set up upnp: %s", err)
|
||||
self.external_ip = CS.get_external_ip()
|
||||
else:
|
||||
self.external_ip = CS.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
return defer.DeferredList(
|
||||
[self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()]
|
||||
)
|
||||
|
||||
|
||||
class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.exchange_rate_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
yield self.exchange_rate_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.exchange_rate_manager.stop()
|
||||
@@ -1,224 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import colorama
|
||||
from docopt import docopt
|
||||
from collections import OrderedDict
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.daemon.auth.client import JSONRPCException, LBRYAPIClient, AuthAPIClient
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
from lbrynet.core.system_info import get_platform
|
||||
from jsonrpc.common import RPCError
|
||||
from requests.exceptions import ConnectionError
|
||||
from urllib2 import URLError, HTTPError
|
||||
from httplib import UNAUTHORIZED
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = OrderedDict()
|
||||
for key, arg in parsed_args.iteritems():
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = guess_type(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def main():
|
||||
argv = sys.argv[1:]
|
||||
|
||||
# check if a config file has been specified. If so, shift
|
||||
# all the arguments so that the parsing can continue without
|
||||
# noticing
|
||||
if len(argv) and argv[0] == "--conf":
|
||||
if len(argv) < 2:
|
||||
print_error("No config file specified for --conf option")
|
||||
print_help()
|
||||
return
|
||||
|
||||
conf.conf_file = argv[1]
|
||||
argv = argv[2:]
|
||||
|
||||
if len(argv):
|
||||
method, args = argv[0], argv[1:]
|
||||
else:
|
||||
print_help()
|
||||
return
|
||||
|
||||
if method in ['help', '--help', '-h']:
|
||||
if len(args) == 1:
|
||||
print_help_for_command(args[0])
|
||||
else:
|
||||
print_help()
|
||||
return
|
||||
|
||||
elif method in ['version', '--version']:
|
||||
print utils.json_dumps_pretty(get_platform(get_ip=False))
|
||||
return
|
||||
|
||||
if method not in Daemon.callable_methods:
|
||||
if method not in Daemon.deprecated_methods:
|
||||
print_error("\"%s\" is not a valid command." % method)
|
||||
return
|
||||
new_method = Daemon.deprecated_methods[method]._new_command
|
||||
print_error("\"%s\" is deprecated, using \"%s\"." % (method, new_method))
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
|
||||
parsed = docopt(fn.__doc__, args)
|
||||
kwargs = set_kwargs(parsed)
|
||||
colorama.init()
|
||||
conf.initialize_settings()
|
||||
|
||||
try:
|
||||
api = LBRYAPIClient.get_client()
|
||||
api.status()
|
||||
except (URLError, ConnectionError) as err:
|
||||
if isinstance(err, HTTPError) and err.code == UNAUTHORIZED:
|
||||
api = AuthAPIClient.config()
|
||||
# this can happen if the daemon is using auth with the --http-auth flag
|
||||
# when the config setting is to not use it
|
||||
try:
|
||||
api.status()
|
||||
except:
|
||||
print_error("Daemon requires authentication, but none was provided.",
|
||||
suggest_help=False)
|
||||
return 1
|
||||
else:
|
||||
print_error("Could not connect to daemon. Are you sure it's running?",
|
||||
suggest_help=False)
|
||||
return 1
|
||||
|
||||
# TODO: check if port is bound. Error if its not
|
||||
|
||||
try:
|
||||
result = api.call(method, kwargs)
|
||||
if isinstance(result, basestring):
|
||||
# printing the undumped string is prettier
|
||||
print result
|
||||
else:
|
||||
print utils.json_dumps_pretty(result)
|
||||
except (RPCError, KeyError, JSONRPCException, HTTPError) as err:
|
||||
if isinstance(err, HTTPError):
|
||||
error_body = err.read()
|
||||
try:
|
||||
error_data = json.loads(error_body)
|
||||
except ValueError:
|
||||
print (
|
||||
"There was an error, and the response was not valid JSON.\n" +
|
||||
"Raw JSONRPC response:\n" + error_body
|
||||
)
|
||||
return 1
|
||||
|
||||
print_error(error_data['error']['message'] + "\n", suggest_help=False)
|
||||
|
||||
if 'data' in error_data['error'] and 'traceback' in error_data['error']['data']:
|
||||
print "Here's the traceback for the error you encountered:"
|
||||
print "\n".join(error_data['error']['data']['traceback'])
|
||||
|
||||
print_help_for_command(method)
|
||||
elif isinstance(err, RPCError):
|
||||
print_error(err.msg, suggest_help=False)
|
||||
# print_help_for_command(method)
|
||||
else:
|
||||
print_error("Something went wrong\n", suggest_help=False)
|
||||
print str(err)
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
def guess_type(x, key=None):
|
||||
if not isinstance(x, (unicode, str)):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x in ('true', 'True', 'TRUE'):
|
||||
return True
|
||||
if x in ('false', 'False', 'FALSE'):
|
||||
return False
|
||||
if '.' in x:
|
||||
try:
|
||||
return float(x)
|
||||
except ValueError:
|
||||
# not a float
|
||||
pass
|
||||
try:
|
||||
return int(x)
|
||||
except ValueError:
|
||||
return x
|
||||
|
||||
|
||||
def print_help_suggestion():
|
||||
print "See `{} help` for more information.".format(os.path.basename(sys.argv[0]))
|
||||
|
||||
|
||||
def print_error(message, suggest_help=True):
|
||||
error_style = colorama.Style.BRIGHT + colorama.Fore.RED
|
||||
print error_style + "ERROR: " + message + colorama.Style.RESET_ALL
|
||||
if suggest_help:
|
||||
print_help_suggestion()
|
||||
|
||||
|
||||
def print_help():
|
||||
print "\n".join([
|
||||
"NAME",
|
||||
" lbrynet-cli - LBRY command line client.",
|
||||
"",
|
||||
"USAGE",
|
||||
" lbrynet-cli [--conf <config file>] <command> [<args>]",
|
||||
"",
|
||||
"EXAMPLES",
|
||||
" lbrynet-cli commands # list available commands",
|
||||
" lbrynet-cli status # get daemon status",
|
||||
" lbrynet-cli --conf ~/l1.conf status # like above but using ~/l1.conf as config file",
|
||||
" lbrynet-cli resolve_name what # resolve a name",
|
||||
" lbrynet-cli help resolve_name # get help for a command",
|
||||
])
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print "Help for %s method:\n%s" % (command, fn.__doc__)
|
||||
|
||||
|
||||
def wrap_list_to_term_width(l, width=None, separator=', ', prefix=''):
|
||||
if width is None:
|
||||
try:
|
||||
_, width = os.popen('stty size', 'r').read().split()
|
||||
width = int(width)
|
||||
except:
|
||||
pass
|
||||
if not width:
|
||||
width = 80
|
||||
|
||||
lines = []
|
||||
curr_line = ''
|
||||
for item in l:
|
||||
new_line = curr_line + item + separator
|
||||
if len(new_line) + len(prefix) > width:
|
||||
lines.append(curr_line)
|
||||
curr_line = item + separator
|
||||
else:
|
||||
curr_line = new_line
|
||||
lines.append(curr_line)
|
||||
|
||||
ret = prefix + ("\n" + prefix).join(lines)
|
||||
if ret.endswith(separator):
|
||||
ret = ret[:-len(separator)]
|
||||
return ret
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,233 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import code
|
||||
import argparse
|
||||
import logging.handlers
|
||||
from exceptions import SystemExit
|
||||
from twisted.internet import defer, reactor, threads
|
||||
from lbrynet import analytics
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core import log_support
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
|
||||
get_client = LBRYAPIClient.get_client
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
|
||||
def color(msg, c="white"):
|
||||
_colors = {
|
||||
"normal": (0, 37),
|
||||
"underlined": (2, 37),
|
||||
"red": (1, 31),
|
||||
"green": (1, 32),
|
||||
"yellow": (1, 33),
|
||||
"blue": (1, 33),
|
||||
"magenta": (1, 34),
|
||||
"cyan": (1, 35),
|
||||
"white": (1, 36),
|
||||
"grey": (1, 37)
|
||||
}
|
||||
i, j = _colors[c]
|
||||
return "\033[%i;%i;40m%s\033[0m" % (i, j, msg)
|
||||
|
||||
|
||||
logo = """\
|
||||
╓▄█▄ç
|
||||
,▄█▓▓▀▀▀▓▓▓▌▄,
|
||||
▄▄▓▓▓▀¬ ╙▀█▓▓▓▄▄
|
||||
,▄█▓▓▀▀ ^▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀` ╙▀█▓▓▓▄▄
|
||||
╓▄▓▓▓▀╙ ▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀ ╙▀▓▓
|
||||
╓▄▓▓▓▀` ▄█▓▓▓▀
|
||||
▓▓█▀ ,▄▓▓▓▀╙
|
||||
▓▓m ╟▌▄, ▄█▓▓█▀ ,,╓µ
|
||||
▓▓m ^▀█▓▓▓▄▄ ╓▄▓▓▓▀╙ █▓▓▓▓▓▀
|
||||
▓▓Q '▀▀▓▓▓▌▄, ,▄█▓▓█▀ ▄█▓▓▓▓▓▀
|
||||
▀▓▓▓▌▄, ╙▀█▓▓█▄╗ ╓▄▓▓▓▀ ╓▄▓▓▓▀▀ ▀▀
|
||||
╙▀█▓▓█▄╗ ^▀▀▓▓▓▌▄▄█▓▓▀▀ ▄█▓▓█▀`
|
||||
'▀▀▓▓▓▌▄, ╙▀██▀` ╓▄▓▓▓▀╙
|
||||
╙▀█▓▓█▄╥ ,▄█▓▓▀▀
|
||||
└▀▀▓▓▓▌▄ ▄▒▓▓▓▀╙
|
||||
╙▀█▓▓█▓▓▓▀▀
|
||||
╙▀▀`
|
||||
|
||||
"""
|
||||
else:
|
||||
def color(msg, c=None):
|
||||
return msg
|
||||
|
||||
logo = """\
|
||||
'.
|
||||
++++.
|
||||
+++,;+++,
|
||||
:+++ :+++:
|
||||
+++ ,+++;
|
||||
'++; .+++'
|
||||
`+++ `++++
|
||||
+++. `++++
|
||||
;+++ ++++
|
||||
+++ +++
|
||||
+++: '+
|
||||
,+++ +++
|
||||
+++` +++:
|
||||
`+' ,+++
|
||||
`+ + +++
|
||||
`+ +++ '++' :'+++:
|
||||
`+ ++++ `+++ ++++
|
||||
`+ ++++ +++. :+++'
|
||||
`+, ++++ ;+++ +++++
|
||||
`+++, ++++ +++ +++; +
|
||||
,+++, ++++ +++: .+++
|
||||
,+++: '++++++ +++`
|
||||
,+++: '++ '++'
|
||||
,+++: `+++
|
||||
.+++; +++,
|
||||
.+++; ;+++
|
||||
.+++; +++
|
||||
`+++++:
|
||||
`++
|
||||
|
||||
"""
|
||||
|
||||
welcometext = """\
|
||||
For a list of available commands:
|
||||
>>>help()
|
||||
|
||||
To see the documentation for a given command:
|
||||
>>>help("resolve")
|
||||
|
||||
To exit:
|
||||
>>>exit()
|
||||
"""
|
||||
|
||||
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
|
||||
welcome += "\n".join(["{:<60}".format(w) for w in welcometext.splitlines()])
|
||||
welcome += "\n%s" % ("*" * 60)
|
||||
welcome = color(welcome, "grey")
|
||||
banner = color(logo, "green") + color(welcome, "grey")
|
||||
|
||||
|
||||
def get_methods(daemon):
|
||||
locs = {}
|
||||
|
||||
def wrapped(name, fn):
|
||||
client = get_client()
|
||||
_fn = getattr(client, name)
|
||||
_fn.__doc__ = fn.__doc__
|
||||
return {name: _fn}
|
||||
|
||||
for method_name, method in daemon.callable_methods.iteritems():
|
||||
locs.update(wrapped(method_name, method))
|
||||
return locs
|
||||
|
||||
|
||||
def run_terminal(callable_methods, started_daemon, quiet=False):
|
||||
locs = {}
|
||||
locs.update(callable_methods)
|
||||
|
||||
def help(method_name=None):
|
||||
if not method_name:
|
||||
print "Available api functions: "
|
||||
for name in callable_methods:
|
||||
print "\t%s" % name
|
||||
return
|
||||
if method_name not in callable_methods:
|
||||
print "\"%s\" is not a recognized api function"
|
||||
return
|
||||
print callable_methods[method_name].__doc__
|
||||
return
|
||||
|
||||
locs.update({'help': help})
|
||||
|
||||
if started_daemon:
|
||||
def exit(status=None):
|
||||
if not quiet:
|
||||
print "Stopping lbrynet-daemon..."
|
||||
callable_methods['daemon_stop']()
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
else:
|
||||
def exit(status=None):
|
||||
try:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
except Exception as err:
|
||||
print "error stopping reactor: ", err
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
|
||||
code.interact(banner if not quiet else "", local=locs)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_server_and_listen(use_auth, analytics_manager, quiet):
|
||||
log_support.configure_console()
|
||||
logging.getLogger("lbrynet").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("requests").setLevel(logging.CRITICAL)
|
||||
|
||||
analytics_manager.send_server_startup()
|
||||
yield Daemon().start_listening()
|
||||
|
||||
|
||||
def threaded_terminal(started_daemon, quiet):
|
||||
callable_methods = get_methods(Daemon)
|
||||
d = threads.deferToThread(run_terminal, callable_methods, started_daemon, quiet)
|
||||
d.addErrback(lambda err: err.trap(SystemExit))
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
def start_lbrynet_console(quiet, use_existing_daemon, useauth):
|
||||
if not utils.check_connection():
|
||||
print "Not connected to internet, unable to start"
|
||||
raise Exception("Not connected to internet, unable to start")
|
||||
if not quiet:
|
||||
print "Starting lbrynet-console..."
|
||||
try:
|
||||
get_client().status()
|
||||
d = defer.succeed(False)
|
||||
if not quiet:
|
||||
print "lbrynet-daemon is already running, connecting to it..."
|
||||
except:
|
||||
if not use_existing_daemon:
|
||||
if not quiet:
|
||||
print "Starting lbrynet-daemon..."
|
||||
analytics_manager = analytics.Manager.new_instance()
|
||||
d = start_server_and_listen(useauth, analytics_manager, quiet)
|
||||
else:
|
||||
raise Exception("cannot connect to an existing daemon instance, "
|
||||
"and set to not start a new one")
|
||||
d.addCallback(threaded_terminal, quiet)
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument(
|
||||
"--use_existing_daemon",
|
||||
help="Start lbrynet-daemon if it isn't already running",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="use_existing_daemon"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quiet", dest="quiet", action="store_true", default=False
|
||||
)
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
args = parser.parse_args()
|
||||
start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth)
|
||||
reactor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,105 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
|
||||
# This needs to happen before anything else
|
||||
if 'win' in sys.platform:
|
||||
import certifi
|
||||
os.environ['SSL_CERT_FILE'] = certifi.where()
|
||||
|
||||
from lbrynet.core import log_support
|
||||
|
||||
import argparse
|
||||
import logging.handlers
|
||||
|
||||
from twisted.internet import reactor
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils, system_info
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def test_internet_connection():
|
||||
return utils.check_connection()
|
||||
|
||||
|
||||
def start():
|
||||
"""The primary entry point for launching the daemon."""
|
||||
|
||||
# postpone loading the config file to after the CLI arguments
|
||||
# have been parsed, as they may contain an alternate config file location
|
||||
conf.initialize_settings(load_conf_file=False)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument(
|
||||
"--conf",
|
||||
help="specify an alternative configuration file",
|
||||
type=str,
|
||||
default=None
|
||||
)
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
parser.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
parser.add_argument(
|
||||
'--version', action="store_true",
|
||||
help='Show daemon version and quit'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
update_settings_from_args(args)
|
||||
|
||||
conf.settings.load_conf_file_settings()
|
||||
|
||||
if args.version:
|
||||
version = system_info.get_platform(get_ip=False)
|
||||
version['installation_id'] = conf.settings.installation_id
|
||||
print utils.json_dumps_pretty(version)
|
||||
return
|
||||
|
||||
lbrynet_log = conf.settings.get_log_filename()
|
||||
log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
|
||||
log_support.configure_loggly_handler()
|
||||
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
|
||||
|
||||
try:
|
||||
log.debug('Checking for an existing lbrynet daemon instance')
|
||||
JSONRPCProxy.from_url(conf.settings.get_api_connection_string()).status()
|
||||
log.info("lbrynet-daemon is already running")
|
||||
return
|
||||
except Exception:
|
||||
log.debug('No lbrynet instance found, continuing to start')
|
||||
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if test_internet_connection():
|
||||
daemon = Daemon()
|
||||
daemon.start_listening()
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
|
||||
def update_settings_from_args(args):
|
||||
if args.conf:
|
||||
conf.conf_file = args.conf
|
||||
|
||||
if args.useauth:
|
||||
conf.settings.update({
|
||||
'use_auth_http': args.useauth,
|
||||
}, data_types=(conf.TYPE_CLI,))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start()
|
||||
@@ -1,84 +0,0 @@
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.core import file_utils
|
||||
from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Publisher(object):
|
||||
def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate_id):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.storage = storage
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.wallet = wallet
|
||||
self.certificate_id = certificate_id
|
||||
self.lbry_file = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def create_and_publish_stream(self, name, bid, claim_dict, file_path, claim_address=None,
|
||||
change_address=None):
|
||||
"""Create lbry file and make claim"""
|
||||
log.info('Starting publish for %s', name)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception("File {} not found".format(file_path))
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception("Cannot publish empty file {}".format(file_path))
|
||||
|
||||
file_name = os.path.basename(file_path)
|
||||
with file_utils.get_read_handle(file_path) as read_handle:
|
||||
self.lbry_file = yield create_lbry_file(
|
||||
self.blob_manager, self.storage, self.payment_rate_manager, self.lbry_file_manager, file_name,
|
||||
read_handle
|
||||
)
|
||||
|
||||
if 'source' not in claim_dict['stream']:
|
||||
claim_dict['stream']['source'] = {}
|
||||
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = get_content_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
claim_out = yield self.make_claim(name, bid, claim_dict, claim_address, change_address)
|
||||
|
||||
# check if we have a file already for this claim (if this is a publish update with a new stream)
|
||||
old_stream_hashes = yield self.storage.get_old_stream_hashes_for_claim_id(
|
||||
claim_out['claim_id'], self.lbry_file.stream_hash
|
||||
)
|
||||
if old_stream_hashes:
|
||||
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
|
||||
list(self.lbry_file_manager.lbry_files)):
|
||||
yield self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False)
|
||||
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
|
||||
|
||||
yield self.storage.save_content_claim(
|
||||
self.lbry_file.stream_hash, "%s:%i" % (claim_out['txid'], claim_out['nout'])
|
||||
)
|
||||
defer.returnValue(claim_out)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def publish_stream(self, name, bid, claim_dict, stream_hash, claim_address=None, change_address=None):
|
||||
"""Make a claim without creating a lbry file"""
|
||||
claim_out = yield self.make_claim(name, bid, claim_dict, claim_address, change_address)
|
||||
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
|
||||
yield self.storage.save_content_claim(
|
||||
stream_hash, "%s:%i" % (claim_out['txid'], claim_out['nout'])
|
||||
)
|
||||
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
|
||||
defer.returnValue(claim_out)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def make_claim(self, name, bid, claim_dict, claim_address=None, change_address=None):
|
||||
claim_out = yield self.wallet.claim_name(name, bid, claim_dict,
|
||||
certificate_id=self.certificate_id,
|
||||
claim_address=claim_address,
|
||||
change_address=change_address)
|
||||
defer.returnValue(claim_out)
|
||||
|
||||
|
||||
def get_content_type(filename):
|
||||
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
|
||||
@@ -1,4 +0,0 @@
|
||||
from lbrynet import custom_logger
|
||||
import Components # register Component classes
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
get_client = LBRYAPIClient.get_client
|
||||
@@ -1,48 +0,0 @@
|
||||
import logging
|
||||
from zope.interface import implementer
|
||||
from twisted.cred import portal, checkers, credentials, error as cred_error
|
||||
from twisted.internet import defer
|
||||
from twisted.web import resource
|
||||
from lbrynet.daemon.auth.util import load_api_keys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@implementer(portal.IRealm)
|
||||
class HttpPasswordRealm(object):
|
||||
def __init__(self, resource):
|
||||
self.resource = resource
|
||||
|
||||
def requestAvatar(self, avatarId, mind, *interfaces):
|
||||
log.debug("Processing request for %s", avatarId)
|
||||
if resource.IResource in interfaces:
|
||||
return (resource.IResource, self.resource, lambda: None)
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@implementer(checkers.ICredentialsChecker)
|
||||
class PasswordChecker(object):
|
||||
credentialInterfaces = (credentials.IUsernamePassword,)
|
||||
|
||||
def __init__(self, passwords):
|
||||
self.passwords = passwords
|
||||
|
||||
@classmethod
|
||||
def load_file(cls, key_path):
|
||||
keys = load_api_keys(key_path)
|
||||
return cls.load(keys)
|
||||
|
||||
@classmethod
|
||||
def load(cls, password_dict):
|
||||
passwords = {key: password_dict[key].secret for key in password_dict}
|
||||
log.info("Loaded %i api key(s)", len(passwords))
|
||||
return cls(passwords)
|
||||
|
||||
def requestAvatarId(self, creds):
|
||||
if creds.username in self.passwords:
|
||||
pw = self.passwords.get(creds.username)
|
||||
pw_match = creds.checkPassword(pw)
|
||||
if pw_match:
|
||||
return defer.succeed(creds.username)
|
||||
log.warning('Incorrect username or password')
|
||||
return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
|
||||
@@ -1,139 +0,0 @@
|
||||
import os
|
||||
import json
|
||||
import urlparse
|
||||
import requests
|
||||
from requests.cookies import RequestsCookieJar
|
||||
import logging
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
from lbrynet import conf
|
||||
from lbrynet.daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
|
||||
def copy_cookies(cookies):
|
||||
result = RequestsCookieJar()
|
||||
result.update(cookies)
|
||||
return result
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
Exception.__init__(self)
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
class AuthAPIClient(object):
|
||||
def __init__(self, key, timeout, connection, count, cookies, url, login_url):
|
||||
self.__api_key = key
|
||||
self.__service_url = login_url
|
||||
self.__id_count = count
|
||||
self.__url = url
|
||||
self.__conn = connection
|
||||
self.__cookies = copy_cookies(cookies)
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
raise AttributeError(name)
|
||||
|
||||
def f(*args, **kwargs):
|
||||
return self.call(name, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
def call(self, method, params=None):
|
||||
params = params or {}
|
||||
self.__id_count += 1
|
||||
pre_auth_post_data = {
|
||||
'version': '2',
|
||||
'method': method,
|
||||
'params': params,
|
||||
'id': self.__id_count
|
||||
}
|
||||
to_auth = get_auth_message(pre_auth_post_data)
|
||||
pre_auth_post_data.update({'hmac': self.__api_key.get_hmac(to_auth)})
|
||||
post_data = json.dumps(pre_auth_post_data)
|
||||
cookies = copy_cookies(self.__cookies)
|
||||
req = requests.Request(
|
||||
method='POST', url=self.__service_url, data=post_data, cookies=cookies,
|
||||
headers={
|
||||
'Host': self.__url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
)
|
||||
http_response = self.__conn.send(req.prepare())
|
||||
if http_response is None:
|
||||
raise JSONRPCException({
|
||||
'code': -342, 'message': 'missing HTTP response from server'})
|
||||
http_response.raise_for_status()
|
||||
next_secret = http_response.headers.get(LBRY_SECRET, False)
|
||||
if next_secret:
|
||||
self.__api_key.secret = next_secret
|
||||
self.__cookies = copy_cookies(http_response.cookies)
|
||||
response = http_response.json()
|
||||
if response.get('error') is not None:
|
||||
raise JSONRPCException(response['error'])
|
||||
elif 'result' not in response:
|
||||
raise JSONRPCException({
|
||||
'code': -343, 'message': 'missing JSON-RPC result'})
|
||||
else:
|
||||
return response['result']
|
||||
|
||||
@classmethod
|
||||
def config(cls, key_name=None, key=None, pw_path=None, timeout=HTTP_TIMEOUT, connection=None, count=0,
|
||||
cookies=None, auth=None, url=None, login_url=None):
|
||||
|
||||
api_key_name = key_name or API_KEY_NAME
|
||||
pw_path = os.path.join(conf.settings['data_dir'], ".api_keys") if not pw_path else pw_path
|
||||
if not key:
|
||||
keys = load_api_keys(pw_path)
|
||||
api_key = keys.get(api_key_name, False)
|
||||
else:
|
||||
api_key = APIKey(name=api_key_name, secret=key)
|
||||
if login_url is None:
|
||||
service_url = "http://%s:%s@%s:%i/%s" % (api_key_name,
|
||||
api_key.secret,
|
||||
conf.settings['api_host'],
|
||||
conf.settings['api_port'],
|
||||
conf.settings['API_ADDRESS'])
|
||||
else:
|
||||
service_url = login_url
|
||||
id_count = count
|
||||
|
||||
if auth is None and connection is None and cookies is None and url is None:
|
||||
# This is a new client instance, start an authenticated session
|
||||
url = urlparse.urlparse(service_url)
|
||||
conn = requests.Session()
|
||||
req = requests.Request(method='POST',
|
||||
url=service_url,
|
||||
headers={'Host': url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'},)
|
||||
r = req.prepare()
|
||||
http_response = conn.send(r)
|
||||
cookies = RequestsCookieJar()
|
||||
cookies.update(http_response.cookies)
|
||||
uid = cookies.get(TWISTED_SESSION)
|
||||
api_key = APIKey.new(seed=uid)
|
||||
else:
|
||||
# This is a client that already has a session, use it
|
||||
conn = connection
|
||||
if not cookies.get(LBRY_SECRET):
|
||||
raise Exception("Missing cookie")
|
||||
secret = cookies.get(LBRY_SECRET)
|
||||
api_key = APIKey(secret, api_key_name)
|
||||
return cls(api_key, timeout, conn, id_count, cookies, url, service_url)
|
||||
|
||||
|
||||
class LBRYAPIClient(object):
|
||||
@staticmethod
|
||||
def get_client():
|
||||
if not conf.settings:
|
||||
conf.initialize_settings()
|
||||
return AuthAPIClient.config() if conf.settings['use_auth_http'] else \
|
||||
JSONRPCProxy.from_url(conf.settings.get_api_connection_string())
|
||||
@@ -1,38 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from .auth import PasswordChecker, HttpPasswordRealm
|
||||
from .util import initialize_api_key_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AuthJSONRPCResource(resource.Resource):
|
||||
def __init__(self, protocol):
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild("", protocol)
|
||||
self.putChild(conf.settings['API_ADDRESS'], protocol)
|
||||
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
def getServerFactory(self):
|
||||
if conf.settings['use_auth_http']:
|
||||
log.info("Using authenticated API")
|
||||
pw_path = os.path.join(conf.settings['data_dir'], ".api_keys")
|
||||
initialize_api_key_file(pw_path)
|
||||
checker = PasswordChecker.load_file(pw_path)
|
||||
realm = HttpPasswordRealm(self)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
factory = guard.BasicCredentialFactory('Login to lbrynet api')
|
||||
root = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ])
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
root = self
|
||||
return server.Site(root)
|
||||
@@ -1,581 +0,0 @@
|
||||
import logging
|
||||
import urlparse
|
||||
import json
|
||||
import inspect
|
||||
import signal
|
||||
|
||||
from decimal import Decimal
|
||||
from functools import wraps
|
||||
from zope.interface import implements
|
||||
from twisted.web import server, resource
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.error import ConnectionDone, ConnectionLost
|
||||
from txjsonrpc import jsonrpclib
|
||||
from traceback import format_exc
|
||||
|
||||
from lbrynet import conf, analytics
|
||||
from lbrynet.core.Error import InvalidAuthenticationToken
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet
|
||||
from lbrynet.core.looping_call_manager import LoopingCallManager
|
||||
from lbrynet.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.undecorated import undecorated
|
||||
from .util import APIKey, get_auth_message
|
||||
from .client import LBRY_SECRET
|
||||
from .factory import AuthJSONRPCResource
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
EMPTY_PARAMS = [{}]
|
||||
|
||||
|
||||
class JSONRPCError(object):
|
||||
# http://www.jsonrpc.org/specification#error_object
|
||||
CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text.
|
||||
CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object.
|
||||
CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available.
|
||||
CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s).
|
||||
CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?)
|
||||
CODE_APPLICATION_ERROR = -32500 # Generic error with our app??
|
||||
CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed
|
||||
|
||||
MESSAGES = {
|
||||
CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
|
||||
CODE_INVALID_REQUEST: "JSON data is not a valid Request",
|
||||
CODE_METHOD_NOT_FOUND: "Method Not Found",
|
||||
CODE_INVALID_PARAMS: "Invalid Params",
|
||||
CODE_INTERNAL_ERROR: "Internal Error",
|
||||
CODE_AUTHENTICATION_ERROR: "Authentication Failed",
|
||||
}
|
||||
|
||||
HTTP_CODES = {
|
||||
CODE_INVALID_REQUEST: 400,
|
||||
CODE_PARSE_ERROR: 400,
|
||||
CODE_INVALID_PARAMS: 400,
|
||||
CODE_METHOD_NOT_FOUND: 404,
|
||||
CODE_INTERNAL_ERROR: 500,
|
||||
CODE_APPLICATION_ERROR: 500,
|
||||
CODE_AUTHENTICATION_ERROR: 401,
|
||||
}
|
||||
|
||||
def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
|
||||
assert isinstance(code, (int, long)), "'code' must be an int"
|
||||
assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
|
||||
self.code = code
|
||||
if message is None:
|
||||
message = self.MESSAGES[code] if code in self.MESSAGES else "API Error"
|
||||
self.message = message
|
||||
self.data = {} if data is None else data
|
||||
self.traceback = []
|
||||
if traceback is not None:
|
||||
trace_lines = traceback.split("\n")
|
||||
for i, t in enumerate(trace_lines):
|
||||
if "--- <exception caught here> ---" in t:
|
||||
if len(trace_lines) > i + 1:
|
||||
self.traceback = [j for j in trace_lines[i+1:] if j]
|
||||
break
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'code': self.code,
|
||||
'message': self.message,
|
||||
'data': self.traceback
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def create_from_exception(cls, exception, code=CODE_APPLICATION_ERROR, traceback=None):
|
||||
return cls(exception.message, code=code, traceback=traceback)
|
||||
|
||||
|
||||
def default_decimal(obj):
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
|
||||
|
||||
class UnknownAPIMethodError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def trap(err, *to_trap):
|
||||
err.trap(*to_trap)
|
||||
|
||||
|
||||
def jsonrpc_dumps_pretty(obj, **kwargs):
|
||||
try:
|
||||
id_ = kwargs.pop("id")
|
||||
except KeyError:
|
||||
id_ = None
|
||||
|
||||
if isinstance(obj, JSONRPCError):
|
||||
data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
|
||||
else:
|
||||
data = {"jsonrpc": "2.0", "result": obj, "id": id_}
|
||||
|
||||
return json.dumps(data, cls=jsonrpclib.JSONRPCEncoder, sort_keys=True, indent=2,
|
||||
separators=(',', ': '), **kwargs) + "\n"
|
||||
|
||||
|
||||
class JSONRPCServerType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
klass.callable_methods = {}
|
||||
klass.deprecated_methods = {}
|
||||
|
||||
for methodname in dir(klass):
|
||||
if methodname.startswith("jsonrpc_"):
|
||||
method = getattr(klass, methodname)
|
||||
if not hasattr(method, '_deprecated'):
|
||||
klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
else:
|
||||
klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
return klass
|
||||
|
||||
|
||||
class AuthorizedBase(object):
|
||||
__metaclass__ = JSONRPCServerType
|
||||
|
||||
@staticmethod
|
||||
def deprecated(new_command=None):
|
||||
def _deprecated_wrapper(f):
|
||||
f._new_command = new_command
|
||||
f._deprecated = True
|
||||
return f
|
||||
return _deprecated_wrapper
|
||||
|
||||
@staticmethod
|
||||
def requires(*components, **conditions):
|
||||
if conditions and ["conditions"] != conditions.keys():
|
||||
raise SyntaxError("invalid conditions argument")
|
||||
condition_names = conditions.get("conditions", [])
|
||||
|
||||
def _wrap(fn):
|
||||
@defer.inlineCallbacks
|
||||
@wraps(fn)
|
||||
def _inner(*args, **kwargs):
|
||||
component_manager = args[0].component_manager
|
||||
for condition_name in condition_names:
|
||||
condition_result, err_msg = yield component_manager.evaluate_condition(condition_name)
|
||||
if not condition_result:
|
||||
raise ComponentStartConditionNotMet(err_msg)
|
||||
if not component_manager.all_components_running(*components):
|
||||
raise ComponentsNotStarted("the following required components have not yet started: "
|
||||
"%s" % json.dumps(components))
|
||||
result = yield fn(*args, **kwargs)
|
||||
defer.returnValue(result)
|
||||
return _inner
|
||||
return _wrap
|
||||
|
||||
|
||||
class AuthJSONRPCServer(AuthorizedBase):
|
||||
"""
|
||||
Authorized JSONRPC server used as the base class for the LBRY API
|
||||
|
||||
API methods are named with a leading "jsonrpc_"
|
||||
|
||||
Attributes:
|
||||
sessions (dict): (dict): {<session id>: <lbrynet.daemon.auth.util.APIKey>}
|
||||
callable_methods (dict): {<api method name>: <api method>}
|
||||
|
||||
Authentication:
|
||||
If use_authentication is true, basic HTTP and HMAC authentication will be used for all requests and the
|
||||
service url will require a username and password.
|
||||
|
||||
To start an authenticated session a client sends an HTTP POST to <user>:<password>@<api host>:<api port>.
|
||||
If accepted, the server replies with a TWISTED_SESSION cookie containing a session id and the message "OK".
|
||||
The client initializes their shared secret for hmac to be the b64 encoded sha256 of their session id.
|
||||
|
||||
To send an authenticated request a client sends an HTTP POST to the auth api url with the TWISTED_SESSION
|
||||
cookie and includes a hmac token in the message using the previously set shared secret. If the token is valid
|
||||
the server will randomize the shared secret and return the new value under the LBRY_SECRET header, which the
|
||||
client uses to generate the token for their next request.
|
||||
"""
|
||||
implements(resource.IResource)
|
||||
|
||||
isLeaf = True
|
||||
allowed_during_startup = []
|
||||
component_attributes = {}
|
||||
|
||||
def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, to_skip=None,
|
||||
looping_calls=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
analytics_manager=self.analytics_manager,
|
||||
skip_components=to_skip or [],
|
||||
reactor=reactor
|
||||
)
|
||||
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).iteritems()})
|
||||
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).iteritems()}
|
||||
self._use_authentication = use_authentication or conf.settings['use_auth_http']
|
||||
self._component_setup_deferred = None
|
||||
self.announced_startup = False
|
||||
self.sessions = {}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_listening(self):
|
||||
from twisted.internet import reactor, error as tx_error
|
||||
|
||||
try:
|
||||
reactor.listenTCP(
|
||||
conf.settings['api_port'], self.get_server_factory(), interface=conf.settings['api_host']
|
||||
)
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
yield self.setup()
|
||||
self.analytics_manager.send_server_startup_success()
|
||||
except tx_error.CannotListenError:
|
||||
log.error('lbrynet API failed to bind TCP %s:%i for listening', conf.settings['api_host'],
|
||||
conf.settings['api_port'])
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except defer.CancelledError:
|
||||
log.info("shutting down before finished starting")
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except Exception as err:
|
||||
self.analytics_manager.send_server_startup_error(str(err))
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
def setup(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
|
||||
if not self.analytics_manager.is_started:
|
||||
self.analytics_manager.start()
|
||||
for lc_name, lc_time in self._looping_call_times.iteritems():
|
||||
self.looping_call_manager.start(lc_name, lc_time)
|
||||
|
||||
def update_attribute(setup_result, component):
|
||||
setattr(self, self.component_attributes[component.component_name], component.component)
|
||||
|
||||
kwargs = {component: update_attribute for component in self.component_attributes.keys()}
|
||||
self._component_setup_deferred = self.component_manager.setup(**kwargs)
|
||||
return self._component_setup_deferred
|
||||
|
||||
@staticmethod
|
||||
def _already_shutting_down(sig_num, frame):
|
||||
log.info("Already shutting down")
|
||||
|
||||
def _shutdown(self):
|
||||
# ignore INT/TERM signals once shutdown has started
|
||||
signal.signal(signal.SIGINT, self._already_shutting_down)
|
||||
signal.signal(signal.SIGTERM, self._already_shutting_down)
|
||||
self.looping_call_manager.shutdown()
|
||||
if self.analytics_manager:
|
||||
self.analytics_manager.shutdown()
|
||||
try:
|
||||
self._component_setup_deferred.cancel()
|
||||
except (AttributeError, defer.CancelledError):
|
||||
pass
|
||||
if self.component_manager is not None:
|
||||
d = self.component_manager.stop()
|
||||
d.addErrback(log.fail(), 'Failure while shutting down')
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
return d
|
||||
|
||||
def get_server_factory(self):
|
||||
return AuthJSONRPCResource(self).getServerFactory()
|
||||
|
||||
def _set_headers(self, request, data, update_secret=False):
|
||||
if conf.settings['allowed_origin']:
|
||||
request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
|
||||
request.setHeader("Content-Type", "application/json")
|
||||
request.setHeader("Accept", "application/json-rpc")
|
||||
request.setHeader("Content-Length", str(len(data)))
|
||||
if update_secret:
|
||||
session_id = request.getSession().uid
|
||||
request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
|
||||
|
||||
@staticmethod
|
||||
def _render_message(request, message):
|
||||
request.write(message)
|
||||
request.finish()
|
||||
|
||||
def _render_error(self, failure, request, id_):
|
||||
if isinstance(failure, JSONRPCError):
|
||||
error = failure
|
||||
elif isinstance(failure, Failure):
|
||||
# maybe failure is JSONRPCError wrapped in a twisted Failure
|
||||
error = failure.check(JSONRPCError)
|
||||
if error is None:
|
||||
# maybe its a twisted Failure with another type of error
|
||||
error = JSONRPCError(failure.getErrorMessage() or failure.type.__name__,
|
||||
traceback=failure.getTraceback())
|
||||
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
|
||||
log.warning("error processing api request: %s\ntraceback: %s", error.message,
|
||||
"\n".join(error.traceback))
|
||||
else:
|
||||
# last resort, just cast it as a string
|
||||
error = JSONRPCError(str(failure))
|
||||
|
||||
response_content = jsonrpc_dumps_pretty(error, id=id_)
|
||||
self._set_headers(request, response_content)
|
||||
request.setResponseCode(200)
|
||||
self._render_message(request, response_content)
|
||||
|
||||
@staticmethod
|
||||
def _handle_dropped_request(result, d, function_name):
|
||||
if not d.called:
|
||||
log.warning("Cancelling dropped api request %s", function_name)
|
||||
d.cancel()
|
||||
|
||||
def render(self, request):
|
||||
try:
|
||||
return self._render(request)
|
||||
except BaseException as e:
|
||||
log.error(e)
|
||||
error = JSONRPCError.create_from_exception(e, traceback=format_exc())
|
||||
self._render_error(error, request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _render(self, request):
|
||||
time_in = utils.now()
|
||||
# if not self._check_headers(request):
|
||||
# self._render_error(Failure(InvalidHeaderError()), request, None)
|
||||
# return server.NOT_DONE_YET
|
||||
session = request.getSession()
|
||||
session_id = session.uid
|
||||
finished_deferred = request.notifyFinish()
|
||||
|
||||
if self._use_authentication:
|
||||
# if this is a new session, send a new secret and set the expiration
|
||||
# otherwise, session.touch()
|
||||
if self._initialize_session(session_id):
|
||||
def expire_session():
|
||||
self._unregister_user_session(session_id)
|
||||
|
||||
session.startCheckingExpiration()
|
||||
session.notifyOnExpire(expire_session)
|
||||
message = "OK"
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, True)
|
||||
self._render_message(request, message)
|
||||
return server.NOT_DONE_YET
|
||||
else:
|
||||
session.touch()
|
||||
|
||||
request.content.seek(0, 0)
|
||||
content = request.content.read()
|
||||
try:
|
||||
parsed = jsonrpclib.loads(content)
|
||||
except ValueError:
|
||||
log.warning("Unable to decode request json")
|
||||
self._render_error(JSONRPCError(None, JSONRPCError.CODE_PARSE_ERROR), request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
request_id = None
|
||||
try:
|
||||
function_name = parsed.get('method')
|
||||
args = parsed.get('params', {})
|
||||
request_id = parsed.get('id', None)
|
||||
token = parsed.pop('hmac', None)
|
||||
except AttributeError as err:
|
||||
log.warning(err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
reply_with_next_secret = False
|
||||
if self._use_authentication:
|
||||
try:
|
||||
self._verify_token(session_id, parsed, token)
|
||||
except InvalidAuthenticationToken as err:
|
||||
log.warning("API validation failed")
|
||||
self._render_error(
|
||||
JSONRPCError.create_from_exception(
|
||||
err, code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
|
||||
traceback=format_exc()
|
||||
),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
request.addCookie("TWISTED_SESSION", session_id)
|
||||
self._update_session_secret(session_id)
|
||||
reply_with_next_secret = True
|
||||
|
||||
try:
|
||||
fn = self._get_jsonrpc_method(function_name)
|
||||
except UnknownAPIMethodError as err:
|
||||
log.warning('Failed to get function %s: %s', function_name, err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, JSONRPCError.CODE_METHOD_NOT_FOUND),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
if args == EMPTY_PARAMS or args == []:
|
||||
_args, _kwargs = (), {}
|
||||
elif isinstance(args, dict):
|
||||
_args, _kwargs = (), args
|
||||
elif len(args) == 1 and isinstance(args[0], dict):
|
||||
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
|
||||
# TODO: also delete EMPTY_PARAMS then
|
||||
_args, _kwargs = (), args[0]
|
||||
elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], dict):
|
||||
_args, _kwargs = args
|
||||
else:
|
||||
raise ValueError('invalid args format')
|
||||
|
||||
params_error, erroneous_params = self._check_params(fn, _args, _kwargs)
|
||||
if params_error is not None:
|
||||
params_error_message = '{} for {} command: {}'.format(
|
||||
params_error, function_name, ', '.join(erroneous_params)
|
||||
)
|
||||
log.warning(params_error_message)
|
||||
self._render_error(
|
||||
JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
d = defer.maybeDeferred(fn, self, *_args, **_kwargs)
|
||||
|
||||
# finished_deferred will callback when the request is finished
|
||||
# and errback if something went wrong. If the errback is
|
||||
# called, cancel the deferred stack. This is to prevent
|
||||
# request.finish() from being called on a closed request.
|
||||
finished_deferred.addErrback(self._handle_dropped_request, d, function_name)
|
||||
|
||||
d.addCallback(self._callback_render, request, request_id, reply_with_next_secret)
|
||||
d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError)
|
||||
d.addErrback(self._render_error, request, request_id)
|
||||
d.addBoth(lambda _: log.debug("%s took %f",
|
||||
function_name,
|
||||
(utils.now() - time_in).total_seconds()))
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _register_user_session(self, session_id):
|
||||
"""
|
||||
Add or update a HMAC secret for a session
|
||||
|
||||
@param session_id:
|
||||
@return: secret
|
||||
"""
|
||||
log.info("Started new api session")
|
||||
token = APIKey.new(seed=session_id)
|
||||
self.sessions.update({session_id: token})
|
||||
|
||||
def _unregister_user_session(self, session_id):
|
||||
log.info("Unregister API session")
|
||||
del self.sessions[session_id]
|
||||
|
||||
def _check_headers(self, request):
|
||||
return (
|
||||
self._check_header_source(request, 'Origin') and
|
||||
self._check_header_source(request, 'Referer')
|
||||
)
|
||||
|
||||
def _check_header_source(self, request, header):
|
||||
"""Check if the source of the request is allowed based on the header value."""
|
||||
source = request.getHeader(header)
|
||||
if not self._check_source_of_request(source):
|
||||
log.warning("Attempted api call from invalid %s: %s", header, source)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _check_source_of_request(self, source):
|
||||
if source is None:
|
||||
return True
|
||||
if conf.settings['api_host'] == '0.0.0.0':
|
||||
return True
|
||||
server, port = self.get_server_port(source)
|
||||
return self._check_server_port(server, port)
|
||||
|
||||
def _check_server_port(self, server, port):
|
||||
api = (conf.settings['api_host'], conf.settings['api_port'])
|
||||
return (server, port) == api or self._is_from_allowed_origin(server, port)
|
||||
|
||||
def _is_from_allowed_origin(self, server, port):
|
||||
allowed_origin = conf.settings['allowed_origin']
|
||||
if not allowed_origin:
|
||||
return False
|
||||
if allowed_origin == '*':
|
||||
return True
|
||||
allowed_server, allowed_port = self.get_server_port(allowed_origin)
|
||||
return (allowed_server, allowed_port) == (server, port)
|
||||
|
||||
def get_server_port(self, origin):
|
||||
parsed = urlparse.urlparse(origin)
|
||||
server_port = parsed.netloc.split(':')
|
||||
assert len(server_port) <= 2
|
||||
if len(server_port) == 2:
|
||||
return server_port[0], int(server_port[1])
|
||||
else:
|
||||
return server_port[0], 80
|
||||
|
||||
def _verify_method_is_callable(self, function_path):
|
||||
if function_path not in self.callable_methods:
|
||||
raise UnknownAPIMethodError(function_path)
|
||||
|
||||
def _get_jsonrpc_method(self, function_path):
|
||||
if function_path in self.deprecated_methods:
|
||||
new_command = self.deprecated_methods[function_path]._new_command
|
||||
log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
|
||||
function_path, new_command)
|
||||
function_path = new_command
|
||||
self._verify_method_is_callable(function_path)
|
||||
return self.callable_methods.get(function_path)
|
||||
|
||||
@staticmethod
|
||||
def _check_params(function, args_tup, args_dict):
|
||||
argspec = inspect.getargspec(undecorated(function))
|
||||
num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
|
||||
|
||||
duplicate_params = [
|
||||
duplicate_param
|
||||
for duplicate_param in argspec.args[1:len(args_tup) + 1]
|
||||
if duplicate_param in args_dict
|
||||
]
|
||||
|
||||
if duplicate_params:
|
||||
return 'Duplicate parameters', duplicate_params
|
||||
|
||||
missing_required_params = [
|
||||
required_param
|
||||
for required_param in argspec.args[len(args_tup)+1:-num_optional_params]
|
||||
if required_param not in args_dict
|
||||
]
|
||||
if len(missing_required_params):
|
||||
return 'Missing required parameters', missing_required_params
|
||||
|
||||
extraneous_params = [] if argspec.keywords is not None else [
|
||||
extra_param
|
||||
for extra_param in args_dict
|
||||
if extra_param not in argspec.args[1:]
|
||||
]
|
||||
if len(extraneous_params):
|
||||
return 'Extraneous parameters', extraneous_params
|
||||
|
||||
return None, None
|
||||
|
||||
def _initialize_session(self, session_id):
|
||||
if not self.sessions.get(session_id):
|
||||
self._register_user_session(session_id)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _verify_token(self, session_id, message, token):
|
||||
if token is None:
|
||||
raise InvalidAuthenticationToken('Authentication token not found')
|
||||
to_auth = get_auth_message(message)
|
||||
api_key = self.sessions.get(session_id)
|
||||
if not api_key.compare_hmac(to_auth, token):
|
||||
raise InvalidAuthenticationToken('Invalid authentication token')
|
||||
|
||||
def _update_session_secret(self, session_id):
|
||||
self.sessions.update({session_id: APIKey.new(name=session_id)})
|
||||
|
||||
def _callback_render(self, result, request, id_, auth_required=False):
|
||||
try:
|
||||
encoded_message = jsonrpc_dumps_pretty(result, id=id_, default=default_decimal)
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, encoded_message, auth_required)
|
||||
self._render_message(request, encoded_message)
|
||||
except Exception as err:
|
||||
log.exception("Failed to render API response: %s", result)
|
||||
self._render_error(err, request, id_)
|
||||
|
||||
@staticmethod
|
||||
def _render_response(result):
|
||||
return defer.succeed(result)
|
||||
@@ -1,91 +0,0 @@
|
||||
import base58
|
||||
import hmac
|
||||
import hashlib
|
||||
import yaml
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
API_KEY_NAME = "api"
|
||||
|
||||
|
||||
def sha(x):
|
||||
h = hashlib.sha256(x).digest()
|
||||
return base58.b58encode(h)
|
||||
|
||||
|
||||
def generate_key(x=None):
|
||||
if x is None:
|
||||
return sha(os.urandom(256))
|
||||
else:
|
||||
return sha(x)
|
||||
|
||||
|
||||
class APIKey(object):
|
||||
def __init__(self, secret, name, expiration=None):
|
||||
self.secret = secret
|
||||
self.name = name
|
||||
self.expiration = expiration
|
||||
|
||||
@classmethod
|
||||
def new(cls, seed=None, name=None, expiration=None):
|
||||
secret = generate_key(seed)
|
||||
key_name = name if name else sha(secret)
|
||||
return APIKey(secret, key_name, expiration)
|
||||
|
||||
def _raw_key(self):
|
||||
return base58.b58decode(self.secret)
|
||||
|
||||
def get_hmac(self, message):
|
||||
decoded_key = self._raw_key()
|
||||
signature = hmac.new(decoded_key, message, hashlib.sha256)
|
||||
return base58.b58encode(signature.digest())
|
||||
|
||||
def compare_hmac(self, message, token):
|
||||
decoded_token = base58.b58decode(token)
|
||||
target = base58.b58decode(self.get_hmac(message))
|
||||
|
||||
try:
|
||||
if len(decoded_token) != len(target):
|
||||
return False
|
||||
return hmac.compare_digest(decoded_token, target)
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def load_api_keys(path):
|
||||
if not os.path.isfile(path):
|
||||
raise Exception("Invalid api key path")
|
||||
|
||||
with open(path, "r") as f:
|
||||
data = yaml.load(f.read())
|
||||
|
||||
keys_for_return = {}
|
||||
for key_name in data:
|
||||
key = data[key_name]
|
||||
secret = key['secret']
|
||||
expiration = key['expiration']
|
||||
keys_for_return.update({key_name: APIKey(secret, key_name, expiration)})
|
||||
return keys_for_return
|
||||
|
||||
|
||||
def save_api_keys(keys, path):
|
||||
with open(path, "w") as f:
|
||||
key_dict = {keys[key_name].name: {'secret': keys[key_name].secret,
|
||||
'expiration': keys[key_name].expiration}
|
||||
for key_name in keys}
|
||||
data = yaml.safe_dump(key_dict)
|
||||
f.write(data)
|
||||
|
||||
|
||||
def initialize_api_key_file(key_path):
|
||||
keys = {}
|
||||
new_api_key = APIKey.new(name=API_KEY_NAME)
|
||||
keys.update({new_api_key.name: new_api_key})
|
||||
save_api_keys(keys, key_path)
|
||||
|
||||
|
||||
def get_auth_message(message_dict):
|
||||
return json.dumps(message_dict, sort_keys=True)
|
||||
@@ -8,7 +8,7 @@ DELAY_INCREMENT = 0.0001
|
||||
QUEUE_SIZE_THRESHOLD = 100
|
||||
|
||||
|
||||
class CallLaterManager(object):
|
||||
class CallLaterManager:
|
||||
def __init__(self, callLater):
|
||||
"""
|
||||
:param callLater: (IReactorTime.callLater)
|
||||
@@ -1,16 +1,18 @@
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import reduce
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
try:
|
||||
ip = ipaddress.ip_address(address.decode()) # this needs to be unicode, thus the decode()
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class _Contact(object):
|
||||
class _Contact:
|
||||
""" Encapsulation for remote contact
|
||||
|
||||
This class contains information on a single remote contact, and also
|
||||
@@ -19,8 +21,8 @@ class _Contact(object):
|
||||
|
||||
def __init__(self, contactManager, id, ipAddress, udpPort, networkProtocol, firstComm):
|
||||
if id is not None:
|
||||
if not len(id) == constants.key_bits / 8:
|
||||
raise ValueError("invalid node id: %s" % id.encode('hex'))
|
||||
if not len(id) == constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: {}".format(hexlify(id).decode()))
|
||||
if not 0 <= udpPort <= 65536:
|
||||
raise ValueError("invalid port")
|
||||
if not is_valid_ipv4(ipAddress):
|
||||
@@ -38,7 +40,7 @@ class _Contact(object):
|
||||
self._token = (None, 0) # token, timestamp
|
||||
|
||||
def update_token(self, token):
|
||||
self._token = token, self.getTime()
|
||||
self._token = token, self.getTime() if token else 0
|
||||
|
||||
@property
|
||||
def token(self):
|
||||
@@ -56,7 +58,7 @@ class _Contact(object):
|
||||
def log_id(self, short=True):
|
||||
if not self.id:
|
||||
return "not initialized"
|
||||
id_hex = self.id.encode('hex')
|
||||
id_hex = hexlify(self.id)
|
||||
return id_hex if not short else id_hex[:8]
|
||||
|
||||
@property
|
||||
@@ -65,7 +67,7 @@ class _Contact(object):
|
||||
|
||||
@property
|
||||
def lastFailed(self):
|
||||
return self._contactManager._rpc_failures.get((self.address, self.port), [None])[-1]
|
||||
return (self.failures or [None])[-1]
|
||||
|
||||
@property
|
||||
def failures(self):
|
||||
@@ -95,25 +97,17 @@ class _Contact(object):
|
||||
return None
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, _Contact):
|
||||
return self.id == other.id
|
||||
elif isinstance(other, str):
|
||||
return self.id == other
|
||||
else:
|
||||
return False
|
||||
if not isinstance(other, _Contact):
|
||||
raise TypeError("invalid type to compare with Contact: %s" % str(type(other)))
|
||||
return (self.id, self.address, self.port) == (other.id, other.address, other.port)
|
||||
|
||||
def __ne__(self, other):
|
||||
if isinstance(other, _Contact):
|
||||
return self.id != other.id
|
||||
elif isinstance(other, str):
|
||||
return self.id != other
|
||||
else:
|
||||
return True
|
||||
def __hash__(self):
|
||||
return hash((self.id, self.address, self.port))
|
||||
|
||||
def compact_ip(self):
|
||||
compact_ip = reduce(
|
||||
lambda buff, x: buff + bytearray([int(x)]), self.address.split('.'), bytearray())
|
||||
return str(compact_ip)
|
||||
return compact_ip
|
||||
|
||||
def set_id(self, id):
|
||||
if not self._id:
|
||||
@@ -156,12 +150,12 @@ class _Contact(object):
|
||||
raise AttributeError("unknown command: %s" % name)
|
||||
|
||||
def _sendRPC(*args, **kwargs):
|
||||
return self._networkProtocol.sendRPC(self, name, args)
|
||||
return self._networkProtocol.sendRPC(self, name.encode(), args)
|
||||
|
||||
return _sendRPC
|
||||
|
||||
|
||||
class ContactManager(object):
|
||||
class ContactManager:
|
||||
def __init__(self, get_time=None):
|
||||
if not get_time:
|
||||
from twisted.internet import reactor
|
||||
@@ -171,12 +165,11 @@ class ContactManager(object):
|
||||
self._rpc_failures = {}
|
||||
|
||||
def get_contact(self, id, address, port):
|
||||
for contact in self._contacts.itervalues():
|
||||
for contact in self._contacts.values():
|
||||
if contact.id == id and contact.address == address and contact.port == port:
|
||||
return contact
|
||||
|
||||
def make_contact(self, id, ipAddress, udpPort, networkProtocol, firstComm=0):
|
||||
ipAddress = str(ipAddress)
|
||||
contact = self.get_contact(id, ipAddress, udpPort)
|
||||
if contact:
|
||||
return contact
|
||||
|
||||
@@ -1,27 +1,21 @@
|
||||
import UserDict
|
||||
import constants
|
||||
from interface import IDataStore
|
||||
from zope.interface import implements
|
||||
from collections import UserDict
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class DictDataStore(UserDict.DictMixin):
|
||||
class DictDataStore(UserDict):
|
||||
""" A datastore using an in-memory Python dictionary """
|
||||
implements(IDataStore)
|
||||
#implements(IDataStore)
|
||||
|
||||
def __init__(self, getTime=None):
|
||||
# Dictionary format:
|
||||
# { <key>: (<contact>, <value>, <lastPublished>, <originallyPublished> <originalPublisherID>) }
|
||||
self._dict = {}
|
||||
super().__init__()
|
||||
if not getTime:
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self.completed_blobs = set()
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
return self._dict.keys()
|
||||
|
||||
def filter_bad_and_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
@@ -29,41 +23,44 @@ class DictDataStore(UserDict.DictMixin):
|
||||
return filter(
|
||||
lambda peer:
|
||||
self._getTime() - peer[3] < constants.dataExpireTimeout and peer[0].contact_is_good is not False,
|
||||
self._dict[key]
|
||||
self[key]
|
||||
)
|
||||
|
||||
def filter_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
return filter(lambda peer: self._getTime() - peer[3] < constants.dataExpireTimeout, self._dict[key])
|
||||
return filter(lambda peer: self._getTime() - peer[3] < constants.dataExpireTimeout, self[key])
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
for key in self._dict.keys():
|
||||
unexpired_peers = self.filter_expired_peers(key)
|
||||
expired_keys = []
|
||||
for key in self.keys():
|
||||
unexpired_peers = list(self.filter_expired_peers(key))
|
||||
if not unexpired_peers:
|
||||
del self._dict[key]
|
||||
expired_keys.append(key)
|
||||
else:
|
||||
self._dict[key] = unexpired_peers
|
||||
self[key] = unexpired_peers
|
||||
for key in expired_keys:
|
||||
del self[key]
|
||||
|
||||
def hasPeersForBlob(self, key):
|
||||
return True if key in self._dict and len(self.filter_bad_and_expired_peers(key)) else False
|
||||
return bool(key in self and len(tuple(self.filter_bad_and_expired_peers(key))))
|
||||
|
||||
def addPeerToBlob(self, contact, key, compact_address, lastPublished, originallyPublished, originalPublisherID):
|
||||
if key in self._dict:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self._dict[key]):
|
||||
self._dict[key].append(
|
||||
if key in self:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self[key]):
|
||||
self[key].append(
|
||||
(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)
|
||||
)
|
||||
else:
|
||||
self._dict[key] = [(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)]
|
||||
self[key] = [(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)]
|
||||
|
||||
def getPeersForBlob(self, key):
|
||||
return [] if key not in self._dict else [val[1] for val in self.filter_bad_and_expired_peers(key)]
|
||||
return [] if key not in self else [val[1] for val in self.filter_bad_and_expired_peers(key)]
|
||||
|
||||
def getStoringContacts(self):
|
||||
contacts = set()
|
||||
for key in self._dict:
|
||||
for values in self._dict[key]:
|
||||
for key in self:
|
||||
for values in self[key]:
|
||||
contacts.add(values[0])
|
||||
return list(contacts)
|
||||
@@ -1,21 +1,21 @@
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class Distance(object):
|
||||
class Distance:
|
||||
"""Calculate the XOR result between two string variables.
|
||||
|
||||
Frequently we re-use one of the points so as an optimization
|
||||
we pre-calculate the long value of that point.
|
||||
we pre-calculate the value of that point.
|
||||
"""
|
||||
|
||||
def __init__(self, key):
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.key = key
|
||||
self.val_key_one = long(key.encode('hex'), 16)
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two):
|
||||
val_key_two = long(key_two.encode('hex'), 16)
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
def is_closer(self, a, b):
|
||||
|
||||
@@ -1,134 +1,75 @@
|
||||
from error import DecodeError
|
||||
from lbrynet.dht.error import DecodeError
|
||||
|
||||
|
||||
class Encoding(object):
|
||||
""" Interface for RPC message encoders/decoders
|
||||
|
||||
All encoding implementations used with this library should inherit and
|
||||
implement this.
|
||||
"""
|
||||
|
||||
def encode(self, data):
|
||||
""" Encode the specified data
|
||||
|
||||
@param data: The data to encode
|
||||
This method has to support encoding of the following
|
||||
types: C{str}, C{int} and C{long}
|
||||
Any additional data types may be supported as long as the
|
||||
implementing class's C{decode()} method can successfully
|
||||
decode them.
|
||||
|
||||
@return: The encoded data
|
||||
@rtype: str
|
||||
"""
|
||||
|
||||
def decode(self, data):
|
||||
""" Decode the specified data string
|
||||
|
||||
@param data: The data (byte string) to decode.
|
||||
@type data: str
|
||||
|
||||
@return: The decoded data (in its correct type)
|
||||
"""
|
||||
def bencode(data):
|
||||
""" Encoder implementation of the Bencode algorithm (Bittorrent). """
|
||||
if isinstance(data, int):
|
||||
return b'i%de' % data
|
||||
elif isinstance(data, (bytes, bytearray)):
|
||||
return b'%d:%s' % (len(data), data)
|
||||
elif isinstance(data, str):
|
||||
return b'%d:%s' % (len(data), data.encode())
|
||||
elif isinstance(data, (list, tuple)):
|
||||
encoded_list_items = b''
|
||||
for item in data:
|
||||
encoded_list_items += bencode(item)
|
||||
return b'l%se' % encoded_list_items
|
||||
elif isinstance(data, dict):
|
||||
encoded_dict_items = b''
|
||||
keys = data.keys()
|
||||
for key in sorted(keys):
|
||||
encoded_dict_items += bencode(key)
|
||||
encoded_dict_items += bencode(data[key])
|
||||
return b'd%se' % encoded_dict_items
|
||||
else:
|
||||
raise TypeError("Cannot bencode '%s' object" % type(data))
|
||||
|
||||
|
||||
class Bencode(Encoding):
|
||||
""" Implementation of a Bencode-based algorithm (Bencode is the encoding
|
||||
algorithm used by Bittorrent).
|
||||
def bdecode(data):
|
||||
""" Decoder implementation of the Bencode algorithm. """
|
||||
assert type(data) == bytes # fixme: _maybe_ remove this after porting
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
try:
|
||||
return _decode_recursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError(str(e))
|
||||
|
||||
@note: This algorithm differs from the "official" Bencode algorithm in
|
||||
that it can encode/decode floating point values in addition to
|
||||
integers.
|
||||
"""
|
||||
|
||||
def encode(self, data):
|
||||
""" Encoder implementation of the Bencode algorithm
|
||||
|
||||
@param data: The data to encode
|
||||
@type data: int, long, tuple, list, dict or str
|
||||
|
||||
@return: The encoded data
|
||||
@rtype: str
|
||||
"""
|
||||
if isinstance(data, (int, long)):
|
||||
return 'i%de' % data
|
||||
elif isinstance(data, str):
|
||||
return '%d:%s' % (len(data), data)
|
||||
elif isinstance(data, (list, tuple)):
|
||||
encodedListItems = ''
|
||||
for item in data:
|
||||
encodedListItems += self.encode(item)
|
||||
return 'l%se' % encodedListItems
|
||||
elif isinstance(data, dict):
|
||||
encodedDictItems = ''
|
||||
keys = data.keys()
|
||||
keys.sort()
|
||||
for key in keys:
|
||||
encodedDictItems += self.encode(key) # TODO: keys should always be bytestrings
|
||||
encodedDictItems += self.encode(data[key])
|
||||
return 'd%se' % encodedDictItems
|
||||
else:
|
||||
print data
|
||||
raise TypeError("Cannot bencode '%s' object" % type(data))
|
||||
|
||||
def decode(self, data):
|
||||
""" Decoder implementation of the Bencode algorithm
|
||||
|
||||
@param data: The encoded data
|
||||
@type data: str
|
||||
|
||||
@note: This is a convenience wrapper for the recursive decoding
|
||||
algorithm, C{_decodeRecursive}
|
||||
|
||||
@return: The decoded data, as a native Python type
|
||||
@rtype: int, list, dict or str
|
||||
"""
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
def _decode_recursive(data, start_index=0):
|
||||
if data[start_index] == ord('i'):
|
||||
end_pos = data[start_index:].find(b'e') + start_index
|
||||
return int(data[start_index + 1:end_pos]), end_pos + 1
|
||||
elif data[start_index] == ord('l'):
|
||||
start_index += 1
|
||||
decoded_list = []
|
||||
while data[start_index] != ord('e'):
|
||||
list_data, start_index = _decode_recursive(data, start_index)
|
||||
decoded_list.append(list_data)
|
||||
return decoded_list, start_index + 1
|
||||
elif data[start_index] == ord('d'):
|
||||
start_index += 1
|
||||
decoded_dict = {}
|
||||
while data[start_index] != ord('e'):
|
||||
key, start_index = _decode_recursive(data, start_index)
|
||||
value, start_index = _decode_recursive(data, start_index)
|
||||
decoded_dict[key] = value
|
||||
return decoded_dict, start_index
|
||||
elif data[start_index] == ord('f'):
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
end_pos = data[start_index:].find(b'e') + start_index
|
||||
return float(data[start_index + 1:end_pos]), end_pos + 1
|
||||
elif data[start_index] == ord('n'):
|
||||
# This (None/NULL data type) is a non-standard extension
|
||||
# to the original Bencode algorithm
|
||||
return None, start_index + 1
|
||||
else:
|
||||
split_pos = data[start_index:].find(b':') + start_index
|
||||
try:
|
||||
return self._decodeRecursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError(e.message)
|
||||
|
||||
@staticmethod
|
||||
def _decodeRecursive(data, startIndex=0):
|
||||
""" Actual implementation of the recursive Bencode algorithm
|
||||
|
||||
Do not call this; use C{decode()} instead
|
||||
"""
|
||||
if data[startIndex] == 'i':
|
||||
endPos = data[startIndex:].find('e') + startIndex
|
||||
return int(data[startIndex + 1:endPos]), endPos + 1
|
||||
elif data[startIndex] == 'l':
|
||||
startIndex += 1
|
||||
decodedList = []
|
||||
while data[startIndex] != 'e':
|
||||
listData, startIndex = Bencode._decodeRecursive(data, startIndex)
|
||||
decodedList.append(listData)
|
||||
return decodedList, startIndex + 1
|
||||
elif data[startIndex] == 'd':
|
||||
startIndex += 1
|
||||
decodedDict = {}
|
||||
while data[startIndex] != 'e':
|
||||
key, startIndex = Bencode._decodeRecursive(data, startIndex)
|
||||
value, startIndex = Bencode._decodeRecursive(data, startIndex)
|
||||
decodedDict[key] = value
|
||||
return decodedDict, startIndex
|
||||
elif data[startIndex] == 'f':
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
endPos = data[startIndex:].find('e') + startIndex
|
||||
return float(data[startIndex + 1:endPos]), endPos + 1
|
||||
elif data[startIndex] == 'n':
|
||||
# This (None/NULL data type) is a non-standard extension
|
||||
# to the original Bencode algorithm
|
||||
return None, startIndex + 1
|
||||
else:
|
||||
splitPos = data[startIndex:].find(':') + startIndex
|
||||
try:
|
||||
length = int(data[startIndex:splitPos])
|
||||
except ValueError, e:
|
||||
raise DecodeError, e
|
||||
startIndex = splitPos + 1
|
||||
endPos = startIndex + length
|
||||
bytes = data[startIndex:endPos]
|
||||
return bytes, endPos
|
||||
length = int(data[start_index:split_pos])
|
||||
except ValueError:
|
||||
raise DecodeError()
|
||||
start_index = split_pos + 1
|
||||
end_pos = start_index + length
|
||||
b = data[start_index:end_pos]
|
||||
return b, end_pos
|
||||
@@ -1,10 +1,10 @@
|
||||
import binascii
|
||||
import exceptions
|
||||
#import exceptions
|
||||
|
||||
# this is a dict of {"exceptions.<exception class name>": exception class} items used to raise
|
||||
# remote built-in exceptions locally
|
||||
BUILTIN_EXCEPTIONS = {
|
||||
"exceptions.%s" % e: getattr(exceptions, e) for e in dir(exceptions) if not e.startswith("_")
|
||||
# "exceptions.%s" % e: getattr(exceptions, e) for e in dir(exceptions) if not e.startswith("_")
|
||||
}
|
||||
|
||||
|
||||
@@ -13,14 +13,12 @@ class DecodeError(Exception):
|
||||
Should be raised by an C{Encoding} implementation if decode operation
|
||||
fails
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class BucketFull(Exception):
|
||||
"""
|
||||
Raised when the bucket is full
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class UnknownRemoteException(Exception):
|
||||
@@ -37,7 +35,7 @@ class TimeoutError(Exception):
|
||||
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
|
||||
else:
|
||||
msg = 'Timeout connecting to uninitialized node'
|
||||
Exception.__init__(self, msg)
|
||||
super().__init__(msg)
|
||||
self.remote_contact_id = remote_contact_id
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ class IDataStore(Interface):
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
pass
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
pass
|
||||
@@ -59,7 +58,7 @@ class IRoutingTable(Interface):
|
||||
@param count: the amount of contacts to return
|
||||
@type count: int
|
||||
@param _rpcNodeID: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the paramater will get
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type _rpcNodeID: str
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from distance import Distance
|
||||
from error import TimeoutError
|
||||
import constants
|
||||
import struct
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,13 +15,13 @@ def get_contact(contact_list, node_id, address, port):
|
||||
|
||||
|
||||
def expand_peer(compact_peer_info):
|
||||
host = ".".join([str(ord(d)) for d in compact_peer_info[:4]])
|
||||
port, = struct.unpack('>H', compact_peer_info[4:6])
|
||||
host = "{}.{}.{}.{}".format(*compact_peer_info[:4])
|
||||
port = int.from_bytes(compact_peer_info[4:6], 'big')
|
||||
peer_node_id = compact_peer_info[6:]
|
||||
return (peer_node_id, host, port)
|
||||
|
||||
|
||||
class _IterativeFind(object):
|
||||
class _IterativeFind:
|
||||
# TODO: use polymorphism to search for a value or node
|
||||
# instead of using a find_value flag
|
||||
def __init__(self, node, shortlist, key, rpc, exclude=None):
|
||||
@@ -38,7 +37,7 @@ class _IterativeFind(object):
|
||||
# Shortlist of contact objects (the k closest known contacts to the key from the routing table)
|
||||
self.shortlist = shortlist
|
||||
# The search key
|
||||
self.key = str(key)
|
||||
self.key = key
|
||||
# The rpc method name (findValue or findNode)
|
||||
self.rpc = rpc
|
||||
# List of active queries; len() indicates number of active probes
|
||||
@@ -68,28 +67,28 @@ class _IterativeFind(object):
|
||||
|
||||
def getContactTriples(self, result):
|
||||
if self.is_find_value_request:
|
||||
contact_triples = result['contacts']
|
||||
contact_triples = result[b'contacts']
|
||||
else:
|
||||
contact_triples = result
|
||||
for contact_tup in contact_triples:
|
||||
if not isinstance(contact_tup, (list, tuple)) or len(contact_tup) != 3:
|
||||
raise ValueError("invalid contact triple")
|
||||
contact_tup[1] = contact_tup[1].decode() # ips are strings
|
||||
return contact_triples
|
||||
|
||||
def sortByDistance(self, contact_list):
|
||||
"""Sort the list of contacts in order by distance from key"""
|
||||
contact_list.sort(key=lambda c: self.distance(c.id))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def extendShortlist(self, contact, result):
|
||||
# The "raw response" tuple contains the response message and the originating address info
|
||||
originAddress = (contact.address, contact.port)
|
||||
if self.finished_deferred.called:
|
||||
defer.returnValue(contact.id)
|
||||
return contact.id
|
||||
if self.node.contact_manager.is_ignored(originAddress):
|
||||
raise ValueError("contact is ignored")
|
||||
if contact.id == self.node.node_id:
|
||||
defer.returnValue(contact.id)
|
||||
return contact.id
|
||||
|
||||
if contact not in self.active_contacts:
|
||||
self.active_contacts.append(contact)
|
||||
@@ -103,9 +102,9 @@ class _IterativeFind(object):
|
||||
if self.is_find_value_request and self.key in result:
|
||||
# We have found the value
|
||||
for peer in result[self.key]:
|
||||
_, host, port = expand_peer(peer)
|
||||
node_id, host, port = expand_peer(peer)
|
||||
if (host, port) not in self.exclude:
|
||||
self.find_value_result.setdefault(self.key, []).append(peer)
|
||||
self.find_value_result.setdefault(self.key, []).append((node_id, host, port))
|
||||
if self.find_value_result:
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
else:
|
||||
@@ -113,11 +112,11 @@ class _IterativeFind(object):
|
||||
# We are looking for a value, and the remote node didn't have it
|
||||
# - mark it as the closest "empty" node, if it is
|
||||
# TODO: store to this peer after finding the value as per the kademlia spec
|
||||
if 'closestNodeNoValue' in self.find_value_result:
|
||||
if b'closestNodeNoValue' in self.find_value_result:
|
||||
if self.is_closer(contact):
|
||||
self.find_value_result['closestNodeNoValue'] = contact
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
else:
|
||||
self.find_value_result['closestNodeNoValue'] = contact
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
contactTriples = self.getContactTriples(result)
|
||||
for contactTriple in contactTriples:
|
||||
if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
|
||||
@@ -134,14 +133,14 @@ class _IterativeFind(object):
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
|
||||
defer.returnValue(contact.id)
|
||||
return contact.id
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def probeContact(self, contact):
|
||||
fn = getattr(contact, self.rpc)
|
||||
try:
|
||||
response = yield fn(self.key)
|
||||
result = yield self.extendShortlist(contact, response)
|
||||
result = self.extendShortlist(contact, response)
|
||||
defer.returnValue(result)
|
||||
except (TimeoutError, defer.CancelledError, ValueError, IndexError):
|
||||
defer.returnValue(contact.id)
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import logging
|
||||
import constants
|
||||
from distance import Distance
|
||||
from error import BucketFull
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import BucketFull
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket(object):
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
@@ -135,8 +136,8 @@ class KBucket(object):
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
if isinstance(key, str):
|
||||
key = long(key.encode('hex'), 16)
|
||||
if isinstance(key, bytes):
|
||||
key = int.from_bytes(key, 'big')
|
||||
return self.rangeMin <= key < self.rangeMax
|
||||
|
||||
def __len__(self):
|
||||
|
||||
@@ -7,10 +7,10 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import msgtypes
|
||||
from lbrynet.dht import msgtypes
|
||||
|
||||
|
||||
class MessageTranslator(object):
|
||||
class MessageTranslator:
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
Classes inheriting from this should provide a translation services between
|
||||
|
||||
@@ -7,17 +7,17 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from lbrynet.core.utils import generate_id
|
||||
import constants
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class Message(object):
|
||||
class Message:
|
||||
""" Base class for messages - all "unknown" messages use this class """
|
||||
|
||||
def __init__(self, rpcID, nodeID):
|
||||
if len(rpcID) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id: %i bytes (expected 20)" % len(rpcID))
|
||||
if len(nodeID) != constants.key_bits / 8:
|
||||
if len(nodeID) != constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: %i bytes (expected 48)" % len(nodeID))
|
||||
self.id = rpcID
|
||||
self.nodeID = nodeID
|
||||
@@ -29,7 +29,7 @@ class RequestMessage(Message):
|
||||
def __init__(self, nodeID, method, methodArgs, rpcID=None):
|
||||
if rpcID is None:
|
||||
rpcID = generate_id()[:constants.rpc_id_length]
|
||||
Message.__init__(self, rpcID, nodeID)
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.request = method
|
||||
self.args = methodArgs
|
||||
|
||||
@@ -38,7 +38,7 @@ class ResponseMessage(Message):
|
||||
""" Message containing the result from a successful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, response):
|
||||
Message.__init__(self, rpcID, nodeID)
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.response = response
|
||||
|
||||
|
||||
@@ -46,8 +46,7 @@ class ErrorMessage(ResponseMessage):
|
||||
""" Message containing the error from an unsuccessful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
|
||||
ResponseMessage.__init__(self, rpcID, nodeID, errorMessage)
|
||||
super().__init__(rpcID, nodeID, errorMessage)
|
||||
if isinstance(exceptionType, type):
|
||||
self.exceptionType = '%s.%s' % (exceptionType.__module__, exceptionType.__name__)
|
||||
else:
|
||||
self.exceptionType = exceptionType
|
||||
exceptionType = (f'{exceptionType.__module__}.{exceptionType.__name__}').encode()
|
||||
self.exceptionType = exceptionType
|
||||
@@ -1,40 +1,20 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
import binascii
|
||||
import hashlib
|
||||
import struct
|
||||
import logging
|
||||
from functools import reduce
|
||||
|
||||
from twisted.internet import defer, error, task
|
||||
|
||||
from lbrynet.core.utils import generate_id, DeferredDict
|
||||
from lbrynet.core.call_later_manager import CallLaterManager
|
||||
from lbrynet.core.PeerManager import PeerManager
|
||||
from error import TimeoutError
|
||||
import constants
|
||||
import routingtable
|
||||
import datastore
|
||||
import protocol
|
||||
from peerfinder import DHTPeerFinder
|
||||
from contact import ContactManager
|
||||
from iterativefind import iterativeFind
|
||||
|
||||
from lbrynet.utils import generate_id, DeferredDict
|
||||
from lbrynet.dht.call_later_manager import CallLaterManager
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants, routingtable, datastore, protocol
|
||||
from lbrynet.dht.contact import ContactManager
|
||||
from lbrynet.dht.iterativefind import iterativeFind
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def expand_peer(compact_peer_info):
|
||||
host = ".".join([str(ord(d)) for d in compact_peer_info[:4]])
|
||||
port, = struct.unpack('>H', compact_peer_info[4:6])
|
||||
peer_node_id = compact_peer_info[6:]
|
||||
return (peer_node_id, host, port)
|
||||
|
||||
|
||||
def rpcmethod(func):
|
||||
""" Decorator to expose Node methods as remote procedure calls
|
||||
|
||||
@@ -45,7 +25,7 @@ def rpcmethod(func):
|
||||
return func
|
||||
|
||||
|
||||
class MockKademliaHelper(object):
|
||||
class MockKademliaHelper:
|
||||
def __init__(self, clock=None, callLater=None, resolve=None, listenUDP=None):
|
||||
if not listenUDP or not resolve or not callLater or not clock:
|
||||
from twisted.internet import reactor
|
||||
@@ -98,8 +78,8 @@ class Node(MockKademliaHelper):
|
||||
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
|
||||
routingTableClass=None, networkProtocol=None,
|
||||
externalIP=None, peerPort=3333, listenUDP=None,
|
||||
callLater=None, resolve=None, clock=None, peer_finder=None,
|
||||
peer_manager=None, interface='', externalUDPPort=None):
|
||||
callLater=None, resolve=None, clock=None,
|
||||
interface='', externalUDPPort=None):
|
||||
"""
|
||||
@param dataStore: The data store to use. This must be class inheriting
|
||||
from the C{DataStore} interface (or providing the
|
||||
@@ -125,7 +105,7 @@ class Node(MockKademliaHelper):
|
||||
@param peerPort: the port at which this node announces it has a blob for
|
||||
"""
|
||||
|
||||
MockKademliaHelper.__init__(self, clock, callLater, resolve, listenUDP)
|
||||
super().__init__(clock, callLater, resolve, listenUDP)
|
||||
self.node_id = node_id or self._generateID()
|
||||
self.port = udpPort
|
||||
self._listen_interface = interface
|
||||
@@ -139,26 +119,23 @@ class Node(MockKademliaHelper):
|
||||
else:
|
||||
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
|
||||
|
||||
# Initialize this node's network access mechanisms
|
||||
if networkProtocol is None:
|
||||
self._protocol = protocol.KademliaProtocol(self)
|
||||
else:
|
||||
self._protocol = networkProtocol
|
||||
# Initialize the data storage mechanism used by this node
|
||||
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
|
||||
self.token_secret = self._generateID()
|
||||
self.old_token_secret = None
|
||||
self.externalIP = externalIP
|
||||
self.peerPort = peerPort
|
||||
self.externalUDPPort = externalUDPPort or self.port
|
||||
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
|
||||
self.peer_manager = peer_manager or PeerManager()
|
||||
self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager)
|
||||
self._join_deferred = None
|
||||
|
||||
def __del__(self):
|
||||
log.warning("unclean shutdown of the dht node")
|
||||
if hasattr(self, "_listeningPort") and self._listeningPort is not None:
|
||||
self._listeningPort.stopListening()
|
||||
#def __del__(self):
|
||||
# log.warning("unclean shutdown of the dht node")
|
||||
# if hasattr(self, "_listeningPort") and self._listeningPort is not None:
|
||||
# self._listeningPort.stopListening()
|
||||
|
||||
def __str__(self):
|
||||
return '<%s.%s object; ID: %s, IP address: %s, UDP port: %d>' % (
|
||||
self.__module__, self.__class__.__name__, binascii.hexlify(self.node_id), self.externalIP, self.port)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
@@ -203,7 +180,7 @@ class Node(MockKademliaHelper):
|
||||
if not known_node_resolution:
|
||||
known_node_resolution = yield _resolve_seeds()
|
||||
# we are one of the seed nodes, don't add ourselves
|
||||
if (self.externalIP, self.port) in known_node_resolution.itervalues():
|
||||
if (self.externalIP, self.port) in known_node_resolution.values():
|
||||
del known_node_resolution[(self.externalIP, self.port)]
|
||||
known_node_addresses.remove((self.externalIP, self.port))
|
||||
|
||||
@@ -216,7 +193,7 @@ class Node(MockKademliaHelper):
|
||||
def _initialize_routing():
|
||||
bootstrap_contacts = []
|
||||
contact_addresses = {(c.address, c.port): c for c in self.contacts}
|
||||
for (host, port), ip_address in known_node_resolution.iteritems():
|
||||
for (host, port), ip_address in known_node_resolution.items():
|
||||
if (host, port) not in contact_addresses:
|
||||
# Create temporary contact information for the list of addresses of known nodes
|
||||
# The contact node id will be set with the responding node id when we initialize it to None
|
||||
@@ -231,7 +208,7 @@ class Node(MockKademliaHelper):
|
||||
if not bootstrap_contacts:
|
||||
log.warning("no bootstrap contacts to ping")
|
||||
ping_result = yield _ping_contacts(bootstrap_contacts)
|
||||
shortlist = ping_result.keys()
|
||||
shortlist = list(ping_result.keys())
|
||||
if not shortlist:
|
||||
log.warning("failed to ping %i bootstrap contacts", len(bootstrap_contacts))
|
||||
defer.returnValue(None)
|
||||
@@ -239,10 +216,10 @@ class Node(MockKademliaHelper):
|
||||
# find the closest peers to us
|
||||
closest = yield self._iterativeFind(self.node_id, shortlist if not self.contacts else None)
|
||||
yield _ping_contacts(closest)
|
||||
# # query random hashes in our bucket key ranges to fill or split them
|
||||
# random_ids_in_range = self._routingTable.getRefreshList()
|
||||
# while random_ids_in_range:
|
||||
# yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
# query random hashes in our bucket key ranges to fill or split them
|
||||
random_ids_in_range = self._routingTable.getRefreshList()
|
||||
while random_ids_in_range:
|
||||
yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -266,7 +243,7 @@ class Node(MockKademliaHelper):
|
||||
yield _iterative_join()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, known_node_addresses=None):
|
||||
def start(self, known_node_addresses=None, block_on_join=False):
|
||||
""" Causes the Node to attempt to join the DHT network by contacting the
|
||||
known DHT nodes. This can be called multiple times if the previous attempt
|
||||
has failed or if the Node has lost all the contacts.
|
||||
@@ -281,8 +258,11 @@ class Node(MockKademliaHelper):
|
||||
self.start_listening()
|
||||
yield self._protocol._listening
|
||||
# TODO: Refresh all k-buckets further away than this node's closest neighbour
|
||||
yield self.joinNetwork(known_node_addresses or [])
|
||||
self.start_looping_calls()
|
||||
d = self.joinNetwork(known_node_addresses or [])
|
||||
d.addCallback(lambda _: self.start_looping_calls())
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
if block_on_join:
|
||||
yield d
|
||||
|
||||
def start_looping_calls(self):
|
||||
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)
|
||||
@@ -310,25 +290,24 @@ class Node(MockKademliaHelper):
|
||||
@defer.inlineCallbacks
|
||||
def storeToContact(self, blob_hash, contact):
|
||||
try:
|
||||
token = contact.token
|
||||
if not token:
|
||||
find_value_response = yield contact.findValue(blob_hash)
|
||||
token = find_value_response['token']
|
||||
contact.update_token(token)
|
||||
res = yield contact.store(blob_hash, token, self.peerPort, self.node_id, 0)
|
||||
if res != "OK":
|
||||
if not contact.token:
|
||||
yield contact.findValue(blob_hash)
|
||||
res = yield contact.store(blob_hash, contact.token, self.peerPort, self.node_id, 0)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
defer.returnValue(True)
|
||||
log.debug("Stored %s to %s (%s)", binascii.hexlify(blob_hash), contact.log_id(), contact.address)
|
||||
return True
|
||||
except protocol.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s",
|
||||
binascii.hexlify(blob_hash), contact.log_id())
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err.message)
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
contact.update_token(None)
|
||||
log.error("Unexpected error while storing blob_hash %s at %s: %s",
|
||||
binascii.hexlify(blob_hash), contact, err)
|
||||
defer.returnValue(False)
|
||||
return False
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def announceHaveBlob(self, blob_hash):
|
||||
@@ -337,9 +316,7 @@ class Node(MockKademliaHelper):
|
||||
if not self.externalIP:
|
||||
raise Exception("Cannot determine external IP: %s" % self.externalIP)
|
||||
stored_to = yield DeferredDict({contact: self.storeToContact(blob_hash, contact) for contact in contacts})
|
||||
contacted_node_ids = map(
|
||||
lambda contact: contact.id.encode('hex'), filter(lambda contact: stored_to[contact], stored_to.keys())
|
||||
)
|
||||
contacted_node_ids = [binascii.hexlify(contact.id) for contact in stored_to.keys() if stored_to[contact]]
|
||||
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(blob_hash),
|
||||
len(contacted_node_ids), len(contacts))
|
||||
defer.returnValue(contacted_node_ids)
|
||||
@@ -401,7 +378,7 @@ class Node(MockKademliaHelper):
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length!")
|
||||
|
||||
# Execute the search
|
||||
@@ -423,22 +400,15 @@ class Node(MockKademliaHelper):
|
||||
else:
|
||||
pass
|
||||
|
||||
expanded_peers = []
|
||||
if find_result:
|
||||
if key in find_result:
|
||||
for peer in find_result[key]:
|
||||
expanded = expand_peer(peer)
|
||||
if expanded not in expanded_peers:
|
||||
expanded_peers.append(expanded)
|
||||
# TODO: get this working
|
||||
# if 'closestNodeNoValue' in find_result:
|
||||
# closest_node_without_value = find_result['closestNodeNoValue']
|
||||
# try:
|
||||
# response, address = yield closest_node_without_value.findValue(key, rawResponse=True)
|
||||
# yield closest_node_without_value.store(key, response.response['token'], self.peerPort)
|
||||
# except TimeoutError:
|
||||
# pass
|
||||
defer.returnValue(expanded_peers)
|
||||
defer.returnValue(list(set(find_result.get(key, []) if find_result else [])))
|
||||
# TODO: get this working
|
||||
# if 'closestNodeNoValue' in find_result:
|
||||
# closest_node_without_value = find_result['closestNodeNoValue']
|
||||
# try:
|
||||
# response, address = yield closest_node_without_value.findValue(key, rawResponse=True)
|
||||
# yield closest_node_without_value.store(key, response.response['token'], self.peerPort)
|
||||
# except TimeoutError:
|
||||
# pass
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add/update the given contact; simple wrapper for the same method
|
||||
@@ -471,20 +441,10 @@ class Node(MockKademliaHelper):
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
try:
|
||||
contact = self._routingTable.getContact(contactID)
|
||||
df = defer.Deferred()
|
||||
df.callback(contact)
|
||||
df = defer.succeed(self._routingTable.getContact(contactID))
|
||||
except (ValueError, IndexError):
|
||||
def parseResults(nodes):
|
||||
node_ids = [c.id for c in nodes]
|
||||
if contactID in nodes:
|
||||
contact = nodes[node_ids.index(contactID)]
|
||||
return contact
|
||||
else:
|
||||
return None
|
||||
|
||||
df = self.iterativeFindNode(contactID)
|
||||
df.addCallback(parseResults)
|
||||
df.addCallback(lambda nodes: ([node for node in nodes if node.id == contactID] or (None,))[0])
|
||||
return df
|
||||
|
||||
@rpcmethod
|
||||
@@ -493,7 +453,7 @@ class Node(MockKademliaHelper):
|
||||
|
||||
@rtype: str
|
||||
"""
|
||||
return 'pong'
|
||||
return b'pong'
|
||||
|
||||
@rpcmethod
|
||||
def store(self, rpc_contact, blob_hash, token, port, originalPublisherID, age):
|
||||
@@ -528,15 +488,15 @@ class Node(MockKademliaHelper):
|
||||
elif not self.verify_token(token, compact_ip):
|
||||
raise ValueError("Invalid token")
|
||||
if 0 <= port <= 65536:
|
||||
compact_port = str(struct.pack('>H', port))
|
||||
compact_port = port.to_bytes(2, 'big')
|
||||
else:
|
||||
raise TypeError('Invalid port')
|
||||
raise TypeError(f'Invalid port: {port}')
|
||||
compact_address = compact_ip + compact_port + rpc_contact.id
|
||||
now = int(self.clock.seconds())
|
||||
originallyPublished = now - age
|
||||
self._dataStore.addPeerToBlob(rpc_contact, blob_hash, compact_address, now, originallyPublished,
|
||||
originalPublisherID)
|
||||
return 'OK'
|
||||
return b'OK'
|
||||
|
||||
@rpcmethod
|
||||
def findNode(self, rpc_contact, key):
|
||||
@@ -552,7 +512,7 @@ class Node(MockKademliaHelper):
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid contact id length: %i" % len(key))
|
||||
|
||||
contacts = self._routingTable.findCloseNodes(key, sender_node_id=rpc_contact.id)
|
||||
@@ -574,15 +534,15 @@ class Node(MockKademliaHelper):
|
||||
@rtype: dict or list
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid blob hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
'token': self.make_token(rpc_contact.compact_ip()),
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
}
|
||||
|
||||
if self._protocol._protocolVersion:
|
||||
response['protocolVersion'] = self._protocol._protocolVersion
|
||||
response[b'protocolVersion'] = self._protocol._protocolVersion
|
||||
|
||||
# get peers we have stored for this blob
|
||||
has_other_peers = self._dataStore.hasPeersForBlob(key)
|
||||
@@ -592,17 +552,15 @@ class Node(MockKademliaHelper):
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and key in self._dataStore.completed_blobs:
|
||||
compact_ip = str(
|
||||
reduce(lambda buff, x: buff + bytearray([int(x)]), self.externalIP.split('.'), bytearray())
|
||||
)
|
||||
compact_port = str(struct.pack('>H', self.peerPort))
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), self.externalIP.split('.'), bytearray())
|
||||
compact_port = self.peerPort.to_bytes(2, 'big')
|
||||
compact_address = compact_ip + compact_port + self.node_id
|
||||
peers.append(compact_address)
|
||||
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response['contacts'] = self.findNode(rpc_contact, key)
|
||||
response[b'contacts'] = self.findNode(rpc_contact, key)
|
||||
return response
|
||||
|
||||
def _generateID(self):
|
||||
@@ -613,7 +571,7 @@ class Node(MockKademliaHelper):
|
||||
"""
|
||||
return generate_id()
|
||||
|
||||
# from lbrynet.core.utils import profile_deferred
|
||||
# from lbrynet.p2p.utils import profile_deferred
|
||||
# @profile_deferred()
|
||||
@defer.inlineCallbacks
|
||||
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode', exclude=None):
|
||||
@@ -645,7 +603,7 @@ class Node(MockKademliaHelper):
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits / 8:
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
|
||||
if startupShortlist is None:
|
||||
@@ -654,7 +612,7 @@ class Node(MockKademliaHelper):
|
||||
# # Update the "last accessed" timestamp for the appropriate k-bucket
|
||||
# self._routingTable.touchKBucket(key)
|
||||
if len(shortlist) == 0:
|
||||
log.warning("This node doesnt know any other nodes")
|
||||
log.warning("This node doesn't know any other nodes")
|
||||
# This node doesn't know of any other nodes
|
||||
fakeDf = defer.Deferred()
|
||||
fakeDf.callback([])
|
||||
@@ -673,19 +631,14 @@ class Node(MockKademliaHelper):
|
||||
replication/republishing as necessary """
|
||||
yield self._refreshRoutingTable()
|
||||
self._dataStore.removeExpiredPeers()
|
||||
yield self._refreshStoringPeers()
|
||||
self._refreshStoringPeers()
|
||||
defer.returnValue(None)
|
||||
|
||||
def _refreshContacts(self):
|
||||
return defer.DeferredList(
|
||||
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in self.contacts]
|
||||
)
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self.contacts, delay=0)
|
||||
|
||||
def _refreshStoringPeers(self):
|
||||
storing_contacts = self._dataStore.getStoringContacts()
|
||||
return defer.DeferredList(
|
||||
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in storing_contacts]
|
||||
)
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self._dataStore.getStoringContacts(), delay=0)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _refreshRoutingTable(self):
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from lbrynet.interfaces import IPeerFinder
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyPeerFinder(object):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
return defer.succeed([])
|
||||
|
||||
|
||||
class DHTPeerFinder(DummyPeerFinder):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
implements(IPeerFinder)
|
||||
|
||||
def __init__(self, dht_node, peer_manager):
|
||||
"""
|
||||
dht_node - an instance of dht.Node class
|
||||
peer_manager - an instance of PeerManager class
|
||||
"""
|
||||
self.dht_node = dht_node
|
||||
self.peer_manager = peer_manager
|
||||
self.peers = {}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
"""
|
||||
Find peers for blob in the DHT
|
||||
blob_hash (str): blob hash to look for
|
||||
timeout (int): seconds to timeout after
|
||||
filter_self (bool): if True, and if a peer for a blob is itself, filter it
|
||||
from the result
|
||||
|
||||
Returns:
|
||||
list of peers for the blob
|
||||
"""
|
||||
if blob_hash not in self.peers:
|
||||
self.peers[blob_hash] = [(self.dht_node.externalIP, self.dht_node.peerPort)]
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = self.dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
|
||||
timeout = timeout or conf.settings['peer_search_timeout']
|
||||
if timeout:
|
||||
finished_deferred.addTimeout(timeout, self.dht_node.clock)
|
||||
try:
|
||||
peer_list = yield finished_deferred
|
||||
except defer.TimeoutError:
|
||||
log.debug("DHT timed out while looking peers for blob %s after %s seconds",
|
||||
blob_hash, timeout)
|
||||
peer_list = []
|
||||
|
||||
peers = set(peer_list)
|
||||
results = []
|
||||
for node_id, host, port in peers:
|
||||
if filter_self and (host, port) == (self.dht_node.externalIP, self.dht_node.peerPort):
|
||||
continue
|
||||
results.append(self.peer_manager.get_peer(host, port))
|
||||
self.peers[blob_hash].append((host, port))
|
||||
defer.returnValue(results)
|
||||
@@ -1,20 +1,15 @@
|
||||
import logging
|
||||
import socket
|
||||
import errno
|
||||
from collections import deque
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
from error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
import constants
|
||||
import encoding
|
||||
import msgtypes
|
||||
import msgformat
|
||||
from lbrynet.dht import constants, encoding, msgformat, msgtypes
|
||||
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PingQueue(object):
|
||||
class PingQueue:
|
||||
"""
|
||||
Schedules a 15 minute delayed ping after a new node sends us a query. This is so the new node gets added to the
|
||||
routing table after having been given enough time for a pinhole to expire.
|
||||
@@ -22,71 +17,53 @@ class PingQueue(object):
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._get_time = self._node.clock.seconds
|
||||
self._queue = deque()
|
||||
self._enqueued_contacts = {}
|
||||
self._semaphore = defer.DeferredSemaphore(1)
|
||||
self._ping_semaphore = defer.DeferredSemaphore(constants.alpha)
|
||||
self._process_lc = node.get_looping_call(self._semaphore.run, self._process)
|
||||
self._pending_contacts = {}
|
||||
self._process_lc = node.get_looping_call(self._process)
|
||||
|
||||
def _add_contact(self, contact, delay=None):
|
||||
if contact in self._enqueued_contacts:
|
||||
return defer.succeed(None)
|
||||
delay = delay or constants.checkRefreshInterval
|
||||
self._enqueued_contacts[contact] = self._get_time() + delay
|
||||
self._queue.append(contact)
|
||||
return defer.succeed(None)
|
||||
def enqueue_maybe_ping(self, *contacts, **kwargs):
|
||||
delay = kwargs.get('delay', constants.checkRefreshInterval)
|
||||
no_op = (defer.succeed(None), lambda: None)
|
||||
for contact in contacts:
|
||||
if delay and contact not in self._enqueued_contacts:
|
||||
self._pending_contacts.setdefault(contact, self._node.clock.seconds() + delay)
|
||||
else:
|
||||
self._enqueued_contacts.setdefault(contact, no_op)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _process(self):
|
||||
if not len(self._queue):
|
||||
defer.returnValue(None)
|
||||
contact = self._queue.popleft()
|
||||
now = self._get_time()
|
||||
|
||||
# if the oldest contact in the queue isn't old enough to be pinged, add it back to the queue and return
|
||||
if now < self._enqueued_contacts[contact]:
|
||||
self._queue.appendleft(contact)
|
||||
defer.returnValue(None)
|
||||
|
||||
pinged = []
|
||||
checked = []
|
||||
while now > self._enqueued_contacts[contact]:
|
||||
checked.append(contact)
|
||||
if not contact.contact_is_good:
|
||||
pinged.append(contact)
|
||||
if not len(self._queue):
|
||||
break
|
||||
contact = self._queue.popleft()
|
||||
if not now > self._enqueued_contacts[contact]:
|
||||
checked.append(contact)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _ping(contact):
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
|
||||
yield defer.DeferredList([_ping(contact) for contact in pinged])
|
||||
|
||||
for contact in checked:
|
||||
def _ping(self, contact):
|
||||
if contact.contact_is_good:
|
||||
return
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
finally:
|
||||
if contact in self._enqueued_contacts:
|
||||
del self._enqueued_contacts[contact]
|
||||
|
||||
defer.returnValue(None)
|
||||
def _process(self):
|
||||
# move contacts that are scheduled to join the queue
|
||||
if self._pending_contacts:
|
||||
now = self._node.clock.seconds()
|
||||
for contact in [contact for contact, schedule in self._pending_contacts.items() if schedule <= now]:
|
||||
del self._pending_contacts[contact]
|
||||
self._enqueued_contacts.setdefault(contact, (defer.succeed(None), lambda: None))
|
||||
# spread pings across 60 seconds to avoid flood and/or false negatives
|
||||
step = 60.0/float(len(self._enqueued_contacts)) if self._enqueued_contacts else 0
|
||||
for index, (contact, (call, _)) in enumerate(self._enqueued_contacts.items()):
|
||||
if call.called and not contact.contact_is_good:
|
||||
self._enqueued_contacts[contact] = self._node.reactor_callLater(index*step, self._ping, contact)
|
||||
|
||||
def start(self):
|
||||
return self._node.safe_start_looping_call(self._process_lc, 60)
|
||||
|
||||
def stop(self):
|
||||
map(None, (cancel() for _, (call, cancel) in self._enqueued_contacts.items() if not call.called))
|
||||
return self._node.safe_stop_looping_call(self._process_lc)
|
||||
|
||||
def enqueue_maybe_ping(self, contact, delay=None):
|
||||
return self._semaphore.run(self._add_contact, contact, delay)
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
@@ -95,7 +72,6 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._encoder = encoding.Bencode()
|
||||
self._translator = msgformat.DefaultFormat()
|
||||
self._sentMessages = {}
|
||||
self._partialMessages = {}
|
||||
@@ -106,12 +82,12 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
self.started_listening_time = 0
|
||||
|
||||
def _migrate_incoming_rpc_args(self, contact, method, *args):
|
||||
if method == 'store' and contact.protocolVersion == 0:
|
||||
if method == b'store' and contact.protocolVersion == 0:
|
||||
if isinstance(args[1], dict):
|
||||
blob_hash = args[0]
|
||||
token = args[1].pop('token', None)
|
||||
port = args[1].pop('port', -1)
|
||||
originalPublisherID = args[1].pop('lbryid', None)
|
||||
token = args[1].pop(b'token', None)
|
||||
port = args[1].pop(b'port', -1)
|
||||
originalPublisherID = args[1].pop(b'lbryid', None)
|
||||
age = 0
|
||||
return (blob_hash, token, port, originalPublisherID, age), {}
|
||||
return args, {}
|
||||
@@ -122,18 +98,36 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
protocol version keyword argument to calls to contacts who will accept it
|
||||
"""
|
||||
if contact.protocolVersion == 0:
|
||||
if method == 'store':
|
||||
if method == b'store':
|
||||
blob_hash, token, port, originalPublisherID, age = args
|
||||
args = (blob_hash, {'token': token, 'port': port, 'lbryid': originalPublisherID}, originalPublisherID,
|
||||
False)
|
||||
args = (
|
||||
blob_hash, {
|
||||
b'token': token,
|
||||
b'port': port,
|
||||
b'lbryid': originalPublisherID
|
||||
}, originalPublisherID, False
|
||||
)
|
||||
return args
|
||||
return args
|
||||
if args and isinstance(args[-1], dict):
|
||||
args[-1]['protocolVersion'] = self._protocolVersion
|
||||
args[-1][b'protocolVersion'] = self._protocolVersion
|
||||
return args
|
||||
return args + ({'protocolVersion': self._protocolVersion},)
|
||||
return args + ({b'protocolVersion': self._protocolVersion},)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def sendRPC(self, contact, method, args):
|
||||
for _ in range(constants.rpcAttempts):
|
||||
try:
|
||||
response = yield self._sendRPC(contact, method, args)
|
||||
return response
|
||||
except TimeoutError:
|
||||
if contact.contact_is_good:
|
||||
log.debug("RETRY %s ON %s", method, contact)
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
def _sendRPC(self, contact, method, args):
|
||||
"""
|
||||
Sends an RPC to the specified contact
|
||||
|
||||
@@ -156,11 +150,11 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
msg = msgtypes.RequestMessage(self._node.node_id, method, self._migrate_outgoing_rpc_args(contact, method,
|
||||
*args))
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = self._encoder.encode(msgPrimitive)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
|
||||
if args:
|
||||
log.debug("%s:%i SEND CALL %s(%s) TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
args[0].encode('hex'), contact.address, contact.port)
|
||||
hexlify(args[0]), contact.address, contact.port)
|
||||
else:
|
||||
log.debug("%s:%i SEND CALL %s TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
contact.address, contact.port)
|
||||
@@ -168,20 +162,23 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df = defer.Deferred()
|
||||
|
||||
def _remove_contact(failure): # remove the contact from the routing table and track the failure
|
||||
contact.update_last_failed()
|
||||
try:
|
||||
self._node.removeContact(contact)
|
||||
if not contact.contact_is_good:
|
||||
self._node.removeContact(contact)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
contact.update_last_failed()
|
||||
return failure
|
||||
|
||||
def _update_contact(result): # refresh the contact in the routing table
|
||||
contact.update_last_replied()
|
||||
if method == 'findValue':
|
||||
if 'protocolVersion' not in result:
|
||||
if method == b'findValue':
|
||||
if b'token' in result:
|
||||
contact.update_token(result[b'token'])
|
||||
if b'protocolVersion' not in result:
|
||||
contact.update_protocol_version(0)
|
||||
else:
|
||||
contact.update_protocol_version(result.pop('protocolVersion'))
|
||||
contact.update_protocol_version(result.pop(b'protocolVersion'))
|
||||
d = self._node.addContact(contact)
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
@@ -212,18 +209,17 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
@note: This is automatically called by Twisted when the protocol
|
||||
receives a UDP datagram
|
||||
"""
|
||||
|
||||
if datagram[0] == '\x00' and datagram[25] == '\x00':
|
||||
totalPackets = (ord(datagram[1]) << 8) | ord(datagram[2])
|
||||
if chr(datagram[0]) == '\x00' and chr(datagram[25]) == '\x00':
|
||||
totalPackets = (datagram[1] << 8) | datagram[2]
|
||||
msgID = datagram[5:25]
|
||||
seqNumber = (ord(datagram[3]) << 8) | ord(datagram[4])
|
||||
seqNumber = (datagram[3] << 8) | datagram[4]
|
||||
if msgID not in self._partialMessages:
|
||||
self._partialMessages[msgID] = {}
|
||||
self._partialMessages[msgID][seqNumber] = datagram[26:]
|
||||
if len(self._partialMessages[msgID]) == totalPackets:
|
||||
keys = self._partialMessages[msgID].keys()
|
||||
keys.sort()
|
||||
data = ''
|
||||
data = b''
|
||||
for key in keys:
|
||||
data += self._partialMessages[msgID][key]
|
||||
datagram = data
|
||||
@@ -231,11 +227,11 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
else:
|
||||
return
|
||||
try:
|
||||
msgPrimitive = self._encoder.decode(datagram)
|
||||
msgPrimitive = encoding.bdecode(datagram)
|
||||
message = self._translator.fromPrimitive(msgPrimitive)
|
||||
except (encoding.DecodeError, ValueError) as err:
|
||||
# We received some rubbish here
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", datagram.encode('hex'),
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", hexlify(datagram),
|
||||
address[0], address[1], err)
|
||||
return
|
||||
except (IndexError, KeyError):
|
||||
@@ -305,7 +301,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
# the node id of the node we sent a message to (these messages are treated as an error)
|
||||
if remoteContact.id and remoteContact.id != message.nodeID: # sent_to_id will be None for bootstrap
|
||||
log.debug("mismatch: (%s) %s:%i (%s vs %s)", method, remoteContact.address, remoteContact.port,
|
||||
remoteContact.log_id(False), message.nodeID.encode('hex'))
|
||||
remoteContact.log_id(False), hexlify(message.nodeID))
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
return
|
||||
elif not remoteContact.id:
|
||||
@@ -325,7 +321,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
If the data is spread over multiple UDP datagrams, the packets have the
|
||||
following structure::
|
||||
| | | | | |||||||||||| 0x00 |
|
||||
|Transmision|Total number|Sequence number| RPC ID |Header end|
|
||||
|Transmission|Total number|Sequence number| RPC ID |Header end|
|
||||
| type ID | of packets |of this packet | | indicator|
|
||||
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
|
||||
| | | | | |||||||||||| |
|
||||
@@ -343,7 +339,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
# 1st byte is transmission type id, bytes 2 & 3 are the
|
||||
# total number of packets in this transmission, bytes 4 &
|
||||
# 5 are the sequence number for this specific packet
|
||||
totalPackets = len(data) / self.msgSizeLimit
|
||||
totalPackets = len(data) // self.msgSizeLimit
|
||||
if len(data) % self.msgSizeLimit > 0:
|
||||
totalPackets += 1
|
||||
encTotalPackets = chr(totalPackets >> 8) + chr(totalPackets & 0xff)
|
||||
@@ -352,7 +348,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
while seqNumber < totalPackets:
|
||||
packetData = data[startPos:startPos + self.msgSizeLimit]
|
||||
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
|
||||
txData = '\x00%s%s%s\x00%s' % (encTotalPackets, encSeqNumber, rpcID, packetData)
|
||||
txData = f'\x00{encTotalPackets}{encSeqNumber}{rpcID}\x00{packetData}'
|
||||
self._scheduleSendNext(txData, address)
|
||||
|
||||
startPos += self.msgSizeLimit
|
||||
@@ -368,10 +364,10 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
if self.transport:
|
||||
try:
|
||||
self.transport.write(txData, address)
|
||||
except socket.error as err:
|
||||
except OSError as err:
|
||||
if err.errno == errno.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesnt seem to affect anything -grin
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
elif err.errno == errno.ENETUNREACH:
|
||||
# this should probably try to retransmit when the network connection is back
|
||||
@@ -388,15 +384,16 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
"""
|
||||
msg = msgtypes.ResponseMessage(rpcID, self._node.node_id, response)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = self._encoder.encode(msgPrimitive)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _sendError(self, contact, rpcID, exceptionType, exceptionMessage):
|
||||
""" Send an RPC error message to the specified contact
|
||||
"""
|
||||
exceptionMessage = exceptionMessage.encode()
|
||||
msg = msgtypes.ErrorMessage(rpcID, self._node.node_id, exceptionType, exceptionMessage)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = self._encoder.encode(msgPrimitive)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _handleRPC(self, senderContact, rpcID, method, args):
|
||||
@@ -414,7 +411,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df.addErrback(handleError)
|
||||
|
||||
# Execute the RPC
|
||||
func = getattr(self._node, method, None)
|
||||
func = getattr(self._node, method.decode(), None)
|
||||
if callable(func) and hasattr(func, "rpcmethod"):
|
||||
# Call the exposed Node method and return the result to the deferred callback chain
|
||||
# if args:
|
||||
@@ -423,19 +420,19 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
# else:
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self._node.externalIP, self._node.port, method,
|
||||
senderContact.address, senderContact.port)
|
||||
if args and isinstance(args[-1], dict) and 'protocolVersion' in args[-1]: # args don't need reformatting
|
||||
senderContact.update_protocol_version(int(args[-1].pop('protocolVersion')))
|
||||
if args and isinstance(args[-1], dict) and b'protocolVersion' in args[-1]: # args don't need reformatting
|
||||
senderContact.update_protocol_version(int(args[-1].pop(b'protocolVersion')))
|
||||
a, kw = tuple(args[:-1]), args[-1]
|
||||
else:
|
||||
senderContact.update_protocol_version(0)
|
||||
a, kw = self._migrate_incoming_rpc_args(senderContact, method, *args)
|
||||
try:
|
||||
if method != 'ping':
|
||||
if method != b'ping':
|
||||
result = func(senderContact, *a)
|
||||
else:
|
||||
result = func()
|
||||
except Exception, e:
|
||||
log.exception("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
except Exception as e:
|
||||
log.error("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
df.errback(e)
|
||||
else:
|
||||
df.callback(result)
|
||||
@@ -452,7 +449,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
log.error("deferred timed out, but is not present in sent messages list!")
|
||||
return
|
||||
remoteContact, df, timeout_call, timeout_canceller, method, args = self._sentMessages[messageID]
|
||||
if self._partialMessages.has_key(messageID):
|
||||
if messageID in self._partialMessages:
|
||||
# We are still receiving this message
|
||||
self._msgTimeoutInProgress(messageID, timeout_canceller, remoteContact, df, method, args)
|
||||
return
|
||||
@@ -478,7 +475,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
|
||||
def _hasProgressBeenMade(self, messageID):
|
||||
return (
|
||||
self._partialMessagesProgress.has_key(messageID) and
|
||||
messageID in self._partialMessagesProgress and
|
||||
(
|
||||
len(self._partialMessagesProgress[messageID]) !=
|
||||
len(self._partialMessages[messageID])
|
||||
|
||||
@@ -6,19 +6,17 @@
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import random
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
import constants
|
||||
import kbucket
|
||||
from error import TimeoutError
|
||||
from distance import Distance
|
||||
from interface import IRoutingTable
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.dht import constants, kbucket
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht.distance import Distance
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TreeRoutingTable(object):
|
||||
class TreeRoutingTable:
|
||||
""" This class implements a routing table used by a Node class.
|
||||
|
||||
The Kademlia routing table is a binary tree whFose leaves are k-buckets,
|
||||
@@ -33,7 +31,7 @@ class TreeRoutingTable(object):
|
||||
C{PING} RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
implements(IRoutingTable)
|
||||
#implements(IRoutingTable)
|
||||
|
||||
def __init__(self, parentNodeID, getTime=None):
|
||||
"""
|
||||
@@ -48,6 +46,7 @@ class TreeRoutingTable(object):
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self._ongoing_replacements = set()
|
||||
|
||||
def get_contacts(self):
|
||||
contacts = []
|
||||
@@ -134,8 +133,14 @@ class TreeRoutingTable(object):
|
||||
to_replace = not_good_contacts[0]
|
||||
else:
|
||||
to_replace = self._buckets[bucketIndex]._contacts[0]
|
||||
df = to_replace.ping()
|
||||
df.addErrback(replaceContact, to_replace)
|
||||
if to_replace not in self._ongoing_replacements:
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.port)
|
||||
self._ongoing_replacements.add(to_replace)
|
||||
df = to_replace.ping()
|
||||
df.addErrback(replaceContact, to_replace)
|
||||
df.addBoth(lambda _: self._ongoing_replacements.remove(to_replace))
|
||||
else:
|
||||
df = defer.succeed(None)
|
||||
return df
|
||||
else:
|
||||
self.touchKBucketByIndex(bucketIndex)
|
||||
@@ -150,7 +155,7 @@ class TreeRoutingTable(object):
|
||||
@param count: the amount of contacts to return, default of k (8)
|
||||
@type count: int
|
||||
@param sender_node_id: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the paramater will get
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type sender_node_id: str
|
||||
|
||||
@@ -180,12 +185,7 @@ class TreeRoutingTable(object):
|
||||
by this node
|
||||
"""
|
||||
bucketIndex = self._kbucketIndex(contactID)
|
||||
try:
|
||||
contact = self._buckets[bucketIndex].getContact(contactID)
|
||||
except ValueError:
|
||||
raise
|
||||
else:
|
||||
return contact
|
||||
return self._buckets[bucketIndex].getContact(contactID)
|
||||
|
||||
def getRefreshList(self, startIndex=0, force=False):
|
||||
""" Finds all k-buckets that need refreshing, starting at the
|
||||
@@ -213,7 +213,7 @@ class TreeRoutingTable(object):
|
||||
now = int(self._getTime())
|
||||
for bucket in self._buckets[startIndex:]:
|
||||
if force or now - bucket.lastAccessed >= constants.refreshTimeout:
|
||||
searchID = self._randomIDInBucketRange(bucketIndex)
|
||||
searchID = self.midpoint_id_in_bucket_range(bucketIndex)
|
||||
refreshIDs.append(searchID)
|
||||
bucketIndex += 1
|
||||
return refreshIDs
|
||||
@@ -261,22 +261,25 @@ class TreeRoutingTable(object):
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def _randomIDInBucketRange(self, bucketIndex):
|
||||
def random_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns a random ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
idValue = random.randrange(
|
||||
self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax)
|
||||
randomID = hex(idValue)[2:]
|
||||
if randomID[-1] == 'L':
|
||||
randomID = randomID[:-1]
|
||||
if len(randomID) % 2 != 0:
|
||||
randomID = '0' + randomID
|
||||
randomID = randomID.decode('hex')
|
||||
randomID = (constants.key_bits / 8 - len(randomID)) * '\x00' + randomID
|
||||
return randomID
|
||||
|
||||
random_id = int(random.randrange(self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax))
|
||||
return random_id.to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns the middle ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
|
||||
half = int((self._buckets[bucketIndex].rangeMax - self._buckets[bucketIndex].rangeMin) // 2)
|
||||
return int(self._buckets[bucketIndex].rangeMin + half).to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def _splitBucket(self, oldBucketIndex):
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import sys
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
import logging
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
import aiohttp
|
||||
from lbrynet.extras.compat import force_asyncioreactor_install
|
||||
force_asyncioreactor_install()
|
||||
|
||||
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
|
||||
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbrynet.conf import Config, CLIConfig
|
||||
from lbrynet.utils import check_connection
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(lbrynet_name)
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
|
||||
def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
|
||||
|
||||
def print_help():
|
||||
print(dedent("""
|
||||
NAME
|
||||
lbrynet - LBRY command line client.
|
||||
|
||||
USAGE
|
||||
lbrynet [--data_dir=<blob and database directory>] [--wallet_dir=<wallet directory>]
|
||||
[--download_dir=<downloads directory>] <command> [<args>]
|
||||
|
||||
EXAMPLES
|
||||
lbrynet start # starts the daemon and listens for jsonrpc commands
|
||||
lbrynet help # display this message
|
||||
lbrynet help <command_name> # get help for a command(doesn't need the daemon to be running)
|
||||
lbrynet commands # list available commands
|
||||
lbrynet status # get the running status of the daemon
|
||||
lbrynet resolve what # resolve a name
|
||||
|
||||
lbrynet --wallet_dir=~/wallet2 start # start the daemon using an alternative wallet directory
|
||||
lbrynet --data_dir=~/lbry start # start the daemon using an alternative data directory
|
||||
|
||||
lbrynet --data_dir=~/lbry <command_name> # run a command on a daemon using an alternative data directory,
|
||||
# which can contain a full daemon_settings.yml config file.
|
||||
# Note: since the daemon is what runs the wallet and
|
||||
# downloads files, only the --data_dir setting is needed when
|
||||
# running commands. The wallet_dir and download_dir would only
|
||||
# by used when starting the daemon.
|
||||
"""))
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print(dedent(fn.__doc__))
|
||||
else:
|
||||
print("Invalid command name")
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
k = None
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
main = argparse.ArgumentParser('lbrynet', add_help=False)
|
||||
main.add_argument(
|
||||
'--version', dest='cli_version', action="store_true",
|
||||
help='Show lbrynet CLI version and exit.'
|
||||
)
|
||||
main.add_argument(
|
||||
'-h', '--help', dest='help', action="store_true",
|
||||
help='Show this help message and exit'
|
||||
)
|
||||
CLIConfig.contribute_args(main)
|
||||
sub = main.add_subparsers(dest='command')
|
||||
help = sub.add_parser('help', help='Detailed help for remote commands.')
|
||||
help.add_argument('help_command', nargs='*')
|
||||
start = sub.add_parser('start', help='Start lbrynet server.')
|
||||
start.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
Config.contribute_args(start)
|
||||
api = Daemon.get_api_definitions()
|
||||
for group in sorted(api):
|
||||
group_command = sub.add_parser(group, help=api[group]['doc'])
|
||||
group_command.set_defaults(group_doc=group_command)
|
||||
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
continue
|
||||
commands = group_command.add_subparsers(dest='subcommand')
|
||||
for command in api[group]['commands']:
|
||||
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
|
||||
for deprecated in Daemon.deprecated_methods:
|
||||
group_command = sub.add_parser(deprecated)
|
||||
group_command.add_subparsers(dest='subcommand')
|
||||
return main
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
return 0
|
||||
|
||||
elif args.command == 'start':
|
||||
|
||||
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
log.debug('Final Settings: %s', conf.settings_dict)
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
daemon = Daemon(conf)
|
||||
|
||||
if check_connection():
|
||||
from twisted.internet import reactor
|
||||
reactor._asyncioEventloop.create_task(daemon.start())
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
elif args.command == 'help':
|
||||
|
||||
if args.help_command:
|
||||
method = '_'.join(args.help_command)
|
||||
else:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
if method not in Daemon.callable_methods:
|
||||
print('Invalid command name: {method}')
|
||||
return 1
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
print(fn.__doc__)
|
||||
|
||||
elif args.command is not None:
|
||||
|
||||
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
method = args.command
|
||||
elif args.subcommand is not None:
|
||||
method = f'{args.command}_{args.subcommand}'
|
||||
else:
|
||||
args.group_doc.print_help()
|
||||
return 0
|
||||
|
||||
if method in Daemon.deprecated_methods:
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
return 0
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, command_args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(conf, method, params))
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
|
||||
def force_asyncioreactor_install():
|
||||
import sys
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
from twisted.internet import defer
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
@@ -1,7 +1,9 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted._threads import AlreadyQuit
|
||||
from ComponentManager import ComponentManager
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -14,7 +16,7 @@ class ComponentType(type):
|
||||
return klass
|
||||
|
||||
|
||||
class Component(object):
|
||||
class Component(metaclass=ComponentType):
|
||||
"""
|
||||
lbrynet-daemon component helper
|
||||
|
||||
@@ -22,11 +24,11 @@ class Component(object):
|
||||
methods
|
||||
"""
|
||||
|
||||
__metaclass__ = ComponentType
|
||||
depends_on = []
|
||||
component_name = None
|
||||
|
||||
def __init__(self, component_manager):
|
||||
self.conf: Config = component_manager.conf
|
||||
self.component_manager = component_manager
|
||||
self._running = False
|
||||
|
||||
@@ -37,37 +39,35 @@ class Component(object):
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup(self):
|
||||
async def _setup(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.start)
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop(self):
|
||||
async def _stop(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.stop)
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||