Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9b4d2b565e | ||
|
|
edbab2a844 | ||
|
|
314a45a7c2 | ||
|
|
4bebfe9e25 | ||
|
|
cbd85cd7d8 | ||
|
|
2728b1ed60 | ||
|
|
e24d626b07 | ||
|
|
ead3550588 | ||
|
|
53579dc613 | ||
|
|
e86f8daf17 | ||
|
|
1f1d352429 | ||
|
|
ce76b7d14f | ||
|
|
58a1f444da | ||
|
|
4e382b77a0 | ||
|
|
72ec32c486 | ||
|
|
2e230828fd | ||
|
|
65bc93f2fe | ||
|
|
02c868512f | ||
|
|
cd98367039 | ||
|
|
bc498e60b7 | ||
|
|
4019f2a2e1 | ||
|
|
36dea2afbf | ||
|
|
73927e2b57 | ||
|
|
a1292254bb | ||
|
|
8398d57024 | ||
|
|
be75c1bc5b | ||
|
|
fc40f5525a | ||
|
|
bd4dd8e114 | ||
|
|
1c1b5096be | ||
|
|
f99995a990 | ||
|
|
2c7a73fd52 | ||
|
|
3382c18f81 | ||
|
|
da511700a7 | ||
|
|
03db114ba8 | ||
|
|
c79ecdb0dc | ||
|
|
a8e3132559 | ||
|
|
7fb5b04bb4 | ||
|
|
2730aa9ad6 | ||
|
|
efb30a9560 | ||
|
|
9c28b9f523 | ||
|
|
4e42ae97ea | ||
|
|
4b1905e448 | ||
|
|
00b6808ded | ||
|
|
55c90e0657 | ||
|
|
99c2130be7 | ||
|
|
d21b1bcef1 | ||
|
|
1bf4048c46 | ||
|
|
c7b254ec88 | ||
|
|
f71696ee50 | ||
|
|
8d6a7155ef | ||
|
|
4e4eeae79c | ||
|
|
8735e1bb53 | ||
|
|
f06366b7dd | ||
|
|
a7d60a18e0 | ||
|
|
b8732e451b | ||
|
|
9ffcd9c1b1 | ||
|
|
3479c6ea8d | ||
|
|
9bcd550f0a | ||
|
|
091c49115c | ||
|
|
0f32260e2c | ||
|
|
e71389c7e3 | ||
|
|
d9acb7fdab | ||
|
|
9743162cda | ||
|
|
aef31991f9 | ||
|
|
efd46a9715 | ||
|
|
85cc843d36 | ||
|
|
d9ced3c6f7 | ||
|
|
967cc4c63c | ||
|
|
eba74b97bd | ||
|
|
1ec3218f18 | ||
|
|
30a8a951b2 | ||
|
|
5e0a82da54 | ||
|
|
b4224812ba | ||
|
|
613278de12 | ||
|
|
14b2bb3019 | ||
|
|
f9b4d46599 | ||
|
|
9caea54490 | ||
|
|
fa9b9233d1 | ||
|
|
77da61c44b | ||
|
|
56b744caec | ||
|
|
534fbb784f | ||
|
|
732fdaa377 | ||
|
|
ae0ac4f369 | ||
|
|
031e4401f2 | ||
|
|
10f8143a7b | ||
|
|
bc34c22e58 | ||
|
|
4c1a2b7e72 | ||
|
|
de455c0cce | ||
|
|
6a1a91f8f1 | ||
|
|
0a2a6c812d | ||
|
|
39195568cc | ||
|
|
ebec2fe955 | ||
|
|
ddde036e3a | ||
|
|
75d1f7f897 | ||
|
|
28613f1990 | ||
|
|
5db5d20f77 | ||
|
|
e0ff3171d8 | ||
|
|
2dab24ea14 | ||
|
|
1785c8ba6a | ||
|
|
59893e8fa8 | ||
|
|
ed23622521 | ||
|
|
51263f380e | ||
|
|
2725f384d9 | ||
|
|
a822d05692 | ||
|
|
1211fa031d | ||
|
|
e862f94e54 | ||
|
|
a324a3147e | ||
|
|
114f6ef8f8 | ||
|
|
0b5051823f | ||
|
|
a5b33fdb0b | ||
|
|
950c1e6d44 | ||
|
|
9f15573742 | ||
|
|
d4835153cd | ||
|
|
0941a207f1 | ||
|
|
4c22362cc1 | ||
|
|
69dacffc7a | ||
|
|
ae9a6b524f | ||
|
|
a1c99594b3 | ||
|
|
9bf533fc73 | ||
|
|
602eada430 | ||
|
|
01d906b06a | ||
|
|
05e556d066 | ||
|
|
1b944688c1 | ||
|
|
711c4f3196 | ||
|
|
0caa83c10c | ||
|
|
9bdba2172a | ||
|
|
0f2f5f91df | ||
|
|
49b255a2b5 | ||
|
|
da8ce591bc | ||
|
|
7bb76a41bb | ||
|
|
0fce59f4f2 | ||
|
|
9c88f820f9 | ||
|
|
4bc512bacb | ||
|
|
00fbe306c5 | ||
|
|
bea6e2913f | ||
|
|
c56ac12913 | ||
|
|
c393822c2b | ||
|
|
9a3c6ea18b | ||
|
|
1c8fb9a324 | ||
|
|
ea0c6eca32 | ||
|
|
2cfd29564b | ||
|
|
b85655ff58 | ||
|
|
ca2f0d827d | ||
|
|
42d40b2e1c | ||
|
|
82118e6692 | ||
|
|
5679ffb6a9 | ||
|
|
b3cd128779 | ||
|
|
2aec457a19 | ||
|
|
c3799b9c8c | ||
|
|
21fb3b9fae | ||
|
|
515b3124db | ||
|
|
c9a8016ae7 | ||
|
|
869a95c4c7 | ||
|
|
53f2147ecb | ||
|
|
fe9ec918e2 | ||
|
|
87b6eeeef5 | ||
|
|
8b9c6f5315 | ||
|
|
4c06db1901 | ||
|
|
dbe071cbb7 | ||
|
|
b5d7d961de | ||
|
|
791a060123 | ||
|
|
d8f3984477 | ||
|
|
6e17060819 | ||
|
|
363db1073e | ||
|
|
cfcc4205ab | ||
|
|
a577f38d80 | ||
|
|
8112658d70 | ||
|
|
363ad8452c | ||
|
|
4bfd2e630b | ||
|
|
d3099643e4 | ||
|
|
f5b4d9f384 | ||
|
|
522cb9cb82 | ||
|
|
5b297f165f | ||
|
|
c3f61b3e69 | ||
|
|
9f87d502c3 | ||
|
|
96927ec985 | ||
|
|
7c18a4a8f7 | ||
|
|
df006c71d3 | ||
|
|
8a73315541 | ||
|
|
11dc17d131 | ||
|
|
6a2fcbab67 | ||
|
|
da8256ad52 | ||
|
|
8bc611cc88 | ||
|
|
84bac4bab7 | ||
|
|
58dd576adc | ||
|
|
257d025d98 | ||
|
|
90033692d4 | ||
|
|
803cd98fef | ||
|
|
1cf32436f2 | ||
|
|
180e1d4768 | ||
|
|
57c3d2590c | ||
|
|
0eac1e6ed0 | ||
|
|
0b4c4cf6ca | ||
|
|
17b692d258 | ||
|
|
02475ff985 | ||
|
|
73a5828865 | ||
|
|
ffe04950bc | ||
|
|
9c2ec3a8eb | ||
|
|
30185d2548 | ||
|
|
4643309e6b | ||
|
|
61cb3e2a8d | ||
|
|
f258df95e6 | ||
|
|
ccbb5d88dc | ||
|
|
5d4e4a8753 | ||
|
|
89ada85f0f | ||
|
|
3c880a1684 | ||
|
|
12fc722712 | ||
|
|
ed3896f3e9 | ||
|
|
8a2a8ce05f | ||
|
|
18e63303bf | ||
|
|
c8142a2fb2 | ||
|
|
05c44eab08 | ||
|
|
4528a68f5c | ||
|
|
00876c7d43 | ||
|
|
b38614e29c | ||
|
|
82be3bb2cb | ||
|
|
06741f8560 | ||
|
|
c0be8ddcdf | ||
|
|
3253c6bd60 | ||
|
|
a870ca0a80 | ||
|
|
5da062482d | ||
|
|
7ac88ea3ea | ||
|
|
da6858f1c3 | ||
|
|
4bd3d49827 | ||
|
|
f4efb9c192 | ||
|
|
cd67ca6ce3 | ||
|
|
ecaa5249b4 | ||
|
|
4224d63603 | ||
|
|
f33b14e325 | ||
|
|
ca092262fb | ||
|
|
12097e3bfa | ||
|
|
6931d8e586 | ||
|
|
a49039117f | ||
|
|
632462e1d5 | ||
|
|
56e214feb6 | ||
|
|
d99fc519b5 | ||
|
|
dcff18323d | ||
|
|
89ecd2c571 | ||
|
|
18747d4349 | ||
|
|
2a8a984efa | ||
|
|
11cfd9083e | ||
|
|
f7793cea7f | ||
|
|
df4aa691cd | ||
|
|
113f8ff2d7 | ||
|
|
6097ebb7b8 | ||
|
|
1c8735295b | ||
|
|
6081b77167 | ||
|
|
e124b7fd63 | ||
|
|
8243349b20 | ||
|
|
a9868a4f17 | ||
|
|
dcdd4d8fd6 | ||
|
|
c525c8f5d1 | ||
|
|
16c18c59ff | ||
|
|
e4f8f50b08 | ||
|
|
15e89650ff | ||
|
|
8afe4d1af8 | ||
|
|
2353dbcb00 | ||
|
|
cd16383fa6 | ||
|
|
e19e2f3609 | ||
|
|
cc14ac4a46 | ||
|
|
c7161ba1ac | ||
|
|
8495e62329 | ||
|
|
d438bfdf0e | ||
|
|
4724f53bc4 | ||
|
|
06fab24437 | ||
|
|
cdf67de46c | ||
|
|
d6e7fde90a | ||
|
|
fd51d8ddf0 | ||
|
|
a50cbdcbb8 | ||
|
|
8b2cffd34e | ||
|
|
25b6fbb73e | ||
|
|
50152210e4 | ||
|
|
edc62c82e3 | ||
|
|
3a66a740d3 | ||
|
|
8a6a00f49e | ||
|
|
f08079dc9b | ||
|
|
dffaf745ec | ||
|
|
dc7193bb8f | ||
|
|
4f43a63131 | ||
|
|
c9c88055c3 | ||
|
|
8498eae90b | ||
|
|
73ee5786e2 | ||
|
|
ce40efa6cb | ||
|
|
1dda158087 | ||
|
|
37a03a28ce | ||
|
|
1ecb32b3ed | ||
|
|
0830c8bde2 | ||
|
|
a31783fc84 | ||
|
|
0a3e00b276 | ||
|
|
0b668fb1dd | ||
|
|
bfd0a322ae | ||
|
|
11d516794d | ||
|
|
28b56f05cc | ||
|
|
cb883e9316 | ||
|
|
6e37a0536a | ||
|
|
8f559c370a | ||
|
|
8f0f27da1a | ||
|
|
cb8730e696 | ||
|
|
c4b69055c7 | ||
|
|
bbbdf5486e | ||
|
|
a634c49e1c | ||
|
|
7ee5c1696a | ||
|
|
492cbc83c9 | ||
|
|
d2f01a214d | ||
|
|
c5b8859504 | ||
|
|
290d204659 | ||
|
|
1f231286b4 | ||
|
|
fa9378c2ae | ||
|
|
127c4e90a2 | ||
|
|
28cccd7574 | ||
|
|
7f7a6a619f | ||
|
|
55d47b9f19 | ||
|
|
371efb59c7 | ||
|
|
bc84d16de7 | ||
|
|
bbfb5a7853 | ||
|
|
8ce4039093 | ||
|
|
01d86204d0 | ||
|
|
990179655d | ||
|
|
52a42ce2fc | ||
|
|
05387bd743 | ||
|
|
27788829b0 | ||
|
|
af455ff52e | ||
|
|
6c0f805ec3 | ||
|
|
41204f7cbe | ||
|
|
fca79139ff | ||
|
|
e8bccec71c | ||
|
|
342ae231ea | ||
|
|
8b11521868 | ||
|
|
c26f83d81f | ||
|
|
e6f92a2dc7 | ||
|
|
4698910ab7 | ||
|
|
8c361aef1a | ||
|
|
2c8b99df76 | ||
|
|
10c72dcfd2 | ||
|
|
10dcc429f3 | ||
|
|
31b5026570 | ||
|
|
f38120fdfc | ||
|
|
06cd14e0ff | ||
|
|
3820401fce | ||
|
|
ddf907231d | ||
|
|
2df1092e1a | ||
|
|
d1414bc03a | ||
|
|
c9e3f1fd02 | ||
|
|
33c754f5a6 | ||
|
|
9669e06279 | ||
|
|
389ee2ad70 | ||
|
|
c87c8beeb4 | ||
|
|
f53cde824e | ||
|
|
63142df865 | ||
|
|
023f81674c | ||
|
|
b644125e93 | ||
|
|
c5699db30a | ||
|
|
1f45054d40 | ||
|
|
c1525760d8 | ||
|
|
b773cbbec6 | ||
|
|
043e2d0ab9 | ||
|
|
16221d08a9 | ||
|
|
6ccab8bcb8 | ||
|
|
a87834bacc | ||
|
|
105fa0b947 | ||
|
|
8db7c37fa7 | ||
|
|
54c29d4a8d | ||
|
|
a2eb0cad33 | ||
|
|
4c122f4ab0 | ||
|
|
a43127fffc | ||
|
|
66ab1386c5 | ||
|
|
22381f9427 | ||
|
|
5efbafec39 | ||
|
|
0ff308e3eb | ||
|
|
9531bdf396 | ||
|
|
978139a631 | ||
|
|
86117c431b | ||
|
|
585a17c830 | ||
|
|
04c119543f | ||
|
|
33c4856f00 | ||
|
|
ead32f32d8 | ||
|
|
e54dc9a01d | ||
|
|
c0ccfe2046 | ||
|
|
40374ec202 | ||
|
|
6ed2c5831e | ||
|
|
dc0699b929 | ||
|
|
eb1bdcc312 | ||
|
|
e9cfbea75f | ||
|
|
ccc94a0db9 | ||
|
|
bcf7a28fc8 | ||
|
|
1880f64da2 | ||
|
|
45b3837638 | ||
|
|
ee7ca0978f | ||
|
|
fc3573f823 | ||
|
|
8e15eddf9b | ||
|
|
880b974b84 | ||
|
|
5b78215063 | ||
|
|
bf7882d393 | ||
|
|
7a798616d6 | ||
|
|
6f909f9654 | ||
|
|
528ce42821 | ||
|
|
931038d478 | ||
|
|
60d6666abf | ||
|
|
ca041b5dc4 | ||
|
|
f43b38a5ea | ||
|
|
769329c90e | ||
|
|
23596b434a | ||
|
|
22f57f6490 | ||
|
|
99e4f9b00b | ||
|
|
bee67117c8 | ||
|
|
502e2227b5 | ||
|
|
d3173b157e | ||
|
|
c0a40605ad | ||
|
|
c20ba29c7a | ||
|
|
850f51140e | ||
|
|
5ec891c9ac | ||
|
|
5e936ef3cc | ||
|
|
d23e3cf066 | ||
|
|
bacc3a0b3a | ||
|
|
32e2ee8bf5 | ||
|
|
beeaec986a | ||
|
|
e32aa928ac | ||
|
|
1a562d70aa | ||
|
|
8dbadbd75b | ||
|
|
370a91fdaa | ||
|
|
1916b990b7 | ||
|
|
84d9461deb | ||
|
|
9e5ac6f73e | ||
|
|
5dbe2fdcec | ||
|
|
eabf9f1c4a | ||
|
|
bb4bf620f4 | ||
|
|
7abfbc4c5f | ||
|
|
b2d0c53bbe | ||
|
|
ddd8fd1a77 | ||
|
|
3af5e9dc5c | ||
|
|
23f1cf9c82 | ||
|
|
26f40b62f9 | ||
|
|
b15fba9c34 | ||
|
|
b0fd6e57df | ||
|
|
b04455f9c7 | ||
|
|
b4f5f2068f | ||
|
|
948d606268 | ||
|
|
f5c533e2f1 | ||
|
|
a0a122f0ef | ||
|
|
a2a29fc02c | ||
|
|
c9c84784c9 | ||
|
|
8fa2524e54 | ||
|
|
41257c07e6 | ||
|
|
e7b90055ed | ||
|
|
3f08b5f711 | ||
|
|
f0e7f38519 | ||
|
|
12b428f02d | ||
|
|
7708c4a15c | ||
|
|
9410cd9e77 | ||
|
|
0c42bc6382 | ||
|
|
b4364f661c | ||
|
|
63c71760dd | ||
|
|
25d9f008de | ||
|
|
41fbb1399c | ||
|
|
2cc2505cfd | ||
|
|
7fac63a37b | ||
|
|
c18c1725c4 | ||
|
|
5e6a742bd3 | ||
|
|
790baea191 | ||
|
|
e8e2a0d353 | ||
|
|
da3ed0756e | ||
|
|
05f7f5c73c | ||
|
|
b19e3adadd | ||
|
|
824f2f1d0f | ||
|
|
f515583ec4 | ||
|
|
c1a5806bc0 | ||
|
|
c66225ccf8 | ||
|
|
8f3b53197f | ||
|
|
4027733f2c | ||
|
|
3d606b95aa | ||
|
|
85845414b8 | ||
|
|
a95c49bd08 | ||
|
|
c422b5252e | ||
|
|
cb4c93bcd9 | ||
|
|
4edd13f4f4 | ||
|
|
a4de0056ab | ||
|
|
ee77bb0670 | ||
|
|
6482619fac | ||
|
|
b00c20180b | ||
|
|
001ebd02e3 | ||
|
|
15517732ae | ||
|
|
ebb6813df3 | ||
|
|
bfd822782d | ||
|
|
e66a1d3577 | ||
|
|
61d27f010e | ||
|
|
52d471d660 | ||
|
|
33c9a7a098 | ||
|
|
8201fc186d | ||
|
|
9d7dcf4af7 | ||
|
|
49b18a1fca | ||
|
|
26d2347812 | ||
|
|
29dea4901e | ||
|
|
1c2290dc8e | ||
|
|
8c664a939b | ||
|
|
12bc804622 | ||
|
|
593e8a94ad | ||
|
|
2c4e4d4526 | ||
|
|
74844f18ad | ||
|
|
93406fe07f | ||
|
|
820e3091bf | ||
|
|
c4b65fddac | ||
|
|
25dcf592a0 | ||
|
|
02bae15b02 | ||
|
|
b32b7091ad | ||
|
|
28e662a53e | ||
|
|
a440656a33 | ||
|
|
4e765b265f | ||
|
|
ec35eb2397 | ||
|
|
be4e1c99d5 | ||
|
|
031d9c28ed | ||
|
|
6d53a582cf | ||
|
|
64668d0043 | ||
|
|
92d423aefd | ||
|
|
0ec9b6545f | ||
|
|
003b3df378 | ||
|
|
f409d43dcc | ||
|
|
b2c2edfb12 | ||
|
|
e7dc841bb7 | ||
|
|
2303f88bd4 | ||
|
|
7291efac83 | ||
|
|
8c61185387 | ||
|
|
371591f96e | ||
|
|
98d3a4cb46 | ||
|
|
ab1bdb7eaf | ||
|
|
8ef3f81f0c | ||
|
|
cf56d8fa4a | ||
|
|
881efabc0c | ||
|
|
aef274577f | ||
|
|
5d6ca0aae6 | ||
|
|
9a4230dd96 | ||
|
|
3f4a4728e6 | ||
|
|
63cb2cf571 | ||
|
|
54b4b6984e | ||
|
|
98f90a4ad6 | ||
|
|
e29a91b69a | ||
|
|
fdcc5e6829 | ||
|
|
da99f3f22b | ||
|
|
992be881af | ||
|
|
a0cbd5af33 | ||
|
|
68f5588bd9 | ||
|
|
531fa63590 | ||
|
|
9048244cdc | ||
|
|
74754c9870 | ||
|
|
a7c2ad470c | ||
|
|
a132496643 | ||
|
|
f600038712 | ||
|
|
eae8f8a00e | ||
|
|
ac33663444 | ||
|
|
ba8b7ca090 | ||
|
|
1aa715c927 | ||
|
|
931925be15 | ||
|
|
e2ed9f21d2 | ||
|
|
e6cd4c9877 | ||
|
|
e2cc212eb8 | ||
|
|
74cf14df48 | ||
|
|
18c690d801 | ||
|
|
9e09d1fc38 | ||
|
|
454755169a | ||
|
|
bc9b990728 | ||
|
|
7ad46e70be | ||
|
|
4f9b5c4624 | ||
|
|
9744287370 | ||
|
|
59a5a3164f | ||
|
|
6fb3313eaf | ||
|
|
2bfbae9d4e | ||
|
|
daa0ff036d | ||
|
|
d642ba039d | ||
|
|
9350b9e127 | ||
|
|
9755e9b11f | ||
|
|
a64424f28c | ||
|
|
eee1acb24a | ||
|
|
0e76ca0767 | ||
|
|
e8afad7af8 | ||
|
|
f9642018ae | ||
|
|
41d96fec00 | ||
|
|
88f270180c | ||
|
|
76df0e3940 | ||
|
|
7128b6401e | ||
|
|
824fbcc25a | ||
|
|
fba91139e3 | ||
|
|
090840ee25 | ||
|
|
d3d8426430 | ||
|
|
40d4fb5641 | ||
|
|
d986fc27f3 | ||
|
|
9af1cae249 | ||
|
|
056ee2d1ce | ||
|
|
301e4ae0fa | ||
|
|
d359949a5c | ||
|
|
d4900fd607 | ||
|
|
f596d42ca2 | ||
|
|
04b36e2252 | ||
|
|
2add545ab6 | ||
|
|
70cf179139 | ||
|
|
b6086a7d6d | ||
|
|
62ab4c769d | ||
|
|
c920663444 | ||
|
|
1d30ae447a | ||
|
|
ff5980dfca | ||
|
|
e4b77a4a0a | ||
|
|
666b568eea | ||
|
|
998892ef0b | ||
|
|
847a79f526 | ||
|
|
07841a7803 | ||
|
|
4e55d674da | ||
|
|
3522f9af7d | ||
|
|
c27fe7af31 | ||
|
|
30c7559d3a | ||
|
|
7e2fad2397 | ||
|
|
06466db1b0 | ||
|
|
213a7ab66c | ||
|
|
a1650d7f03 | ||
|
|
5737d26ffe | ||
|
|
c85226d5a6 | ||
|
|
37b2297fb7 | ||
|
|
450ac7aa57 | ||
|
|
68e7c66e3a | ||
|
|
55f4e2939a | ||
|
|
b18679af53 | ||
|
|
d0b64ef255 | ||
|
|
e2f97ba0b4 | ||
|
|
c6e614ebec | ||
|
|
76b4b1e939 | ||
|
|
29ff39fdc9 | ||
|
|
d2e1eeb61e | ||
|
|
e86bada307 | ||
|
|
59750843d4 | ||
|
|
ab46a33913 | ||
|
|
5bf5a7d0b1 | ||
|
|
23f2a42372 | ||
|
|
14b4c06969 | ||
|
|
6d4c606abd | ||
|
|
eb039201e8 | ||
|
|
723105157f | ||
|
|
17d32ca696 | ||
|
|
0eb8e1609f | ||
|
|
85390302e2 | ||
|
|
993f2e54f9 | ||
|
|
87b7e5c742 | ||
|
|
8b9fef4ed5 | ||
|
|
d4cd8308f1 | ||
|
|
73eebdb0e3 | ||
|
|
0c30655605 | ||
|
|
e1455c3a0d | ||
|
|
205fd66a18 | ||
|
|
705b65fa3c | ||
|
|
fb2823f59e | ||
|
|
ddfaf51393 | ||
|
|
f1231bd8e5 | ||
|
|
19aa1eebd5 | ||
|
|
24e7b7797c | ||
|
|
caa8f73f7c | ||
|
|
0674b48223 | ||
|
|
5bf75ef139 | ||
|
|
6e310517e0 | ||
|
|
e2db99f7ab | ||
|
|
3151c10556 | ||
|
|
3ddf8680fb | ||
|
|
25ec8fde23 | ||
|
|
bcd026a1b6 | ||
|
|
1a6290eacc | ||
|
|
f4217d5593 | ||
|
|
8bba5c04db | ||
|
|
345301a9ee | ||
|
|
0f907a209b | ||
|
|
03c0ff8529 | ||
|
|
80dd02483e | ||
|
|
8452620f99 | ||
|
|
287e62b71b | ||
|
|
934a46a8b6 | ||
|
|
23e06ba872 | ||
|
|
1fd87b7caa | ||
|
|
f7c105fd80 | ||
|
|
9e2cda0cdc | ||
|
|
4fb5e932e6 | ||
|
|
fe4dac8236 | ||
|
|
2a45c91175 | ||
|
|
439a840700 | ||
|
|
a38e5c855e | ||
|
|
90d3a0ef9c | ||
|
|
f989814e69 | ||
|
|
9f4ccfe7c3 | ||
|
|
520ac9b50a | ||
|
|
dbb6c6f3a4 | ||
|
|
993d606bc6 | ||
|
|
ffb48ef287 | ||
|
|
62fcbc23e3 | ||
|
|
a43b726f32 | ||
|
|
3bdd33567b | ||
|
|
6bdb0280be | ||
|
|
77746cea5f | ||
|
|
25cede0dc3 | ||
|
|
acfe034990 | ||
|
|
e2021deae1 | ||
|
|
37cc257555 | ||
|
|
a22758d5c7 | ||
|
|
aa8375673b | ||
|
|
5831e253cd | ||
|
|
34f5bc93ae | ||
|
|
aa45b0e121 | ||
|
|
8e0f71ac74 | ||
|
|
55445b1bb7 | ||
|
|
3cc8a4b168 | ||
|
|
8859c1b0c1 | ||
|
|
867275461c | ||
|
|
2190f4ac85 | ||
|
|
b51d3a28e3 | ||
|
|
7720724ec0 | ||
|
|
dbd71c93e7 | ||
|
|
d137528f67 | ||
|
|
e292abceee | ||
|
|
8de6bd7c7a | ||
|
|
bb9b221cf6 | ||
|
|
5a1bab2eaa | ||
|
|
4f87e87d3e | ||
|
|
c7106dfc97 | ||
|
|
11b1f8c7af | ||
|
|
f5e6ff0c32 | ||
|
|
71123c0de1 | ||
|
|
35b2e56bd4 | ||
|
|
41abd0a171 | ||
|
|
cea4eca775 | ||
|
|
6b22debb79 | ||
|
|
630d0bf5be | ||
|
|
19fe101d73 | ||
|
|
75cc0dcce9 | ||
|
|
0e8e489ad6 | ||
|
|
b03e7e14f3 | ||
|
|
e2050b4fc1 | ||
|
|
d1aa69bfa3 | ||
|
|
7e2456fa69 | ||
|
|
ecac2ae2ca | ||
|
|
bfc02dd3e5 | ||
|
|
c527c32e5f | ||
|
|
777419b4a9 | ||
|
|
832a32474f | ||
|
|
f2ddc9bd98 | ||
|
|
d053db8dfd | ||
|
|
a58a81a841 | ||
|
|
df40c7fb76 | ||
|
|
ef65d8dd1c | ||
|
|
a8780f03a8 | ||
|
|
61e78c1b69 | ||
|
|
a167beaad1 | ||
|
|
4c76e1b98f | ||
|
|
f614a13db4 | ||
|
|
6fae07d29e | ||
|
|
2126f69c93 | ||
|
|
6f0581104a | ||
|
|
47c92f04d2 | ||
|
|
06be60e3a3 | ||
|
|
45a6228b58 | ||
|
|
773cbe3ace | ||
|
|
c6648ff6bc | ||
|
|
f737b3aaf7 | ||
|
|
41d937d1b4 | ||
|
|
8e349f17c1 | ||
|
|
4ab5ace668 | ||
|
|
f46ae75f73 | ||
|
|
61c5a5b2a4 | ||
|
|
2c137a99d1 | ||
|
|
d1ed8a769a | ||
|
|
4bb3e2a154 | ||
|
|
a5e8c561e4 | ||
|
|
7b6e233d22 | ||
|
|
ac8a63bc33 | ||
|
|
15ebc39f95 | ||
|
|
46cb6b204a | ||
|
|
b219806959 | ||
|
|
8fc6feb316 | ||
|
|
7377158631 | ||
|
|
86ba7ffd5c | ||
|
|
ed227a239e | ||
|
|
7360015fa4 | ||
|
|
04ee2f786d | ||
|
|
43451c485f | ||
|
|
565f26bdad | ||
|
|
b7478ba817 | ||
|
|
1c92b5d2ea | ||
|
|
b744a54340 | ||
|
|
8805d1d0cf | ||
|
|
b674e5c57a | ||
|
|
139af85b52 | ||
|
|
ee0f7375d6 | ||
|
|
893fe8823e | ||
|
|
f54d42b8fa | ||
|
|
3dc1a523cf | ||
|
|
90438ddf9f | ||
|
|
08e336e476 | ||
|
|
eebf32362a | ||
|
|
0e864f5df1 | ||
|
|
854dea797e | ||
|
|
c88631d910 | ||
|
|
897fba35df | ||
|
|
74b4e4508a | ||
|
|
74a872d763 | ||
|
|
3b65f5962c | ||
|
|
5fa2dfeca7 | ||
|
|
7cac8dad7a | ||
|
|
b80a1f66da | ||
|
|
20da97de6a | ||
|
|
bb9d3bff74 | ||
|
|
ec873129d5 | ||
|
|
a90f413f94 | ||
|
|
db1abd8cca | ||
|
|
18a0b07181 | ||
|
|
8fd7489d99 | ||
|
|
ee84253b90 | ||
|
|
06e45ee32a | ||
|
|
7696f30009 | ||
|
|
c211d87164 | ||
|
|
dc57b62157 | ||
|
|
c71ac5b535 | ||
|
|
5a467f55ba | ||
|
|
9a24bddead | ||
|
|
d3a63c38fc | ||
|
|
08aa024499 | ||
|
|
5db1a4d328 | ||
|
|
e95adecd41 | ||
|
|
8b68a752ed | ||
|
|
51a5510b38 | ||
|
|
2c9c0da501 | ||
|
|
60195ed6d4 | ||
|
|
e37fbc4fdb | ||
|
|
3e774fc158 | ||
|
|
c62ee6bb0c | ||
|
|
3aea476014 | ||
|
|
8c5f7a9849 | ||
|
|
6bc3e8a3f4 | ||
|
|
6c571b5227 | ||
|
|
8628b127f7 | ||
|
|
91d673a539 | ||
|
|
851ab7ba28 | ||
|
|
7bc1bc487f | ||
|
|
05d50c1821 | ||
|
|
40b6476cf7 | ||
|
|
13bfcb3561 | ||
|
|
1a850b0256 | ||
|
|
78b0f923f4 | ||
|
|
1aa7981ab1 | ||
|
|
17d9c0ab97 | ||
|
|
8d1404bd1e | ||
|
|
4634179a7e | ||
|
|
23872c77e9 | ||
|
|
cac8267e85 | ||
|
|
267c6cbaca | ||
|
|
03b214a9d2 | ||
|
|
8a026071eb | ||
|
|
9fb1cd0944 | ||
|
|
a338d1f0e1 | ||
|
|
8a338b37b2 | ||
|
|
2b2fa29db5 | ||
|
|
ddcfa7f97c | ||
|
|
ddf24de16d | ||
|
|
8277df1034 | ||
|
|
3bed768d10 | ||
|
|
e03bba0b62 | ||
|
|
e61260a263 | ||
|
|
a9261b2032 | ||
|
|
9366a6e561 | ||
|
|
488c04543d | ||
|
|
cb2bb6ee6b | ||
|
|
0bb62515a8 | ||
|
|
f535d96929 | ||
|
|
15c5075d38 | ||
|
|
2449604844 | ||
|
|
6035a84675 | ||
|
|
1151019186 | ||
|
|
8075ced1af | ||
|
|
1b2c46e224 | ||
|
|
7f6e9fff30 | ||
|
|
bb1b7b66f7 | ||
|
|
deae84a563 | ||
|
|
20f113d3da | ||
|
|
61702ba1e4 | ||
|
|
d100a24a29 | ||
|
|
d06211015d | ||
|
|
93ff4f63b5 | ||
|
|
86aa8e1de0 | ||
|
|
495a48add1 | ||
|
|
a6457d2c0a | ||
|
|
5c429e15bd | ||
|
|
05725e0dc9 | ||
|
|
53acb2d9cd | ||
|
|
f050337dbc | ||
|
|
6d7b09a3aa | ||
|
|
dd87aa3ce6 | ||
|
|
b602b09ca3 | ||
|
|
628defa891 | ||
|
|
962edb8e4c | ||
|
|
5dbf9034ec | ||
|
|
a67aab0797 | ||
|
|
52500bfbf5 | ||
|
|
f4790a46ee | ||
|
|
fd5b4f403a | ||
|
|
c393bd97e8 | ||
|
|
d32bb9270a | ||
|
|
9e7b69f4a0 | ||
|
|
3c60b35e2e | ||
|
|
69c0da695b | ||
|
|
68887d145f | ||
|
|
f37808859f | ||
|
|
14c835b41e | ||
|
|
abc122f77b | ||
|
|
0ababb7f3b | ||
|
|
30af8a0e20 | ||
|
|
389fc67d51 | ||
|
|
c2fc15160f | ||
|
|
40d4f5a292 | ||
|
|
38e6b4e6c6 | ||
|
|
3541625825 | ||
|
|
f8f83fccf1 | ||
|
|
b3175af16f | ||
|
|
5f8a06443c | ||
|
|
4e0194d576 | ||
|
|
16fc416d88 | ||
|
|
6faee910de | ||
|
|
f7308beee9 | ||
|
|
5233dc303e | ||
|
|
3d0913ca94 | ||
|
|
c37bb0d1ed | ||
|
|
a4a9a2b0b0 | ||
|
|
9e320921a9 | ||
|
|
21602dc5a1 | ||
|
|
2cf08fda2d | ||
|
|
c2c7d2b327 | ||
|
|
585f802a91 | ||
|
|
26cf04e0c1 | ||
|
|
3bb8f4ac21 | ||
|
|
2afc4c25a2 | ||
|
|
a5c25ff887 | ||
|
|
e823bb2012 | ||
|
|
eec25791a5 | ||
|
|
18a47a048a | ||
|
|
6936dbd588 | ||
|
|
621530b3b8 | ||
|
|
123fda1844 | ||
|
|
1299c23bcc | ||
|
|
1747f53276 | ||
|
|
669a567545 | ||
|
|
6bdc9069c9 | ||
|
|
ad1e2650e3 | ||
|
|
1bf1b7624c | ||
|
|
0b53fde352 | ||
|
|
4eb10b56c1 | ||
|
|
227323b7a4 | ||
|
|
1256beea96 | ||
|
|
01cc4f28e0 | ||
|
|
689ac431d3 | ||
|
|
ccfb1f3921 | ||
|
|
cc008766ec | ||
|
|
4b1530b554 | ||
|
|
5eecae8889 | ||
|
|
5aa10031bc | ||
|
|
52010ade41 | ||
|
|
101985ca11 | ||
|
|
8f61fb3fc4 | ||
|
|
39b2e44492 | ||
|
|
26bc6d05a0 | ||
|
|
47f71fd363 | ||
|
|
4574e3317c | ||
|
|
b59c098d58 | ||
|
|
22ff87b885 | ||
|
|
42edf49468 | ||
|
|
da0b4e9a1a | ||
|
|
7d5dd1ff51 | ||
|
|
d1ccac1b04 | ||
|
|
a289886365 | ||
|
|
2c7fc3fb46 | ||
|
|
fc24d9b5aa | ||
|
|
da366495b0 | ||
|
|
552ed77882 | ||
|
|
c53e0baca6 | ||
|
|
aa3353ae49 | ||
|
|
521ad57b61 | ||
|
|
f62456bb68 | ||
|
|
f7864e8cb7 | ||
|
|
1690e71ad1 | ||
|
|
7d02e7d370 | ||
|
|
7404b127bf | ||
|
|
10810226ff | ||
|
|
a2497e0854 | ||
|
|
c0f22decd7 | ||
|
|
3da82ce895 | ||
|
|
ba90fb3349 | ||
|
|
618d0ebd5b | ||
|
|
f301741807 | ||
|
|
0044d6cf75 | ||
|
|
5a76a03cb6 | ||
|
|
09846413bd | ||
|
|
a8cb255d24 | ||
|
|
6172924021 | ||
|
|
7bb570c051 | ||
|
|
d5d0d0e31c | ||
|
|
83e797d7f6 | ||
|
|
6c2b3d9ed6 | ||
|
|
c6f4c17597 | ||
|
|
d19afd856b | ||
|
|
29e59e783a | ||
|
|
107860ea19 | ||
|
|
a9bfd8d6ae | ||
|
|
9ee95d57fc | ||
|
|
4a8a8b8d33 | ||
|
|
c825d88dbf | ||
|
|
15ade508aa | ||
|
|
53df361d23 | ||
|
|
8fe15f507b | ||
|
|
3dde7af576 | ||
|
|
7af6e9e0dd | ||
|
|
56d394fb5f | ||
|
|
3585d861ff | ||
|
|
f181af8547 | ||
|
|
84baa5e065 | ||
|
|
781b915717 | ||
|
|
fa29c92760 | ||
|
|
fec917b9c1 | ||
|
|
4f3b5cd802 | ||
|
|
250831a86a | ||
|
|
323bccb0ae | ||
|
|
0084d4684f | ||
|
|
740fad5cbe | ||
|
|
fa8190356a | ||
|
|
4ca33505b5 | ||
|
|
20b6b22334 | ||
|
|
b8c91c61bb | ||
|
|
c30ea04959 | ||
|
|
27a09d4a4f | ||
|
|
23bf43f40c | ||
|
|
6f60b8b827 | ||
|
|
965f8f1238 | ||
|
|
ab78b7f5fa | ||
|
|
083092ad5e | ||
|
|
7bce37a720 | ||
|
|
14ac2bac39 | ||
|
|
32fa2460ff | ||
|
|
f796f701f2 | ||
|
|
fe4ea9b33a | ||
|
|
c278972f26 | ||
|
|
bce41ddab4 | ||
|
|
666a62ae3d | ||
|
|
5bf20096b2 | ||
|
|
52099c7da7 | ||
|
|
9619b8dfca | ||
|
|
e8faadc324 | ||
|
|
667d5ac622 | ||
|
|
afba045c29 | ||
|
|
b64fa51567 | ||
|
|
fe3e25982c | ||
|
|
9ad55e93b9 | ||
|
|
9e7e168175 | ||
|
|
f84df50d7f | ||
|
|
f0ed4e5513 | ||
|
|
0af445cbbc | ||
|
|
99fe28366f | ||
|
|
39005ffc14 | ||
|
|
239e5c2126 | ||
|
|
28e88d89a5 | ||
|
|
c85b795fc6 | ||
|
|
979f8efa79 | ||
|
|
4c4bccfe73 | ||
|
|
8cb0f99e30 | ||
|
|
44554b63f4 | ||
|
|
6076861cd0 | ||
|
|
8b427a92ac | ||
|
|
6c4736d1bd | ||
|
|
9c50cfdcdb | ||
|
|
8f80e8a3da | ||
|
|
cd44ae02a3 | ||
|
|
848f1f025c | ||
|
|
d2cc8355dc | ||
|
|
ee00d1984a | ||
|
|
2fb71610a4 | ||
|
|
948be0f040 | ||
|
|
b9bb337a07 | ||
|
|
d93f86030c | ||
|
|
d9e1e88c05 | ||
|
|
f579c8dbb8 | ||
|
|
0005ba0715 | ||
|
|
13d74d1f76 | ||
|
|
ddf678af8a | ||
|
|
2c6d6d1602 | ||
|
|
7d0e9f6ab9 | ||
|
|
200a22c93b | ||
|
|
31a90e7f60 | ||
|
|
2bf098b5a3 | ||
|
|
458561fe6f | ||
|
|
157d6dca93 | ||
|
|
7b1b7fbf13 | ||
|
|
876cdce51f | ||
|
|
205a10bc8e | ||
|
|
ef8d1cfdc2 | ||
|
|
879c01ba00 | ||
|
|
8bb7517f32 | ||
|
|
70d9b3ed74 | ||
|
|
f39a21cf7b | ||
|
|
67b169c5a1 | ||
|
|
ff10546152 | ||
|
|
bd6fe4f986 | ||
|
|
360f4e50a0 | ||
|
|
f11f486f5a | ||
|
|
fe53cc97d9 | ||
|
|
baf6ff3f50 | ||
|
|
051e63d76b | ||
|
|
8cdda7d845 | ||
|
|
6a8789050c | ||
|
|
9140c95a0c | ||
|
|
4b52c88d45 | ||
|
|
b7dd9c176a | ||
|
|
6f2e2e727e | ||
|
|
48b137fe4c | ||
|
|
a6326ee7a1 | ||
|
|
c91034ee4c | ||
|
|
7c505f2e23 | ||
|
|
bfcd84a568 | ||
|
|
a314c530bc | ||
|
|
4cb2a8f2aa | ||
|
|
9d272befcd | ||
|
|
1695b26416 | ||
|
|
91fb78b077 | ||
|
|
0bbc8f5b7e | ||
|
|
464bd11c3a | ||
|
|
b88e771eb2 | ||
|
|
7a76763610 | ||
|
|
02d2bad03f | ||
|
|
f65a50a6a3 | ||
|
|
0a9ef07787 | ||
|
|
0e9ad16a2e | ||
|
|
cd0a9ba9e8 | ||
|
|
e08d0eb8f9 | ||
|
|
82f9cc57fd | ||
|
|
ae53129b25 | ||
|
|
543f65fc24 | ||
|
|
896a16c15d | ||
|
|
3d7dee1208 | ||
|
|
e03db79f79 | ||
|
|
0b0150ad73 | ||
|
|
0490250da4 | ||
|
|
dd952ddbe3 | ||
|
|
776fe41e03 | ||
|
|
8b1ea0a6a7 | ||
|
|
183617c85b | ||
|
|
386c67fa70 | ||
|
|
ba7f326e72 | ||
|
|
b9caf2c4a9 | ||
|
|
3188165514 | ||
|
|
1c036cce78 | ||
|
|
a2e648fcfc | ||
|
|
b75a89e876 | ||
|
|
82a3b5e9c5 | ||
|
|
7ecc8103d8 | ||
|
|
6b72b4de11 | ||
|
|
8e1ad0aa8b | ||
|
|
5b7f80487f | ||
|
|
ebc400804f | ||
|
|
9940327b9a | ||
|
|
b300afb504 | ||
|
|
0941618048 | ||
|
|
abc7b11e26 | ||
|
|
85dee27c92 | ||
|
|
5ae3485411 | ||
|
|
8f98469969 | ||
|
|
e1cdbdc91d | ||
|
|
908c9a3bfa | ||
|
|
f45a2cba32 | ||
|
|
763fefb33f | ||
|
|
ce7463f684 | ||
|
|
cd65b6fd13 | ||
|
|
472bb6af1b | ||
|
|
7f96e35b0e | ||
|
|
f9fef3d586 | ||
|
|
ec4f9011b9 | ||
|
|
d634c009d3 | ||
|
|
1872256d7a | ||
|
|
81d107d99c | ||
|
|
a7589914d0 | ||
|
|
0736fa506b | ||
|
|
dfec3c1a15 | ||
|
|
edc89be794 | ||
|
|
d41881596d | ||
|
|
3b34bddf0a | ||
|
|
23c012f417 | ||
|
|
2c8d643f4b | ||
|
|
0abc081edc | ||
|
|
11b36ff4c8 | ||
|
|
c407d32f5d | ||
|
|
87cc487051 | ||
|
|
5ea945ca0b | ||
|
|
1f06339d5a | ||
|
|
09d248b3cc | ||
|
|
28bf5215e4 | ||
|
|
f3a71245e8 | ||
|
|
37fee71898 | ||
|
|
61a354fa60 | ||
|
|
1d4b2c3527 | ||
|
|
694b5b9ee5 | ||
|
|
838508f342 | ||
|
|
694db88ac1 | ||
|
|
cb7c849273 | ||
|
|
9d82be6664 | ||
|
|
66745b0820 | ||
|
|
f845729adb | ||
|
|
6a587d03cf | ||
|
|
affb7d837a | ||
|
|
4ad6405f35 | ||
|
|
906e6b238e | ||
|
|
297865902b | ||
|
|
ab15074ae4 | ||
|
|
b68a1838f2 | ||
|
|
43b2cd1433 | ||
|
|
0036685ece | ||
|
|
d06c670a12 | ||
|
|
6d6cfbc7cc | ||
|
|
3c977f961f | ||
|
|
59c4593bf4 | ||
|
|
5bccfdb244 | ||
|
|
10cb20d08f | ||
|
|
693fef1964 | ||
|
|
fe7700d726 | ||
|
|
fee6de1c1f | ||
|
|
7d04ae2237 | ||
|
|
c748572bf6 | ||
|
|
660bf74d42 | ||
|
|
4db3be345b | ||
|
|
dcea8a8c78 | ||
|
|
815696485f | ||
|
|
b316b56885 | ||
|
|
269dc47b69 | ||
|
|
a7e3246a82 | ||
|
|
96ef5089fc | ||
|
|
c0d96a5733 | ||
|
|
7de4340c38 | ||
|
|
7dd5e7ceeb | ||
|
|
9de7354ea0 | ||
|
|
cfbdda411a | ||
|
|
96c4837cb8 | ||
|
|
2294e6bf36 | ||
|
|
4cffc32328 | ||
|
|
d1b23e25d9 | ||
|
|
e5ba70c3ba | ||
|
|
c88e4aecf5 | ||
|
|
469df84990 | ||
|
|
a4ec706b06 | ||
|
|
30533a7986 | ||
|
|
39b47bc569 | ||
|
|
96104e65ed | ||
|
|
e9bab33e2a | ||
|
|
3b6006d759 | ||
|
|
9535a24706 | ||
|
|
8f2be46536 | ||
|
|
9ca80a7cf7 | ||
|
|
72078f4af3 | ||
|
|
ff9ea37e0c | ||
|
|
9baf261cb3 | ||
|
|
f03a15d503 | ||
|
|
1a70a20c8b | ||
|
|
359cb83fab | ||
|
|
0893858385 | ||
|
|
e1997a6852 | ||
|
|
0f4fee475f | ||
|
|
864e963dea | ||
|
|
3d0debd775 | ||
|
|
6cc2b48a06 | ||
|
|
dee0917642 | ||
|
|
395a68b261 | ||
|
|
0d066b2d11 | ||
|
|
353ede26b8 | ||
|
|
46cfa20f65 | ||
|
|
30b1dd3ce6 | ||
|
|
da601818a3 | ||
|
|
6fa624ec8a | ||
|
|
779ba8e214 | ||
|
|
80f548fdfd | ||
|
|
6f9ec2fa66 | ||
|
|
40437e4a82 | ||
|
|
2ee6fac014 | ||
|
|
8fe5eb514a | ||
|
|
c4312cc387 | ||
|
|
110a9b1e0f | ||
|
|
035a1cf758 | ||
|
|
d4785849e1 | ||
|
|
b41756e38a | ||
|
|
60ee278519 | ||
|
|
b49e44926a | ||
|
|
81187da5b7 | ||
|
|
e70ecc8ac9 | ||
|
|
862055a6de | ||
|
|
7076c0c870 | ||
|
|
2cd6c644a0 | ||
|
|
0edacbe4c8 | ||
|
|
4b330d0e9f | ||
|
|
8ce08f7da1 | ||
|
|
127987f3ba | ||
|
|
3c7eec9456 | ||
|
|
5d45345398 | ||
|
|
b360eb6181 | ||
|
|
369cd516c0 | ||
|
|
24320d6444 | ||
|
|
1951ea09cd | ||
|
|
4d391e7429 | ||
|
|
fb88cdb445 | ||
|
|
572f1d1f6e | ||
|
|
27a8114b6f | ||
|
|
6f3873d99b | ||
|
|
e62e2cc440 | ||
|
|
a5797cdeb3 | ||
|
|
3dd99fdc92 | ||
|
|
0c2265a8da | ||
|
|
01811621a6 | ||
|
|
229ed0d7dc | ||
|
|
7f800ce5bb | ||
|
|
aa291ca79a | ||
|
|
813267ae18 | ||
|
|
fbf7928412 | ||
|
|
7167d47631 | ||
|
|
8b1bb673c1 | ||
|
|
417a8f719f | ||
|
|
ddb88eb1c9 | ||
|
|
d5f0001950 | ||
|
|
4a07b56e8a | ||
|
|
9da1b3c5b7 | ||
|
|
f9f07c47e2 | ||
|
|
f5213fb77a | ||
|
|
48a61605de | ||
|
|
76fb7697ff | ||
|
|
b71a3fee43 | ||
|
|
3a91896d8a | ||
|
|
36ae0d5f20 | ||
|
|
3f727d892c | ||
|
|
c104ed3f8b | ||
|
|
9416376a34 | ||
|
|
66e139c566 | ||
|
|
d0a82c68df | ||
|
|
93e2b3e20a | ||
|
|
2e59e5e3b8 | ||
|
|
727ea12dbd | ||
|
|
ea3225c897 | ||
|
|
bfcfa9dc3c | ||
|
|
ea21821558 | ||
|
|
a9ae1a1016 | ||
|
|
3f5efb1fa3 | ||
|
|
9ba38c84cc | ||
|
|
a5d3c0bbe1 | ||
|
|
d4aa109b73 | ||
|
|
bdeccb49b8 | ||
|
|
35ce7bbd61 | ||
|
|
293933163f | ||
|
|
22fc26797d | ||
|
|
8a95c8b7af | ||
|
|
33674f8cc3 | ||
|
|
6799de645b | ||
|
|
06a88599df | ||
|
|
03877f7ebd | ||
|
|
9f2dbd4291 | ||
|
|
6b7d575e95 | ||
|
|
640db41555 | ||
|
|
d860c2a2b4 | ||
|
|
733bb3bccf | ||
|
|
307751277d | ||
|
|
62c2539475 | ||
|
|
5b15da2c55 | ||
|
|
830f5ff6e3 | ||
|
|
ea1f260ed9 | ||
|
|
3e86254324 | ||
|
|
b5311dd1c3 | ||
|
|
a8a581d35c | ||
|
|
5f129faf32 | ||
|
|
018d78be6f | ||
|
|
b5a3929bb5 | ||
|
|
b67b4b2ef6 | ||
|
|
bed3f551fd | ||
|
|
5b0b8a19ff | ||
|
|
988c9595c7 | ||
|
|
92700808eb | ||
|
|
0dd9aa0d67 | ||
|
|
9cf1d6b710 | ||
|
|
d8d462f43c | ||
|
|
3f22f39ce1 | ||
|
|
e647663c34 | ||
|
|
b5987c75da | ||
|
|
32a973943f | ||
|
|
99c060719a | ||
|
|
d9cd215a60 | ||
|
|
2d3b1fd4eb | ||
|
|
19c2264429 | ||
|
|
ae4b984f0b | ||
|
|
897c113db1 | ||
|
|
50e5a59420 | ||
|
|
8735112f70 | ||
|
|
5a0b5dc680 | ||
|
|
d91e9aa910 | ||
|
|
be1803d558 | ||
|
|
84bbf2f76e | ||
|
|
fe7ea7c679 | ||
|
|
782e197e66 | ||
|
|
88be425ab2 | ||
|
|
7025c5102a | ||
|
|
1ee9b3c7a6 | ||
|
|
4f60a98eb2 | ||
|
|
a71e87a398 | ||
|
|
0ecfad0ae9 | ||
|
|
e909ae566b | ||
|
|
dc2f0adb3e | ||
|
|
180d981ed8 | ||
|
|
fd9bc6214a | ||
|
|
b3f3ed8026 | ||
|
|
967700dc28 | ||
|
|
3ad4ad50ec | ||
|
|
7f980446db | ||
|
|
04ced2e975 | ||
|
|
2253943eba | ||
|
|
a6fcd5d1f2 | ||
|
|
f245822814 | ||
|
|
aa3aff91d0 | ||
|
|
5537dd878f | ||
|
|
5e4e619708 | ||
|
|
d741c705cd | ||
|
|
d83abd81f8 | ||
|
|
640e000b9f | ||
|
|
8d961d6ce6 | ||
|
|
0a612ba36c | ||
|
|
eb7cc8463b | ||
|
|
50f0430d04 | ||
|
|
6e5ea935f6 | ||
|
|
c4eb3de5b2 | ||
|
|
f79b651b05 | ||
|
|
c130879ef7 | ||
|
|
5c391f4bb4 | ||
|
|
b853656aa6 | ||
|
|
8222f940b4 | ||
|
|
71b989b29b | ||
|
|
7e2f7e6f68 | ||
|
|
aefb6c5dc8 | ||
|
|
b892b81dad | ||
|
|
4d63629919 | ||
|
|
4727ee0403 | ||
|
|
fc47e8074f | ||
|
|
ff7829a078 | ||
|
|
d05bbea131 | ||
|
|
e08a681c8f | ||
|
|
6a72b3683e | ||
|
|
b201fc331c | ||
|
|
27337a9243 | ||
|
|
b671a3c2ba | ||
|
|
1cc6b7658c | ||
|
|
12e2827286 | ||
|
|
548d37929e | ||
|
|
eb783b6b6f | ||
|
|
d1b06c3688 | ||
|
|
d16b3429cd | ||
|
|
1e4b5cb450 | ||
|
|
a9da6e4bdf | ||
|
|
a099e2db6a | ||
|
|
7dbeb1f2cf | ||
|
|
550fd185a3 | ||
|
|
1720cce3b0 | ||
|
|
078b254d1d | ||
|
|
6057dc0804 | ||
|
|
5ea7c4f016 | ||
|
|
4334cdace1 | ||
|
|
6803bfae99 | ||
|
|
c49b0fdd49 | ||
|
|
bf3c56f0ec | ||
|
|
bbcecea28d | ||
|
|
a936782ddc | ||
|
|
7ac0fc5a24 | ||
|
|
e115158f86 | ||
|
|
bf34fa39b4 | ||
|
|
5dd29da84f | ||
|
|
2107ffdd18 | ||
|
|
492fc510fa | ||
|
|
c75d240a65 | ||
|
|
dbeb6bc3bc | ||
|
|
4d79a7828c | ||
|
|
aecafbbebd | ||
|
|
130f9cfc4d | ||
|
|
98d02b3fce | ||
|
|
4d52a33ee5 | ||
|
|
7ad916f116 | ||
|
|
05b47a9180 | ||
|
|
9a412bbc76 | ||
|
|
b3a16f95f7 | ||
|
|
871b3be09a | ||
|
|
7085b1e3c3 | ||
|
|
a9e4343775 | ||
|
|
3d4d895d9b | ||
|
|
3d89e3fd93 | ||
|
|
a8e2dd3f2a | ||
|
|
3ae0664fa9 | ||
|
|
98b9f2cf73 | ||
|
|
6fa65ebb9f | ||
|
|
add163a092 | ||
|
|
fcfed3f441 | ||
|
|
7b84d9b10a | ||
|
|
e9a1dbca4b | ||
|
|
145a2654d2 | ||
|
|
9f14cdb1d7 | ||
|
|
5cefa2e995 | ||
|
|
13cbe0fd42 | ||
|
|
c8bb0c0bc8 | ||
|
|
4cc941fbec | ||
|
|
b010192f8e | ||
|
|
2b1af8eea2 | ||
|
|
6ea1f6f78c | ||
|
|
1215470cc1 | ||
|
|
94195b7018 | ||
|
|
3c9c178bdc | ||
|
|
cc64547920 | ||
|
|
a23f2ca956 | ||
|
|
4dacc52ecb | ||
|
|
ccaeecfd46 | ||
|
|
3e45449e8c | ||
|
|
f352a3c597 | ||
|
|
d9a08f0c03 | ||
|
|
c77f24dde3 | ||
|
|
48c31a2119 | ||
|
|
5a578d6d79 | ||
|
|
8cdbc847d1 | ||
|
|
54dbf66cb7 | ||
|
|
177d384126 | ||
|
|
561fe757f3 | ||
|
|
c670ab638f | ||
|
|
eea5cbbafc | ||
|
|
e4c61a4a13 | ||
|
|
0f0afadc94 | ||
|
|
659563f27a | ||
|
|
cbd8a68d0f | ||
|
|
54229282a4 | ||
|
|
514575d0f1 | ||
|
|
756427560c | ||
|
|
900eeb177a | ||
|
|
e67fa3cadc | ||
|
|
c7c32fd4dc | ||
|
|
9471748f4c | ||
|
|
920b1868bb | ||
|
|
eed7c54aaf | ||
|
|
36e58a33ad | ||
|
|
93e8ff749c | ||
|
|
6cb22444ac | ||
|
|
78df101473 | ||
|
|
840508b8ba | ||
|
|
2761254790 | ||
|
|
f47d174f0b | ||
|
|
c50fbc7935 | ||
|
|
75f23a1723 | ||
|
|
f0aa31f9e8 | ||
|
|
aaf59218da | ||
|
|
0203ef66ea | ||
|
|
9c66108dc5 | ||
|
|
558da0a027 | ||
|
|
1d39ec0317 | ||
|
|
281b6468a0 | ||
|
|
9ba6da1f4d | ||
|
|
bde7a7fed6 | ||
|
|
7e021746e4 | ||
|
|
a61bb5f528 | ||
|
|
6a5289b2a2 | ||
|
|
081a5df82c | ||
|
|
3e77ec8008 | ||
|
|
b0b222afa8 | ||
|
|
d85d6f627d | ||
|
|
9505f93da6 | ||
|
|
1671034330 | ||
|
|
3f0ca563e4 | ||
|
|
ffa2a5e8fd | ||
|
|
c3633895d0 | ||
|
|
c15af0a63a | ||
|
|
56e2d77039 | ||
|
|
2cf6fa14a1 | ||
|
|
c374725255 | ||
|
|
babfb6b8d6 | ||
|
|
2ffa8ec7d0 | ||
|
|
b516119224 | ||
|
|
c0e6bb97d8 | ||
|
|
cc30223ff1 | ||
|
|
53c8748942 | ||
|
|
b3a8966e71 | ||
|
|
74800a0767 | ||
|
|
cadd1f993d | ||
|
|
d3807cb86c | ||
|
|
735e9195b5 | ||
|
|
a6619a628e | ||
|
|
456572f87c | ||
|
|
f024b96934 | ||
|
|
25c71bcd98 | ||
|
|
319e7ecc61 | ||
|
|
d9ece5e96a | ||
|
|
4bfe4450b1 | ||
|
|
f9f12ffb61 | ||
|
|
63f81d1033 | ||
|
|
8895d21abf | ||
|
|
2ac1ebeaa0 | ||
|
|
25437602d7 | ||
|
|
8159c71950 | ||
|
|
aa66af7ba6 | ||
|
|
be3633622c | ||
|
|
31655a641c | ||
|
|
b235f6fc6b | ||
|
|
76b297b173 | ||
|
|
1e5cff1d8b | ||
|
|
4e2296d5f6 | ||
|
|
18310632eb | ||
|
|
80ed8d9869 | ||
|
|
2a2658d440 | ||
|
|
710e549c03 | ||
|
|
47fae977dd | ||
|
|
0111919975 | ||
|
|
80837cb040 | ||
|
|
46153a8991 | ||
|
|
a9fccfd4ce | ||
|
|
dfb57e9438 | ||
|
|
eda2731702 | ||
|
|
f47eac5935 | ||
|
|
3bb4dece5a | ||
|
|
0504e231a1 | ||
|
|
6097e14617 | ||
|
|
0b149ce5f1 | ||
|
|
79384e7bf8 | ||
|
|
01eb79cac2 | ||
|
|
128b32c62c | ||
|
|
c1233bd26b | ||
|
|
e23f67def6 | ||
|
|
95f4b29be5 | ||
|
|
5727c708cf | ||
|
|
e49f0f99a1 | ||
|
|
84d340382b | ||
|
|
5bfb067d0f | ||
|
|
d8cb62a92c | ||
|
|
aab8cc40f9 | ||
|
|
9c82689f65 | ||
|
|
ce2755467d | ||
|
|
02b19402ae | ||
|
|
80af66e852 | ||
|
|
15cfa45646 | ||
|
|
ac63849b49 | ||
|
|
02142e352b | ||
|
|
3d92413372 | ||
|
|
5022ed2acc | ||
|
|
31b9d22649 | ||
|
|
7a54894015 | ||
|
|
7100345a09 | ||
|
|
929db15214 | ||
|
|
d9455926c5 | ||
|
|
ba5cd741e1 | ||
|
|
8c94b6243e | ||
|
|
f6473e3383 | ||
|
|
3cd9d6a604 | ||
|
|
142281777e | ||
|
|
8e7b8af180 | ||
|
|
c046cd02d5 | ||
|
|
f5508fcdf1 | ||
|
|
adc2eab6da | ||
|
|
702698d8aa | ||
|
|
73dbe1de37 | ||
|
|
c5706fa28b | ||
|
|
c906a9571e | ||
|
|
038712f431 | ||
|
|
ed2d2c6f86 | ||
|
|
e7158e5a4b | ||
|
|
c66df259a6 | ||
|
|
0b8437fa12 | ||
|
|
a32b6ebf82 | ||
|
|
edbda1696e | ||
|
|
ece218785c | ||
|
|
cc1b3609a1 | ||
|
|
0841aad07a | ||
|
|
30b04ed0c2 | ||
|
|
90b28b3dbd | ||
|
|
0f7ca8f24e | ||
|
|
726d85ebd0 | ||
|
|
c524de9198 | ||
|
|
2fbf03d8ca | ||
|
|
68b8112af0 | ||
|
|
975e83a015 | ||
|
|
1ddb02dbb7 | ||
|
|
1159204ed8 | ||
|
|
a5212e7741 | ||
|
|
1932fd72e3 | ||
|
|
c061fe091b | ||
|
|
3621983a4d | ||
|
|
b6656690ca | ||
|
|
0c9e82ee6b | ||
|
|
b8ebb1faf1 | ||
|
|
690575193f | ||
|
|
9e9cf63c90 | ||
|
|
f5dc213e53 | ||
|
|
138e9c1486 | ||
|
|
7053c0dc93 | ||
|
|
ee8a653b13 | ||
|
|
71fad7e97f | ||
|
|
04740eb928 | ||
|
|
8c63d58c96 | ||
|
|
991672d423 | ||
|
|
55ce38cc87 | ||
|
|
96f3390407 | ||
|
|
aa2426d4e4 | ||
|
|
d0e3e13852 | ||
|
|
a2a1e30d68 | ||
|
|
a6f08b9e0c | ||
|
|
e70842a38c | ||
|
|
2af98d7d42 | ||
|
|
f28eee34e8 | ||
|
|
208610b645 | ||
|
|
e38f8076bc | ||
|
|
6180e1a48b | ||
|
|
e8fe1cd14d | ||
|
|
75d1fa585c | ||
|
|
072ff4b02e | ||
|
|
d8c4e2a72e | ||
|
|
66396de6a3 | ||
|
|
1ca9f575ef | ||
|
|
272e074a15 | ||
|
|
9a738c3783 | ||
|
|
6b4519e959 | ||
|
|
9b1d5213a6 | ||
|
|
f9ed9a4505 | ||
|
|
3ed7ce5630 | ||
|
|
63cd8343c4 | ||
|
|
2d5f8aed24 | ||
|
|
b8d23d0965 | ||
|
|
f4ef92e653 | ||
|
|
c7430f4ae9 | ||
|
|
623fda3087 | ||
|
|
80f27f2b29 | ||
|
|
defbd73b4f | ||
|
|
5285982515 | ||
|
|
f83daa5a22 | ||
|
|
0a07c8f131 | ||
|
|
35481a92f5 | ||
|
|
0995d864e8 | ||
|
|
1a42e43573 | ||
|
|
132aa569fe | ||
|
|
e93a787bd6 | ||
|
|
d6f902653f | ||
|
|
d172d43ddf | ||
|
|
67909724e7 | ||
|
|
1c88cbc5f6 | ||
|
|
3fb4831804 | ||
|
|
b65570fb16 | ||
|
|
871e6e6f64 | ||
|
|
0cd92a96c0 | ||
|
|
ff31ddd2d5 | ||
|
|
e9f5b43785 | ||
|
|
36b41081ab | ||
|
|
1092313892 | ||
|
|
ce605cb8f3 | ||
|
|
0733d885fb | ||
|
|
ef6fe3d1d9 | ||
|
|
82a4fea81a | ||
|
|
14a0252819 | ||
|
|
849aeb65f0 | ||
|
|
a99dc6e93d | ||
|
|
1a5fcc1348 | ||
|
|
c9d88f5628 | ||
|
|
0910711aca | ||
|
|
6a1a9ef6bc | ||
|
|
ef1c059079 | ||
|
|
a05ab37373 | ||
|
|
7ea88cfd22 | ||
|
|
8d14fe6a90 | ||
|
|
49a05b42f9 | ||
|
|
b6da1d7769 | ||
|
|
7e2ad58edd | ||
|
|
7ad0d98404 | ||
|
|
b7052bba95 | ||
|
|
4a5618bcc4 | ||
|
|
c1850fe604 | ||
|
|
fccdf32c9f | ||
|
|
c474411e1b | ||
|
|
9033eac5b1 | ||
|
|
0ba0426591 | ||
|
|
9a9b540701 | ||
|
|
b8ddbb9a26 | ||
|
|
bc3c2b96ed | ||
|
|
622dc87d84 | ||
|
|
18edbdf404 | ||
|
|
02a3eab693 | ||
|
|
dded868a80 | ||
|
|
6fe878bdb9 | ||
|
|
505ece9e36 | ||
|
|
72acfa1e0c | ||
|
|
02cd8b9a44 | ||
|
|
f32250f431 | ||
|
|
1523cad044 | ||
|
|
ed55353700 | ||
|
|
8d3bd45102 | ||
|
|
f69753f6a0 | ||
|
|
b7e2e87ac1 | ||
|
|
3b98508986 | ||
|
|
36ebfc42d9 | ||
|
|
ae5f55ff65 | ||
|
|
01131da5b7 | ||
|
|
66ae865de0 | ||
|
|
8c50f08fec | ||
|
|
e69de47daf | ||
|
|
acef10e848 | ||
|
|
edb9d50d9e | ||
|
|
b989d4f6be | ||
|
|
84dd5c0eca | ||
|
|
db8daf20b7 | ||
|
|
cb985b5a03 | ||
|
|
28c4e38272 | ||
|
|
b29377dc2b | ||
|
|
ca09592a72 | ||
|
|
b0e14fae43 | ||
|
|
dfa68f126b | ||
|
|
4570f68727 | ||
|
|
4705acba55 | ||
|
|
1dea20a358 | ||
|
|
191f661b35 | ||
|
|
f3ab38ba8f | ||
|
|
2d0a892e70 | ||
|
|
00eefe87d7 | ||
|
|
f68fd96555 | ||
|
|
fc50a3242f | ||
|
|
64e9ce2bfa | ||
|
|
e8d04b1921 | ||
|
|
9fedf14e55 | ||
|
|
7d535ef7d1 | ||
|
|
85d610bcdf | ||
|
|
3af8b7bd0c | ||
|
|
6215d85581 | ||
|
|
308d9f5abd | ||
|
|
bcfabba038 | ||
|
|
69139b1987 | ||
|
|
3f8f9d9d73 | ||
|
|
2283579109 | ||
|
|
1e0af94ba2 | ||
|
|
dd631328e0 | ||
|
|
cca2a57c69 | ||
|
|
bbfbcacc6a | ||
|
|
3f95c1a9d5 | ||
|
|
7bd05e9b16 | ||
|
|
10742d7c51 | ||
|
|
bbf7940e45 | ||
|
|
f5af71f4e6 | ||
|
|
1ffcd67c9d | ||
|
|
cf8592ba12 | ||
|
|
1e1d036590 | ||
|
|
64afb6d305 | ||
|
|
20a9e018f7 | ||
|
|
a574c33ac6 | ||
|
|
d21e458fe1 | ||
|
|
bb3ba53c6e | ||
|
|
ba2e4df76b | ||
|
|
33e7e4bee3 | ||
|
|
b83dce2cb3 | ||
|
|
6821676853 | ||
|
|
52024986ee | ||
|
|
2e3342d20d | ||
|
|
419aa1d78f | ||
|
|
de4ddd578f | ||
|
|
5dd68a7ed4 | ||
|
|
881168dc7a | ||
|
|
295e427abe | ||
|
|
67c5d15f51 | ||
|
|
39a7f37f35 | ||
|
|
15d276b0bc | ||
|
|
4387025c48 | ||
|
|
52c20f02c7 | ||
|
|
413dab8094 | ||
|
|
6b1d11a04d | ||
|
|
5441379f38 | ||
|
|
52d954c64e | ||
|
|
415495fc16 | ||
|
|
4223298634 | ||
|
|
e058c71019 | ||
|
|
12b2bbfa26 | ||
|
|
7d22bfffa5 | ||
|
|
4ade80c8a9 | ||
|
|
9674381fc2 | ||
|
|
f6d91280d7 | ||
|
|
e4ebbd8f69 | ||
|
|
7f5560de6d | ||
|
|
0925171e07 | ||
|
|
900d3ad01d | ||
|
|
6a1efedc1f | ||
|
|
0f602e0e62 | ||
|
|
3a425fc47d | ||
|
|
36722cbbd8 | ||
|
|
9572336e96 | ||
|
|
4895cde474 | ||
|
|
21ca1a73cf | ||
|
|
457c1c6ae6 | ||
|
|
c1ff127101 | ||
|
|
5e815a5832 | ||
|
|
4604e9cd54 | ||
|
|
ef580a7680 | ||
|
|
2102776be8 | ||
|
|
f3af7a6b77 | ||
|
|
5abeceac1f | ||
|
|
22f73a0817 | ||
|
|
29b5aef73a | ||
|
|
3e602bce52 | ||
|
|
fe39901885 | ||
|
|
ee9e8b365c | ||
|
|
bd6fa35d4a | ||
|
|
5e1fdf66b0 | ||
|
|
d86e709b90 | ||
|
|
f82cffe882 | ||
|
|
3d88eb92c1 | ||
|
|
0811ebb52d | ||
|
|
491d431ea5 | ||
|
|
ceeebebcad | ||
|
|
04d766aba7 | ||
|
|
3b431e180e | ||
|
|
a2eab1577d | ||
|
|
f8dd3d05dc | ||
|
|
93993e62d6 | ||
|
|
483c88b958 | ||
|
|
4a1e846958 | ||
|
|
927b9afe7c | ||
|
|
9e8827f015 | ||
|
|
ca45f461ae | ||
|
|
7e5af11c2d | ||
|
|
102436274f | ||
|
|
49c26a76b4 | ||
|
|
d469ef7bf2 | ||
|
|
a5374cb876 | ||
|
|
a53e911a0c | ||
|
|
c4a78a149b | ||
|
|
3814912adf | ||
|
|
75ef652cb3 | ||
|
|
1eae3fe5dc | ||
|
|
d0bc383a79 | ||
|
|
ea5d31b530 | ||
|
|
e56ad19e81 | ||
|
|
ed393eb75c | ||
|
|
2e2d309d38 | ||
|
|
6bfb33adb2 | ||
|
|
05138e1db1 | ||
|
|
b65402db41 | ||
|
|
fbef187400 | ||
|
|
25d1b8b9e5 | ||
|
|
ec91594083 | ||
|
|
e67f55bd74 | ||
|
|
276a7f9d2a | ||
|
|
4888d7a567 | ||
|
|
767f78315b | ||
|
|
4311b218fb | ||
|
|
89892042dd | ||
|
|
a22dfb3864 | ||
|
|
f40c42dcf0 | ||
|
|
1d31aa9da3 | ||
|
|
f999073fb4 | ||
|
|
5eb7c64ec3 | ||
|
|
7e79818555 | ||
|
|
a51576b250 | ||
|
|
0bee2689d1 | ||
|
|
c232743963 | ||
|
|
c2d5f092b3 | ||
|
|
8e8e186de9 | ||
|
|
55f47a2b1b | ||
|
|
5c27b9eb90 | ||
|
|
1d018533d1 | ||
|
|
34b8841f5c | ||
|
|
a54166a27c | ||
|
|
4adec39dce | ||
|
|
e63ae08638 | ||
|
|
f1b4c6a1ec | ||
|
|
78b4b3ffa8 | ||
|
|
f51fd48cdb | ||
|
|
9127a816e3 | ||
|
|
1a5cd9b062 | ||
|
|
a86b4c4e58 | ||
|
|
a384960a7b | ||
|
|
5817b80c94 | ||
|
|
97efd63c99 | ||
|
|
5ebcc01bcc | ||
|
|
ddcc008ea6 | ||
|
|
8ec10e2304 | ||
|
|
172f275bc7 | ||
|
|
0dac5943bb | ||
|
|
2dcd2b13be | ||
|
|
8743decd17 | ||
|
|
a15d7ca543 | ||
|
|
a90029ec50 | ||
|
|
563896b126 | ||
|
|
5aecd02668 | ||
|
|
eef6312a17 | ||
|
|
1e48f5c8f4 | ||
|
|
7908fa2814 | ||
|
|
7ffb3ea042 | ||
|
|
842dd90780 | ||
|
|
ee71a03f96 | ||
|
|
5221631b40 | ||
|
|
ade2b4ac0d | ||
|
|
643b95c0e4 | ||
|
|
ed0f9d4fb5 | ||
|
|
6f353c31cb | ||
|
|
404412a950 | ||
|
|
0e1778d8da | ||
|
|
dfc0ca9525 | ||
|
|
53bf1fe4fd | ||
|
|
5c05daa007 | ||
|
|
f74f075b4e | ||
|
|
1b53e8e98c | ||
|
|
4caee284bc | ||
|
|
5368c91399 | ||
|
|
00b1881f20 | ||
|
|
b727bd20c8 | ||
|
|
5a703c116f | ||
|
|
57f728f40e | ||
|
|
74bbde0c34 | ||
|
|
c58fc0781c | ||
|
|
7a266d49c1 | ||
|
|
344eeeb8c0 | ||
|
|
08d50ddfc5 | ||
|
|
f83c892b13 | ||
|
|
7dbec9d395 | ||
|
|
5171c801d9 | ||
|
|
72b4a778a6 | ||
|
|
0254df63f9 | ||
|
|
03c02ff74f | ||
|
|
9277c3c674 | ||
|
|
34b1259dc6 | ||
|
|
257e38de12 | ||
|
|
1984132eae | ||
|
|
8b1d75ea98 | ||
|
|
a79af11371 | ||
|
|
087a9b70b4 | ||
|
|
8b37952128 | ||
|
|
8336930091 | ||
|
|
d875325cee | ||
|
|
f1c5d1cb23 | ||
|
|
b55752e715 | ||
|
|
36a99e6ec8 | ||
|
|
df3c5b9a77 | ||
|
|
4c2ab23c1f | ||
|
|
0a3f81e364 | ||
|
|
69c5631b25 | ||
|
|
cca4331ac4 | ||
|
|
5972dbb6a7 | ||
|
|
794f2d8576 | ||
|
|
e35dce530a | ||
|
|
6034bd3cb2 | ||
|
|
42c77bbb67 | ||
|
|
d504c8f4a3 | ||
|
|
976c8b3ce0 | ||
|
|
905e7e41a6 | ||
|
|
16f48d0400 | ||
|
|
6d7608d851 | ||
|
|
7cd30efbe9 | ||
|
|
8c3652e1ac | ||
|
|
7b9e3aace9 | ||
|
|
568737e123 | ||
|
|
808e3ea6e3 | ||
|
|
e4f14d871f | ||
|
|
1cedd8ee5b | ||
|
|
b80fe24179 | ||
|
|
3cea41a854 | ||
|
|
62b80c13cc | ||
|
|
08b612575c | ||
|
|
b3b804bf2d | ||
|
|
09d8a86e30 | ||
|
|
cc5d917c72 | ||
|
|
7cc3e9d5ef | ||
|
|
87aa9fc6c4 | ||
|
|
8d5c315817 | ||
|
|
f89486f5fe | ||
|
|
cf4d78a24c | ||
|
|
ee88970075 | ||
|
|
f5fa40feb7 | ||
|
|
da13192cfe | ||
|
|
8ff65a65db | ||
|
|
2d882a11a5 | ||
|
|
79adc677a1 | ||
|
|
a8e2dad6d0 | ||
|
|
16954c5626 | ||
|
|
56305c4ede | ||
|
|
30713cc21d | ||
|
|
2bac0a25e0 | ||
|
|
c3aa5b0515 | ||
|
|
30afa43ed4 | ||
|
|
f2e66abe67 | ||
|
|
cdb311a501 | ||
|
|
4c2b3e157c | ||
|
|
52625b710b | ||
|
|
b411bb4e85 | ||
|
|
f16111369f | ||
|
|
628d052749 | ||
|
|
2e26bb0e57 | ||
|
|
652c7d72f6 | ||
|
|
0b046ea8cf | ||
|
|
7133f05fea | ||
|
|
da1c23d7dd | ||
|
|
a329dcd844 | ||
|
|
b57212d5f7 | ||
|
|
12c9e16f37 | ||
|
|
422381ac01 | ||
|
|
4b817cef65 | ||
|
|
4255a9916d | ||
|
|
b1566e904b | ||
|
|
8629076814 | ||
|
|
761e4f2718 | ||
|
|
3441dd4d2d | ||
|
|
9a8139a8b2 | ||
|
|
a0f823910c | ||
|
|
755338ded0 | ||
|
|
ecd85a53a1 | ||
|
|
8faa0d9e01 | ||
|
|
20ca2eac37 | ||
|
|
85b56247a3 | ||
|
|
febc66fb02 | ||
|
|
3f929aee3c | ||
|
|
8c46dd194d | ||
|
|
a9f99d05b9 | ||
|
|
e224c33817 | ||
|
|
4e6afbc8ff | ||
|
|
80b912ea14 | ||
|
|
85a571521b | ||
|
|
98efa9f466 | ||
|
|
a30a9ad763 | ||
|
|
36787eeeeb | ||
|
|
104cafedb0 | ||
|
|
284d307c0f | ||
|
|
f50c98e10d | ||
|
|
a4a5d63da7 | ||
|
|
9f0f9e0928 | ||
|
|
ef62fd7e34 | ||
|
|
0151dd8875 | ||
|
|
2026024c8a | ||
|
|
3d56baddd7 | ||
|
|
c10e7b2b41 | ||
|
|
ba605e985a | ||
|
|
4cd1cdf495 | ||
|
|
f9c644b964 | ||
|
|
da68bcf952 | ||
|
|
2c3b625b6b | ||
|
|
16ef259ae1 | ||
|
|
c56b3e75ea | ||
|
|
772866389f | ||
|
|
41f8b5aee2 | ||
|
|
75052fc773 | ||
|
|
e7a580fd3a | ||
|
|
c967230969 | ||
|
|
f3e6b2ab57 | ||
|
|
7385cb0c89 | ||
|
|
d1be9e7dc8 | ||
|
|
a22052e85b | ||
|
|
ab401541f2 | ||
|
|
dab4427c18 | ||
|
|
ad7d508a3c | ||
|
|
cb30514cbc | ||
|
|
53af9db285 | ||
|
|
b557a14134 | ||
|
|
8922fd6dde | ||
|
|
6b39e549f7 | ||
|
|
eb0dd827b1 | ||
|
|
a48902d7d3 | ||
|
|
7fca1f865d | ||
|
|
c5dc06962d | ||
|
|
45ecdafd52 | ||
|
|
f9e8bf92bd | ||
|
|
de614b293c | ||
|
|
8382075147 | ||
|
|
b605630a9c | ||
|
|
0a89fb1928 | ||
|
|
4c81906b50 | ||
|
|
b6220e985a | ||
|
|
7a97a28e00 | ||
|
|
dda8f281a1 | ||
|
|
28b561a810 | ||
|
|
44aba93d37 | ||
|
|
efb5157027 | ||
|
|
f48c1562f3 | ||
|
|
098b5b28a8 | ||
|
|
998e8e748c | ||
|
|
be96c77467 | ||
|
|
b5e8aec548 | ||
|
|
278c7dc836 | ||
|
|
1425ad87ab | ||
|
|
f985a2a8ad | ||
|
|
20f423a2fc | ||
|
|
210c1215d6 | ||
|
|
1c507393a9 | ||
|
|
b04cfdc091 | ||
|
|
42a3006f6e | ||
|
|
7d5c618246 | ||
|
|
74250982d1 | ||
|
|
8bd6fb5b4e | ||
|
|
bb2fb92b0e | ||
|
|
10b016312d | ||
|
|
b4500a2914 | ||
|
|
da9b1b3f58 | ||
|
|
da31520465 | ||
|
|
f3d3a0e57f | ||
|
|
9a655988dd | ||
|
|
43b541bb7d | ||
|
|
e32853744a | ||
|
|
b017b63745 | ||
|
|
29d6d824fc | ||
|
|
d6983a25b0 | ||
|
|
4cab6726b0 | ||
|
|
509b8f3a29 | ||
|
|
461c2f9055 | ||
|
|
c1d0f9cf1b | ||
|
|
019de08c64 | ||
|
|
30f88f99b7 | ||
|
|
04ee9894c9 | ||
|
|
46368f52f5 | ||
|
|
226e9084c9 | ||
|
|
caf5d2f788 | ||
|
|
591634f175 | ||
|
|
dfaf51a432 | ||
|
|
c2ec066c85 | ||
|
|
3e7c09bb44 | ||
|
|
28f66e30dd | ||
|
|
bac7f097a9 | ||
|
|
fecd29a967 | ||
|
|
c22eed9109 | ||
|
|
2c1ba623a4 | ||
|
|
561cdba7c9 | ||
|
|
e5e25012ff | ||
|
|
c5d653a51a | ||
|
|
c4620a0b0c | ||
|
|
dc969f80f3 | ||
|
|
a82f85f058 | ||
|
|
3ef85e711d | ||
|
|
29f7307f9d | ||
|
|
24eed71959 | ||
|
|
9341ccb643 | ||
|
|
4385b90cca | ||
|
|
9c10ee12fd | ||
|
|
7a622986d6 | ||
|
|
99b3cdd45d | ||
|
|
7b9d84f528 | ||
|
|
5f49827bd2 | ||
|
|
e1b4fe02d6 | ||
|
|
cdab127b98 | ||
|
|
bac7ea5dc0 | ||
|
|
b3b5581f2d | ||
|
|
9a26d869fe | ||
|
|
57138d9629 | ||
|
|
47d897b48d | ||
|
|
6b9f1d519e | ||
|
|
8b966c61d8 | ||
|
|
87e09efff5 | ||
|
|
65a0583c85 | ||
|
|
2000a1b334 | ||
|
|
cdd44af043 | ||
|
|
df99d06d11 | ||
|
|
b32d096be2 | ||
|
|
b9d5d1041c | ||
|
|
994dc9ff33 | ||
|
|
2ac97736ce | ||
|
|
925e106c43 | ||
|
|
59002413f8 | ||
|
|
32293ee3bb | ||
|
|
e04f35af4f | ||
|
|
c50f83400a | ||
|
|
31fa974379 | ||
|
|
aa8c362cb8 | ||
|
|
e48cc40610 | ||
|
|
a931f59dda | ||
|
|
4140fc4d28 | ||
|
|
f0d39e6081 | ||
|
|
c11872a229 | ||
|
|
f4fe730dc9 | ||
|
|
a4c83a08c7 | ||
|
|
c3277d24c3 | ||
|
|
28a23590ef | ||
|
|
162275992b | ||
|
|
08e25f4622 | ||
|
|
9a300d773f | ||
|
|
a5362f3170 | ||
|
|
7b653b90a8 | ||
|
|
f464b3ae7c | ||
|
|
d4b7064d2f | ||
|
|
2540b9969f | ||
|
|
4355b799cb | ||
|
|
a7fb434f9f | ||
|
|
b3465f3bbf | ||
|
|
4dfc3ff94c | ||
|
|
2670279b0d | ||
|
|
f4fd4245be | ||
|
|
200d239e0b | ||
|
|
41a0590182 | ||
|
|
63b2c49354 | ||
|
|
c99d7ce89d | ||
|
|
79e382e064 | ||
|
|
fa1d4dae0f | ||
|
|
35f84b4414 | ||
|
|
a882bdb08c | ||
|
|
b5c2e1c95b | ||
|
|
cfd6ea5a34 | ||
|
|
b3cf5e8698 | ||
|
|
2d07223995 | ||
|
|
32bb9fd4ec | ||
|
|
37b5f22c1a | ||
|
|
e0b00c09ce | ||
|
|
71c4dba34d | ||
|
|
9a3187e36c | ||
|
|
c714a3f91c | ||
|
|
5d4846cf60 | ||
|
|
543b327d09 | ||
|
|
1ccd2a3cc9 | ||
|
|
94398071eb | ||
|
|
143476c76c | ||
|
|
e85683e40f | ||
|
|
141162e5ae | ||
|
|
13acbfb64f | ||
|
|
a55e3d8578 | ||
|
|
2486ae6d3f | ||
|
|
ba991b47ea | ||
|
|
a096d0db37 | ||
|
|
e4a30c052e | ||
|
|
ccfe80ecfa | ||
|
|
df5b889f7d | ||
|
|
55f25c2613 | ||
|
|
34eaa822a0 | ||
|
|
5072f503c4 | ||
|
|
8e6b5a5e55 | ||
|
|
246647641a | ||
|
|
06a1bdbcba | ||
|
|
a0b75d7ada | ||
|
|
d36036a9fe | ||
|
|
a68d891770 | ||
|
|
f272a9ead2 | ||
|
|
8db2e10696 | ||
|
|
bea4f7da98 | ||
|
|
3cd74c06d6 | ||
|
|
053681e4a5 | ||
|
|
e3225f7123 | ||
|
|
ef62b0e746 | ||
|
|
7756d12993 | ||
|
|
3b92d12a44 | ||
|
|
4449bf3c75 | ||
|
|
0200ed1341 | ||
|
|
02bde900d6 | ||
|
|
36e299b4cf | ||
|
|
4e34f861c9 | ||
|
|
42df2fe242 | ||
|
|
7f11da8650 | ||
|
|
dff0fcb1fd | ||
|
|
c31d22f996 | ||
|
|
14ab2b52e9 | ||
|
|
e91f34f09b | ||
|
|
6c733f84e9 | ||
|
|
71c2cd5065 | ||
|
|
45f9a10bfb | ||
|
|
6ad203f767 | ||
|
|
5bf895eefe | ||
|
|
5a8f5b14c1 | ||
|
|
27ccb37646 | ||
|
|
b4c528f6b5 | ||
|
|
517a255d5a | ||
|
|
d76feeb320 | ||
|
|
e4f23b3fce | ||
|
|
02d8dc22e9 | ||
|
|
2fc3ec7224 | ||
|
|
a243ac7a01 | ||
|
|
6f1ee78623 | ||
|
|
6633370a34 | ||
|
|
a22e29532c | ||
|
|
96c8d1f68b | ||
|
|
4aa3bd56c4 | ||
|
|
7d1cf7975a | ||
|
|
59d08a92d0 | ||
|
|
33c1a002da | ||
|
|
8f04d3418a | ||
|
|
f5b556c6cb | ||
|
|
a18f3bf08a | ||
|
|
41a0a0c64c | ||
|
|
b4c8a317d0 | ||
|
|
30ac075555 | ||
|
|
fcb317ca40 | ||
|
|
9991f79326 | ||
|
|
7cf3f8c7cd | ||
|
|
eeef183cc6 | ||
|
|
96bc02c8cb | ||
|
|
af276a24ed | ||
|
|
a92dbae459 | ||
|
|
b0bfb5e443 | ||
|
|
671e6da104 | ||
|
|
485be533c9 | ||
|
|
53ff4cd651 | ||
|
|
a98c760d93 | ||
|
|
9b0c95e37d | ||
|
|
b419b8f642 | ||
|
|
c32f8d0998 | ||
|
|
c6d253cc25 | ||
|
|
c05cb62cdb | ||
|
|
ddd213dda3 | ||
|
|
8aab844a48 | ||
|
|
8fc1cbfc54 | ||
|
|
82ea330dda | ||
|
|
c63aa3033a | ||
|
|
7837738c17 | ||
|
|
bdb0ad4836 | ||
|
|
0dbf810cfe | ||
|
|
58aa4887ad | ||
|
|
3ff0916f49 | ||
|
|
cc9fb3ff41 | ||
|
|
2c51928b9d | ||
|
|
4451978af6 | ||
|
|
d4193c55ae | ||
|
|
ec3e365b3c | ||
|
|
c245c69389 | ||
|
|
9a9a7c9d7a | ||
|
|
20b516b5fa | ||
|
|
4dcfeca092 | ||
|
|
8ca8c73868 | ||
|
|
1a1c101b9a | ||
|
|
a0a0e2d324 | ||
|
|
d544199eff | ||
|
|
407ebeb699 | ||
|
|
9ba12b3e3f | ||
|
|
97976cbee9 | ||
|
|
cb5c772233 | ||
|
|
2bfbb824bf | ||
|
|
05189651ee | ||
|
|
0c80134523 | ||
|
|
6c686520c4 | ||
|
|
49ae029572 | ||
|
|
10e43203ed | ||
|
|
e25be5a522 | ||
|
|
47ff64a9e9 | ||
|
|
35f3887d79 | ||
|
|
dacfc8b00a | ||
|
|
0840ac7d06 | ||
|
|
f34b96f565 | ||
|
|
808fbed4f3 | ||
|
|
b48800049b | ||
|
|
7e5d4166d8 | ||
|
|
6b724ba8db | ||
|
|
25e1427a84 | ||
|
|
82944bc103 | ||
|
|
1e73b2437d | ||
|
|
574cd304e3 | ||
|
|
2edcd1f5b2 | ||
|
|
e144b26619 | ||
|
|
9ef9220689 | ||
|
|
23b6a8c495 | ||
|
|
4b9552fdf5 | ||
|
|
a249097170 | ||
|
|
6f4b2d7967 | ||
|
|
888d77c96c | ||
|
|
e2ae1ca866 | ||
|
|
7ef9a0f0bf | ||
|
|
885a0882f2 | ||
|
|
2ddeca2976 | ||
|
|
2085012874 | ||
|
|
ebb495ab8b | ||
|
|
905274d6d4 | ||
|
|
120d7edf12 | ||
|
|
12cd06cae5 | ||
|
|
42c26102d8 | ||
|
|
e906564f70 | ||
|
|
b8b2dc3b6d | ||
|
|
d69f76e68a | ||
|
|
ec1ba02cca | ||
|
|
49869d071a | ||
|
|
39275682d9 | ||
|
|
1dbc3a049b | ||
|
|
0a7b2683ff | ||
|
|
9714985855 | ||
|
|
d809b8eee0 | ||
|
|
52e2ed2aa7 | ||
|
|
ab3cc3d2b6 | ||
|
|
0a33324321 | ||
|
|
a948335cb9 | ||
|
|
6d2e5af6bc | ||
|
|
9a7c71394e | ||
|
|
1ebf184e9f | ||
|
|
48b013fa85 | ||
|
|
e96dee478d | ||
|
|
25d2a12a92 | ||
|
|
b0af31a71c | ||
|
|
9fdd766429 | ||
|
|
28ce23350d | ||
|
|
1abe890067 | ||
|
|
75fbedeeac | ||
|
|
a1861a2f84 | ||
|
|
0eecfb52ed | ||
|
|
ce1c28ffb9 | ||
|
|
9667b81c8d | ||
|
|
afac596a7b | ||
|
|
636692ba6a | ||
|
|
203f7e7ea6 | ||
|
|
8aee19b0ef | ||
|
|
49ed31c895 | ||
|
|
526075cd5e | ||
|
|
71340805ec | ||
|
|
654585aa68 | ||
|
|
441d3c1220 | ||
|
|
fc2d7258fb | ||
|
|
8195cedcf4 | ||
|
|
3e98477707 | ||
|
|
7b9479eca9 | ||
|
|
f15b67d386 | ||
|
|
f3141727d7 | ||
|
|
c7e4638736 | ||
|
|
d04981969f | ||
|
|
a148d4f446 | ||
|
|
a9e105f9eb | ||
|
|
c8afce2a3a | ||
|
|
ebf9d11f4d | ||
|
|
7a3bdf93e9 | ||
|
|
35d5c585ec | ||
|
|
45eb4f7201 | ||
|
|
e4c2585e23 | ||
|
|
a05f2fe3fb | ||
|
|
94cdd6829c | ||
|
|
34aa51e98a | ||
|
|
6e93c68a77 | ||
|
|
f16eec2ad2 | ||
|
|
657c1a4626 | ||
|
|
6bf6e943cc | ||
|
|
6252f903ee | ||
|
|
a1e827fb3b | ||
|
|
40218e02e1 | ||
|
|
0891b0d294 | ||
|
|
756c43c087 | ||
|
|
6488e66088 | ||
|
|
31ee3fb12b | ||
|
|
b80d45ad45 | ||
|
|
ea2a3c3b70 | ||
|
|
c0312159eb | ||
|
|
e124f582aa | ||
|
|
54150a8f2a | ||
|
|
385c221ef8 | ||
|
|
7df6e99e28 | ||
|
|
76a5f2f516 | ||
|
|
936c1f2ada | ||
|
|
5787802077 | ||
|
|
c902d25524 | ||
|
|
d152336f42 | ||
|
|
7ab0ae18be | ||
|
|
6189eddecf | ||
|
|
cc7471a9af | ||
|
|
eabaca361c | ||
|
|
141ca2498c | ||
|
|
964db2a779 | ||
|
|
f2867542b1 | ||
|
|
8f6e886821 | ||
|
|
dfca7bc2c4 | ||
|
|
f94b04bb19 | ||
|
|
be1e97c616 | ||
|
|
9790e9f843 | ||
|
|
e339d3857f | ||
|
|
0e4f21842c | ||
|
|
d4ea9a1ba3 | ||
|
|
5300fa929b | ||
|
|
8a5a66a06a | ||
|
|
43ed4427b4 | ||
|
|
22b0d92e6b | ||
|
|
7eb528232e | ||
|
|
33b6b85e72 | ||
|
|
8177119671 | ||
|
|
4b407a9dd2 | ||
|
|
9b904a7b4a | ||
|
|
e0424a1e38 | ||
|
|
00c08e05f5 | ||
|
|
0836d9c5ee | ||
|
|
ff5b6c4eab | ||
|
|
bb5cd49327 | ||
|
|
6f40ef6015 | ||
|
|
c7758506ac | ||
|
|
192ac6959a | ||
|
|
465a4643a8 | ||
|
|
1bf0065d9d | ||
|
|
c14efc843c | ||
|
|
0506dfba10 | ||
|
|
7e4f36e8b4 | ||
|
|
55dc482a61 | ||
|
|
05b9f9cae6 | ||
|
|
c79868de3e | ||
|
|
7eeac060ab | ||
|
|
c8b2b7b279 | ||
|
|
f4b3187494 |
No files matched your search
@@ -0,0 +1,44 @@
|
||||
version: 1.0.{build}
|
||||
|
||||
environment:
|
||||
GH_TOKEN:
|
||||
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
|
||||
key_pass:
|
||||
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
|
||||
pfx_key:
|
||||
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
|
||||
AWS_ACCESS_KEY_ID:
|
||||
secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
|
||||
|
||||
#notifications:
|
||||
# - provider: Slack
|
||||
# incoming_webhook:
|
||||
# secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
|
||||
|
||||
|
||||
clone_folder: c:\projects\lbry
|
||||
|
||||
test_script:
|
||||
- cd C:\projects\lbry\
|
||||
- pip install cython
|
||||
- pip install mock pylint unqlite
|
||||
- pip install .
|
||||
- pylint lbrynet
|
||||
# disable tests for now so that appveyor can build the app
|
||||
- set PYTHONPATH=.
|
||||
- trial tests
|
||||
# TODO: integration tests do not work
|
||||
#- python -m unittest discover tests/integration
|
||||
#- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
|
||||
|
||||
build_script:
|
||||
- cd C:\projects\lbry\build\
|
||||
- ps: .\build.ps1
|
||||
|
||||
|
||||
artifacts:
|
||||
- path: build\dist\*.zip
|
||||
name: lbrynet-daemon
|
||||
@@ -0,0 +1,20 @@
|
||||
[bumpversion]
|
||||
current_version = 0.14.1
|
||||
commit = True
|
||||
tag = True
|
||||
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+)(?P<candidate>\d+))?
|
||||
serialize =
|
||||
{major}.{minor}.{patch}{release}{candidate}
|
||||
{major}.{minor}.{patch}
|
||||
|
||||
[bumpversion:part:candidate]
|
||||
first_value = 1
|
||||
|
||||
[bumpversion:part:release]
|
||||
optional_value = production
|
||||
values =
|
||||
rc
|
||||
production
|
||||
|
||||
[bumpversion:file:lbrynet/__init__.py]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
/CHANGELOG.md merge=union
|
||||
@@ -0,0 +1,48 @@
|
||||
<!--
|
||||
Thanks for reporting an issue to LBRY and helping us improve!
|
||||
|
||||
To make it possible for us to help you, please fill out below information carefully.
|
||||
|
||||
Before reporting any issues, please make sure that you're using the latest version.
|
||||
- App releases: https://github.com/lbryio/lbry-app/releases
|
||||
- Standalone daemon: https://github.com/lbryio/lbry/releases
|
||||
|
||||
We are also available on Slack at https://slack.lbry.io
|
||||
-->
|
||||
|
||||
|
||||
## The Issue
|
||||
|
||||
### Steps to reproduce
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
### Expected behaviour
|
||||
Tell us what should happen
|
||||
|
||||
### Actual behaviour
|
||||
Tell us what happens instead
|
||||
|
||||
|
||||
## System Configuration
|
||||
|
||||
<!-- For the app, this info is in the About section at the bottom of the Help page.
|
||||
You can include a screenshot instead of typing it out -->
|
||||
|
||||
<!-- For the daemon, run:
|
||||
curl 'http://localhost:5279/lbryapi' --data '{"method":"version"}'
|
||||
and include the full output -->
|
||||
|
||||
- LBRY Daemon version:
|
||||
- LBRY App version:
|
||||
- LBRY Installation ID:
|
||||
- Operating system:
|
||||
|
||||
|
||||
## Anything Else
|
||||
<!-- Include anything else that does not fit into the above sections -->
|
||||
|
||||
|
||||
## Screenshots
|
||||
<!-- If a screenshot would help explain the bug, please include one or two here -->
|
||||
@@ -1 +1,25 @@
|
||||
*.pyc
|
||||
*.egg
|
||||
*.so
|
||||
*.xml
|
||||
*.iml
|
||||
*.log
|
||||
*.pem
|
||||
*.decTest
|
||||
*.prof
|
||||
.#*
|
||||
|
||||
/build/build
|
||||
/build/dist
|
||||
/bulid/requirements_base.txt
|
||||
|
||||
/lbrynet.egg-info
|
||||
/docs_build
|
||||
/lbry-venv
|
||||
|
||||
.idea/
|
||||
.coverage
|
||||
.DS_Store
|
||||
|
||||
# temporary files from the twisted.trial test runner
|
||||
_trial_temp/
|
||||
@@ -0,0 +1,439 @@
|
||||
[MASTER]
|
||||
|
||||
# Specify a configuration file.
|
||||
#rcfile=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
# `\.#.*` - add emacs tmp files to the blacklist
|
||||
ignore-patterns=\.#.*
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=yes
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=1
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
# be used to obtain the result of joining multiple strings with the addition
|
||||
# operator. Joining a lot of strings can lead to a maximum recursion error in
|
||||
# Pylint and this flag can prevent that. It has one side effect, the resulting
|
||||
# AST will be different than the one from reality.
|
||||
optimize-ast=no
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||
confidence=
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
# --disable=W"
|
||||
disable=
|
||||
anomalous-backslash-in-string,
|
||||
arguments-differ,
|
||||
attribute-defined-outside-init,
|
||||
bad-continuation,
|
||||
bare-except,
|
||||
broad-except,
|
||||
cell-var-from-loop,
|
||||
consider-iterating-dictionary,
|
||||
cyclic-import,
|
||||
dangerous-default-value,
|
||||
duplicate-code,
|
||||
exec-used,
|
||||
fixme,
|
||||
global-statement,
|
||||
inherit-non-class,
|
||||
invalid-name,
|
||||
len-as-condition,
|
||||
locally-disabled,
|
||||
logging-not-lazy,
|
||||
missing-docstring,
|
||||
multiple-imports,
|
||||
no-else-return,
|
||||
no-init,
|
||||
no-member,
|
||||
no-self-use,
|
||||
not-context-manager,
|
||||
protected-access,
|
||||
redefined-builtin,
|
||||
redefined-outer-name,
|
||||
redefined-variable-type,
|
||||
relative-import,
|
||||
signature-differs,
|
||||
super-init-not-called,
|
||||
too-few-public-methods,
|
||||
too-many-arguments,
|
||||
too-many-branches,
|
||||
too-many-instance-attributes,
|
||||
too-many-lines,
|
||||
too-many-locals,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
too-many-statements,
|
||||
trailing-newlines,
|
||||
undefined-loop-variable,
|
||||
ungrouped-imports,
|
||||
unnecessary-lambda,
|
||||
unused-argument,
|
||||
unused-variable,
|
||||
wildcard-import,
|
||||
wrong-import-order,
|
||||
wrong-import-position
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html. You can also give a reporter class, eg
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
output-format=text
|
||||
|
||||
# Put messages in a separate file for each module / package specified on the
|
||||
# command line instead of printing them on stdout. Reports (if any) will be
|
||||
# written in a file name "pylint_global.[txt|html]".
|
||||
files-output=no
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=no
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details
|
||||
#msg-template=
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||
# not used).
|
||||
dummy-variables-rgx=_$|dummy
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,_cb
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format
|
||||
logging-modules=logging
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# List of builtins function names that should not be used, separated by a comma
|
||||
bad-functions=map,filter,input
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
# allow `d` as its used frequently for deferred callback chains
|
||||
good-names=i,j,k,ex,Run,_,d
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=foo,bar,baz,toto,tutu,tata
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name
|
||||
include-naming-hint=no
|
||||
|
||||
# Regular expression matching correct function names
|
||||
function-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for function names
|
||||
function-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct variable names
|
||||
variable-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for variable names
|
||||
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct constant names
|
||||
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Naming hint for constant names
|
||||
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Regular expression matching correct attribute names
|
||||
attr-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for attribute names
|
||||
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct argument names
|
||||
argument-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for argument names
|
||||
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct class attribute names
|
||||
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Naming hint for class attribute names
|
||||
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Regular expression matching correct inline iteration names
|
||||
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Naming hint for inline iteration names
|
||||
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class names
|
||||
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Naming hint for class names
|
||||
class-name-hint=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Regular expression matching correct module names
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Naming hint for module names
|
||||
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Regular expression matching correct method names
|
||||
method-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for method names
|
||||
method-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=^_
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=-1
|
||||
|
||||
|
||||
[ELIF]
|
||||
|
||||
# Maximum number of nested blocks for function / method body
|
||||
max-nested-blocks=5
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||
# install python-enchant package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to indicated private dictionary in
|
||||
# --spelling-private-dict-file option instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=100
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=no
|
||||
|
||||
# List of optional constructs for which whitespace checking is disabled. `dict-
|
||||
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
|
||||
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
|
||||
# `empty-line` allows space-only lines.
|
||||
no-space-check=trailing-comma,dict-separator
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=1000
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,XXX,TODO
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Ignore imports when computing similarities.
|
||||
ignore-imports=no
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||
ignore-mixin-members=yes
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb,distutils
|
||||
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet.reactor,RequestMessage
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,TERMIOS,Bastion,rexec
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
ignored-argument-names=_.*
|
||||
|
||||
# Maximum number of locals for function / method body
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of return / yield for function / method body
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of branch for function / method body
|
||||
max-branches=12
|
||||
|
||||
# Maximum number of statements in function / method body
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
# Maximum number of boolean expressions in a if statement
|
||||
max-bool-expr=5
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,__new__,setUp
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,_fields,_replace,_source,_make
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when being caught. Defaults to
|
||||
# "Exception"
|
||||
overgeneral-exceptions=Exception
|
||||
@@ -0,0 +1,45 @@
|
||||
os: linux
|
||||
dist: trusty
|
||||
language: python
|
||||
python: 2.7
|
||||
|
||||
addons:
|
||||
srcclr:
|
||||
debug: true
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- libgmp3-dev
|
||||
- build-essential
|
||||
- git
|
||||
- libssl-dev
|
||||
- libffi-dev
|
||||
|
||||
before_install:
|
||||
- virtualenv venv
|
||||
- source venv/bin/activate
|
||||
|
||||
install:
|
||||
- pip install -U pip
|
||||
- pip install -r requirements.txt
|
||||
- pip install .
|
||||
|
||||
script:
|
||||
- pip install cython
|
||||
- pip install mock pylint unqlite
|
||||
- pylint lbrynet
|
||||
- PYTHONPATH=. trial tests
|
||||
- python -m unittest discover tests/integration
|
||||
- rvm install ruby-2.3.1
|
||||
- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
@@ -0,0 +1,483 @@
|
||||
# Change Log
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/) with
|
||||
regard to the json-rpc api. As we're currently pre-1.0 release, we
|
||||
can and probably will change functionality and break backwards compatability
|
||||
at anytime.
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
* Add link to instructions on how to change the default peer port
|
||||
*
|
||||
|
||||
### Changed
|
||||
*
|
||||
*
|
||||
|
||||
### Fixed
|
||||
* peer_port is settable using `settings_set`
|
||||
* Fix for https://github.com/lbryio/lbry/issues/750
|
||||
*
|
||||
|
||||
### Deprecated
|
||||
*
|
||||
*
|
||||
|
||||
### Removed
|
||||
*
|
||||
*
|
||||
|
||||
## [0.14.1] - 2017-07-07
|
||||
|
||||
### Fixed
|
||||
* Fixed timeout behaviour when calling API command get
|
||||
* Fixed https://github.com/lbryio/lbry/issues/765
|
||||
|
||||
|
||||
|
||||
## [0.14.0] - 2017-07-05
|
||||
|
||||
### Added
|
||||
* Missing docstring for `blob_list`
|
||||
* Added convenient import for setting up a daemon client, `from lbrynet.daemon import get_client`
|
||||
* Added unit tests for CryptBlob.py
|
||||
|
||||
|
||||
### Changed
|
||||
* Change `max_key_fee` setting to be a dictionary with values for `currency` and `amount`
|
||||
* Renamed `lbrynet.lbryfile` to `lbrynet.lbry_file`
|
||||
* Renamed `lbrynet.lbryfilemanager` to `lbrynet.file_manager`
|
||||
* Renamed `lbrynet.lbrynet_daemon` to `lbrynet.daemon`
|
||||
* Initialize lbrynet settings when configuring an api client if they are not set yet
|
||||
* Updated lbryum imports
|
||||
* Improve error message when resolving a claim fails using the "get" command
|
||||
|
||||
|
||||
### Removed
|
||||
* Removed unused settings from conf.py and `settings_set`
|
||||
* Removed download_directory argument from API command get
|
||||
|
||||
|
||||
### Fixed
|
||||
* Fixed some log messages throwing exceptions
|
||||
* Fix shutdown of the blob tracker by Session
|
||||
* Fixed claim_new_support docstrings
|
||||
* Fixed BlobManager causing functional tests to fail, removed its unneeded manage() loop
|
||||
* Increased max_key_fee
|
||||
* Fixed unit tests on appveyor Windows build
|
||||
* Fixed [#692](https://github.com/lbryio/lbry/issues/692)
|
||||
|
||||
|
||||
|
||||
## [0.13.1] - 2017-06-15
|
||||
|
||||
### Added
|
||||
* Add `claim_send_to_address`
|
||||
* Add `change_address` argument to `publish`
|
||||
* Add `unique_contacts` count to `status` response
|
||||
|
||||
|
||||
### Changed
|
||||
* Support resolution of multiple uris with `resolve`, all results are keyed by uri
|
||||
* Add `error` responses for failed resolves
|
||||
* Add `claim_list_by_channel`, supports multiple channel resolution
|
||||
* Rename delete_target_file argument of delete API command to delete_from_download_dir
|
||||
* Rename delete_all CLI flag -a to --delete_all
|
||||
|
||||
|
||||
### Removed
|
||||
* Remove `claims_in_channel` from `resolve` response
|
||||
|
||||
|
||||
### Fixed
|
||||
* Race condition from improper initialization and shutdown of the blob manager database
|
||||
* Various fixes for GetStream class used in API command get
|
||||
* Download analytics error
|
||||
* Fixed flag options in file_delete API command
|
||||
|
||||
|
||||
|
||||
## [0.11.0] - 2017-06-09
|
||||
|
||||
### Added
|
||||
* Added claim_address option to publish API command
|
||||
* Added message for InsufficientFundsError exception
|
||||
* Add CLI docs
|
||||
|
||||
|
||||
### Changed
|
||||
* Do not catch base exception in API command resolve
|
||||
* Remove deprecated `lbrynet.metadata` and update what used it to instead use `lbryschema`
|
||||
* Get version information locally instead of via api for cli
|
||||
|
||||
|
||||
### Deprecated
|
||||
* Old fee metadata format in publish API command is deprecated, throw relevant exception
|
||||
* Removed deprecated `get_best_blockhash`
|
||||
* Removed deprecated `is_running`
|
||||
* Removed deprecated `daemon_status`
|
||||
* Removed deprecated `is_first_run`
|
||||
* Removed deprecated `get_lbry_session_info`
|
||||
* Removed deprecated `get_time_behind_blockchain`
|
||||
* Removed deprecated `get_settings`
|
||||
* Removed deprecated `set_settings`
|
||||
* Removed deprecated `get_balance`
|
||||
* Removed deprecated `stop`
|
||||
* Removed deprecated `get_claim_info`
|
||||
* Removed deprecated `stop_lbry_file`
|
||||
* Removed deprecated `start_lbry_file`
|
||||
* Removed deprecated `get_est_cost`
|
||||
* Removed deprecated `abandon_claim`
|
||||
* Removed deprecated `support_claim`
|
||||
* Removed deprecated `get_my_claim`
|
||||
* Removed deprecated `get_name_claims`
|
||||
* Removed deprecated `get_claims_for_tx`
|
||||
* Removed deprecated `get_transaction_history`
|
||||
* Removed deprecated `get_transaction`
|
||||
* Removed deprecated `address_is_mine`
|
||||
* Removed deprecated `get_public_key_from_wallet`
|
||||
* Removed deprecated `get_new_address`
|
||||
* Removed deprecated `get_block`
|
||||
* Removed deprecated `descriptor_get`
|
||||
* Removed deprecated `download_descriptor`
|
||||
* Removed deprecated `get_peers_for_hash`
|
||||
* Removed deprecated `announce_all_blobs_to_dht`
|
||||
* Removed deprecated `get_blob_hashes`
|
||||
* Removed deprecated `reflect_all_blobs`
|
||||
* Removed deprecated `get_start_notice`
|
||||
|
||||
|
||||
### Fixed
|
||||
* Download analytics error
|
||||
|
||||
|
||||
|
||||
## [0.10.3production4] - 2017-06-01
|
||||
|
||||
### Added
|
||||
* Prevent publish of files with size 0
|
||||
* Add `dht_status` parameter to `status` to include bandwidth and peer info
|
||||
* Positional and flag arguments in lbrynet-cli
|
||||
|
||||
|
||||
### Changed
|
||||
* Changed keyword arguments in lbrynet-cli to use a -- prefix
|
||||
* Using the help function in lbrynet-cli no longer requires lbrynet-daemon to be running
|
||||
|
||||
|
||||
|
||||
## [0.10.3production3] - 2017-05-30
|
||||
|
||||
### Changed
|
||||
* Update `publish` to use {'currency': ..., 'address': ..., 'amount'} format for fee parameter, previously used old {currency: {'address': ..., 'amount': ...}} format
|
||||
|
||||
|
||||
### Fixed
|
||||
* Allow claim_show to be used without specifying name
|
||||
* Fix licenseUrl field in `publish`
|
||||
|
||||
|
||||
|
||||
## [0.10.3production2] - 2017-05-30
|
||||
|
||||
### Fixed
|
||||
* Allow claim_show to be used without specifying name
|
||||
|
||||
|
||||
|
||||
## [0.10.3] - 2017-05-23
|
||||
|
||||
### Added
|
||||
* Add decorator to support queueing api calls
|
||||
* Added force option to API command resolve
|
||||
|
||||
|
||||
### Changed
|
||||
* Added optional `address` and `include_unconfirmed` params to `jsonrpc_wallet_balance` method
|
||||
* Wait for subscriptions before announcing wallet has finished starting
|
||||
* Cache claims in wallet storage for use looking claims up by id or outpoint
|
||||
* Try to use cached claim info for `file_list`
|
||||
* Convert wallet storage to inlinecallbacks
|
||||
* Improve internal name_metadata sqlite table
|
||||
|
||||
|
||||
### Fixed
|
||||
* Fix race condition in publish that resulted in claims being rejected when making many publishes concurrently
|
||||
|
||||
|
||||
|
||||
## [0.10.1] - 2017-05-03
|
||||
|
||||
### Fixed
|
||||
* Fix multiple reactor.stop() calls
|
||||
* Properly shut down lbryum wallet from lbrynet
|
||||
* Set LBRYumWallet.config upon initialization, fixes attribute error
|
||||
|
||||
|
||||
|
||||
## [0.10.0] - 2017-04-25
|
||||
|
||||
### Added
|
||||
* Add `lbryschema_version` to response from `version`
|
||||
* Added call to `get_address_balance` when `address` conditional returns true
|
||||
* Added `address` conditional to `jsonrpc_wallet_balance`
|
||||
* Added `get_address_balance` method to the `Wallet` class
|
||||
### Changed
|
||||
* Added optional `address` and `include_unconfirmed` params to `jsonrpc_wallet_balance` method
|
||||
* Wait for subscriptions before announcing wallet has finished starting
|
||||
### Fixed
|
||||
* fix stream_cost_estimate throwing exception on non decodeable claims
|
||||
* fixed signing of Windows binaries
|
||||
* fixed a few pylint warnings
|
||||
|
||||
|
||||
## [0.10.0rc2] - 2017-04-17
|
||||
### Changed
|
||||
* Return full `lbry_id` and `installation_id` from `status`
|
||||
|
||||
|
||||
## [0.10.0rc1] - 2017-04-13
|
||||
### Fixed
|
||||
* Fix uncaught exception in `stream_cost_estimate`
|
||||
|
||||
|
||||
## [0.9.2rc22] - 2017-04-12
|
||||
### Added
|
||||
* Add `claim_id` parameter to `claim_show`
|
||||
* Add `hex` field to claim responses for the raw claim value
|
||||
* Add an `error` field to to file responses if an error occurs
|
||||
### Changed
|
||||
* Use `uri` instead of `name` in `get_availability`
|
||||
* Add `channel_name` to claim and file responses where applicable
|
||||
* Return None (instead of errors) if a uri cannot be resolved
|
||||
* Use `uri` instead of `name` for `stream_cost_estimate`, update cost estimate for lbryschema
|
||||
### Fixed
|
||||
* `file_list` for files with bad signatures
|
||||
* return None from resolve commands when nothing is found
|
||||
* return lbry files with claims that are abandoned
|
||||
* unhelpful error messages in `publish` and `channel_new`
|
||||
|
||||
## [0.9.2rc9] - 2017-04-08
|
||||
### Added
|
||||
* Use `claim_id` instead of outpoint for `claim_abandon`
|
||||
* Add `channel_name` parameter to `publish`
|
||||
* Add `delete_all` parameter to `file_delete` to allow deleting multiple files
|
||||
* Add `channel_list_mine`
|
||||
* Add `channel_new`
|
||||
* Add `resolve` to resolve lbry uris
|
||||
### Changed
|
||||
* Use `uri` instead of `name` for `get`, remove explicit `claim_id` parameter
|
||||
* Increase default download timeout
|
||||
* Use lbry.io api for exchange rate data
|
||||
|
||||
## [0.9.2rc4] - 2017-04-06
|
||||
### Changed
|
||||
* Use lbryschema library for metadata
|
||||
### Fixed
|
||||
* Removed update_metadata function that could cause update problems
|
||||
* Fix DHT contact bug
|
||||
|
||||
## [0.9.2rc3] - 2017-03-29
|
||||
### Added
|
||||
* Create wallet_unused_address API command
|
||||
* Add `claim_id` parameter to `get`, used to specify non-default claim for `name`
|
||||
### Changed
|
||||
* wallet_new_address API command always returns new address
|
||||
* Improved ConnectionManager speed
|
||||
* Remove unused `stream_info` parameter in `get`
|
||||
|
||||
## [0.9.2rc2] - 2017-03-25
|
||||
### Added
|
||||
* Add `wallet_list` command
|
||||
* Add checks for missing/extraneous params when calling jsonrpc commands
|
||||
* Added colors to cli error messages
|
||||
### Changed
|
||||
* Removed check_pending logic from Daemon
|
||||
* Switched to txrequests so requests can use twisted event loop
|
||||
* Renamed API command file_seed to file_set_status
|
||||
### Fixed
|
||||
* Fix restart procedure in DaemonControl
|
||||
* Create download directory if it doesn't exist
|
||||
* Fixed descriptor_get
|
||||
* Fixed jsonrpc_reflect()
|
||||
* Fixed api help return
|
||||
* Fixed API command descriptor_get
|
||||
* Fixed API command transaction_show
|
||||
* Fixed error handling for jsonrpc commands
|
||||
|
||||
## [0.9.2rc1] - 2017-03-21
|
||||
### Added
|
||||
* Add `wallet_list` command
|
||||
### Changed
|
||||
* Dont add expected payment to wallet when payment rate is 0
|
||||
### Fixed
|
||||
* Fixed descriptor_get
|
||||
* Fixed jsonrpc_reflect()
|
||||
* Fixed api help return
|
||||
* Fixed API command descriptor_get
|
||||
* Fixed API command transaction_show
|
||||
* Handle failure to decode claim cache file
|
||||
|
||||
## [0.9.1] - 2017-03-17
|
||||
### Fixed
|
||||
* Fix wallet_public_key API command
|
||||
|
||||
## [0.9.1rc5] - 2017-03-16
|
||||
### Added
|
||||
* publish API command can take metadata fields as arguments
|
||||
* Added `reflect_uploads` config to disable reflecting on upload
|
||||
### Fixed
|
||||
* Fixed jsonrpc_reflect()
|
||||
* Fixed api help return
|
||||
|
||||
## [0.9.1rc2] - 2017-03-15
|
||||
### Added
|
||||
* Added `--version` flag
|
||||
### Changed
|
||||
* Removed `simplejson` dependency in favor of bulitin `json`
|
||||
|
||||
## [0.9.0rc17] - 2017-03-10
|
||||
### Fixed
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
* Made dictionary key names in API commmand outputs to be more consistent
|
||||
|
||||
## [0.9.0rc15] - 2017-03-09
|
||||
### Added
|
||||
* Add file filters: `claim_id`, `outpoint`, and `rowid`
|
||||
* Make loggly logs less verbose
|
||||
### Changed
|
||||
* Change file filter `uri` to `name` and return field `lbry_uri` to `name`
|
||||
* Refactor file_list, add `full_status` argument to populate resource intensive fields
|
||||
* Remove deprecated file commands: `get_lbry_files`, `get_lbry_file`, and `file_get`
|
||||
* Remove deprecated `delete_lbry_file` command
|
||||
* Return standard file json from `get`
|
||||
### Fixed
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
|
||||
## [0.9.0rc12] - 2017-03-06
|
||||
### Fixed
|
||||
* Fixed ExchangeRateManager freezing the app
|
||||
* Fixed download not timing out properly when downloading sd blob
|
||||
* Fixed ExchangeRateManager freezing the app
|
||||
* Fixed download not timing out properly when downloading sd blob
|
||||
* Fixed get not reassembling an already downloaded file that was deleted from download directory
|
||||
|
||||
## [0.9.0rc11] - 2017-02-27
|
||||
### Fixed
|
||||
* Added timeout to ClientProtocol
|
||||
* Add check for when local height of wallet is less than zero
|
||||
|
||||
## [0.9.0rc9] - 2017-02-22
|
||||
### Changed
|
||||
* Add blockchain status to jsonrpc_status
|
||||
|
||||
## [0.8.7] - 2017-02-21
|
||||
|
||||
## [0.8.6] - 2017-02-19
|
||||
|
||||
## [0.8.6rc0] - 2017-02-19
|
||||
### Changed
|
||||
* Add `file_get` by stream hash
|
||||
* Add utils.call_later to replace reactor.callLater
|
||||
|
||||
### Fixed
|
||||
* Fix unhandled error in `get`
|
||||
* Fix sd blob timeout handling in `get_availability`, return 0.0
|
||||
|
||||
## [0.8.5] - 2017-02-18
|
||||
|
||||
## [0.8.5rc0] - 2017-02-18
|
||||
### Fixed
|
||||
* Fix result expected by ui from file_get for missing files
|
||||
|
||||
## [0.8.4] - 2017-02-17
|
||||
|
||||
## [0.8.4rc0] - 2017-02-17
|
||||
### Changed
|
||||
* Remove unused upload_allowed option
|
||||
* Remove code related to packaging as that step is now done in the electron client
|
||||
* Remove lbryum version check; use lbry-electron as version source
|
||||
* Include download url in version check
|
||||
|
||||
### Fixed
|
||||
* add misssing traceback to logging
|
||||
|
||||
## [0.8.3] - 2017-02-15
|
||||
### Fixed
|
||||
* Get lbry files with pending claims
|
||||
* Add better logging to help track down [#478](https://github.com/lbryio/lbry/issues/478)
|
||||
* Catch UnknownNameErrors when resolving a name. [#479](https://github.com/lbryio/lbry/issues/479)
|
||||
|
||||
### Changed
|
||||
* Add blob_get, descriptor_get, and blob_delete
|
||||
* Add filter keyword args to blob_list
|
||||
* Refactor get_availability
|
||||
* Add optional peer search timeout, add peer_search_timeout setting
|
||||
|
||||
## [0.8.3rc3] - 2017-02-14
|
||||
|
||||
## [0.8.3rc2] - 2017-02-13
|
||||
|
||||
## [0.8.3rc1] - 2017-02-13
|
||||
### Changed
|
||||
* make connection manager unit testeable
|
||||
|
||||
### Fixed
|
||||
* Change EWOULDBLOCK error in DHT to warning. #481
|
||||
* mark peers as down if it fails download protocol
|
||||
* Made hash reannounce time to be adjustable to fix [#432](https://github.com/lbryio/lbry/issues/432)
|
||||
|
||||
|
||||
## [0.8.3rc0] - 2017-02-10
|
||||
### Changed
|
||||
* Convert EncryptedFileDownloader to inlineCallbacks
|
||||
* Convert EncryptedFileManager to use inlineCallbacks
|
||||
* Convert Daemon._delete_lbry_file to inlineCallbacks
|
||||
* Add uri to stream reflector to de-obfuscate reflector logs
|
||||
* Simplify lbrynet.lbrynet_daemon.Publisher
|
||||
* Reflect streams in file manager looping call rather than in each file
|
||||
* Convert GetStream to inclineCallbacks
|
||||
* Change callback condition in GetStream to the first data blob completing
|
||||
* Add local and remote heights to blockchain status
|
||||
|
||||
### Fixed
|
||||
* Fix recursion depth error upon failed blob
|
||||
* Call stopProducing in reflector client file_sender when uploading is done
|
||||
* Ensure streams in stream_info_manager are saved in lbry_file_manager
|
||||
* Fixed file_delete not deleting data from stream_info_manager [#470](https://github.com/lbryio/lbry/issues/470)
|
||||
* Fixed upload of bug reports to Slack ([#472](https://github.com/lbryio/lbry/issues/472))
|
||||
* Fixed claim updates [#473](https://github.com/lbryio/lbry/issues/473)
|
||||
* Handle ConnectionLost error in reflector client
|
||||
* Fix updating a claim where the stream doesn't change
|
||||
* Fix claim_abandon
|
||||
|
||||
## [0.8.1] - 2017-02-01
|
||||
### Changed
|
||||
* reflect all the blobs in a stream
|
||||
* change command line flags so that the more common usage is the default
|
||||
* change daemon function signatures to include names arguments
|
||||
|
||||
### Fixed
|
||||
* disable verbose twisted logs
|
||||
* improved wallet balance calculations
|
||||
* fix block too deep error
|
||||
|
||||
## [0.8.0] - 2017-01-24
|
||||
### Changed
|
||||
* renamed api endpoints
|
||||
* improved command line user experience
|
||||
* integrate twisted logging with python logging
|
||||
* Updated READMEs
|
||||
|
||||
### Fixed
|
||||
* Fixed bug where ConnectionManager wasn't being stopped
|
||||
* Fixed: #343
|
||||
* Stop hanging if github is down
|
||||
* paths for debian package have been updated to be correct
|
||||
* improved output of the publish command
|
||||
@@ -0,0 +1,6 @@
|
||||
# Add a CHANGELOG entry for app changes
|
||||
has_app_changes = !git.modified_files.grep(/lbrynet/).empty?
|
||||
if !git.modified_files.include?("CHANGELOG.md") && has_app_changes
|
||||
fail("Please include a CHANGELOG entry.")
|
||||
message "See http://keepachangelog.com/en/0.3.0/ for details on good changelog guidelines"
|
||||
end
|
||||
@@ -0,0 +1,86 @@
|
||||
#### Getting LBRY for development
|
||||
|
||||
Q: How do I get lbry for command line?
|
||||
|
||||
A: In order to run lbry from command line, you need more than the packaged app/deb.
|
||||
|
||||
###### On OS X
|
||||
|
||||
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app.
|
||||
|
||||
###### On Linux
|
||||
|
||||
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
sudo python setup.py install
|
||||
|
||||
#### Using LBRY
|
||||
|
||||
Q: How do I run lbry from command line?
|
||||
|
||||
A: The command is `lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I stop lbry from the command line?
|
||||
|
||||
A: You can ctrl-c or run `stop-lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I run lbry with lbrycrdd (the blockchain node application)?
|
||||
|
||||
A: Start lbry with the --wallet flag set: `lbrynet-daemon --wallet=lbrycrd`
|
||||
|
||||
Note: when you change the wallet it is persistant until you specify you want to use another wallet - lbryum - with the --wallet flag again.
|
||||
|
||||
***********
|
||||
|
||||
Q: Where are all the behind the scenes files?
|
||||
|
||||
A: On linux, the relevant directories are `~/.lbrynet`, `~/.lbrycrd`, and `~/.lbryum`, depending on which wallets you've used. On OS X, the folders of interest are `~/Library/Application Support/LBRY`, `~/.lbrycrd` and `~/.lbryum`, also depending on which wallets you've used.
|
||||
|
||||
***********
|
||||
|
||||
Q: How can I see the log in the console?
|
||||
|
||||
A: Run lbry with the --log-to-console flag set: `lbrynet-daemon --log-to-console`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I specify a web-UI to use?
|
||||
|
||||
A: If the files for the UI you'd like to use are storred locally on your computer, start lbry with the --ui flag: `lbrynet-daemon --ui=/full/path/to/ui/files/root/folder`
|
||||
|
||||
Note, once set with the UI flag the given UI will be cached by lbry and used as the default going forward. Also, it will only successfully load a UI if it contains a conforming requirements.txt file to specify required lbrynet and lbryum versions. [Here](https://github.com/lbryio/lbry-web-ui/blob/master/dist/requirements.txt) is an example requirements.txt file.
|
||||
|
||||
To reset your ui to pull from lbryio, or to try a UI still in development, run lbry with the --branch flag: `lbrynet=daemon --branch=master`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I see the list of API functions I can call, and how do I call them?
|
||||
|
||||
A: Here is an example script to get the documentation for the various API calls. To use any of the functions displayed, just provide any specified arguments in a dictionary.
|
||||
|
||||
Note: the lbry api can only be used while either the app or lbrynet-daemon command line are running
|
||||
|
||||
import sys
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
sys.exit(0)
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
status = api.status()
|
||||
if not status['is_running']:
|
||||
print status
|
||||
else:
|
||||
for cmd in api.commands():
|
||||
print "%s:\n%s" % (cmd, api.help({'command': cmd}))
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
To use the LBRYWallet, which enables spending and accepting LBRYcrds in exchange for data, the
|
||||
LBRYcrd application (insert link to LBRYcrd website here) must be installed and running. If
|
||||
this is not desired, the testing client can be used to simulate trading points, which is
|
||||
built into LBRYnet.
|
||||
|
||||
on Ubuntu:
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
|
||||
Getting the source
|
||||
------------------
|
||||
|
||||
Don't you already have it?
|
||||
|
||||
Setting up the environment
|
||||
--------------------------
|
||||
|
||||
It's recommended that you use a virtualenv
|
||||
|
||||
sudo apt-get install python-virtualenv
|
||||
cd <source base directory>
|
||||
virtualenv .
|
||||
source bin/activate
|
||||
|
||||
(to deactivate the virtualenv, enter 'deactivate')
|
||||
|
||||
python setup.py install
|
||||
|
||||
this will install all of the libraries and a few applications
|
||||
|
||||
For running the file sharing application, see RUNNING
|
||||
@@ -0,0 +1,100 @@
|
||||
# Installing LBRY
|
||||
|
||||
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### OS X Prerequisites
|
||||
|
||||
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install mpfr libmpc protobuf
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
```
|
||||
|
||||
### Linux Prerequisites
|
||||
|
||||
On Ubuntu (we recommend 16.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux flavor, install the equivalent of the above packages for your system.
|
||||
|
||||
### Windows Prerequisites
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
|
||||
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- Yapsy
|
||||
- cx_Freeze
|
||||
- requests
|
||||
- gmpy
|
||||
|
||||
Install the above binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
|
||||
|
||||
## Installation
|
||||
|
||||
To install:
|
||||
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
pip install -U -r requirements.txt
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
|
||||
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
## Making a Change
|
||||
|
||||
If you are not already inside the virtual environment, run:
|
||||
|
||||
```
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
This will update `lbrynet-daemon` and other executables.
|
||||
@@ -1,22 +1,15 @@
|
||||
Copyright (c) 2015, LBRY, Inc.
|
||||
All rights reserved.
|
||||
The MIT License (MIT)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
Copyright (c) 2015-2016 LBRY Inc
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -1,57 +0,0 @@
|
||||
LBRYnet
|
||||
=======
|
||||
|
||||
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
|
||||
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
|
||||
table, used by peers to discover other peers.
|
||||
|
||||
Overview
|
||||
--------
|
||||
|
||||
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
|
||||
guarantees that peers can verify the correctness of each chunk without having to know anything
|
||||
about its contents, and can confidently re-transmit the chunk to other peers. Peers wishing to
|
||||
transmit chunks to other peers announce to the distributed hash table that they are associated
|
||||
with the sha384 hash sum in question. When a peer wants to download that chunk from the network,
|
||||
it asks the distributed hash table which peers are associated with that sha384 hash sum. The
|
||||
distributed hash table can also be used more generally. It simply stores IP addresses and
|
||||
ports which are associated with 384-bit numbers, and can be used by any type of application to
|
||||
help peers find each other. For example, an application for which clients don't know all of the
|
||||
necessary chunks may use some identifier, chosen by the application, to find clients which do
|
||||
know all of the necessary chunks.
|
||||
|
||||
Running
|
||||
-------
|
||||
|
||||
LBRYnet comes with an file sharing application, called 'lbrynet-console', which breaks
|
||||
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
|
||||
a special file called a 'stream descriptor' containing the hash sums and some other file metadata,
|
||||
and makes the chunks available for download by other peers. A peer wishing to download the file
|
||||
must first obtain the 'stream descriptor' and then may open it with his 'lbrynet-console' client,
|
||||
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
|
||||
chunks into the original file, according to the metadata included in the 'stream descriptor'.
|
||||
|
||||
To install and use this client, see INSTALL and RUNNING
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
See INSTALL
|
||||
|
||||
Developers
|
||||
----------
|
||||
|
||||
Documentation: doc.lbry.io
|
||||
Source code: trac.lbry.io/browser
|
||||
|
||||
To contribute to the development of LBRYnet or lbrynet-console, contact jimmy@lbry.io
|
||||
|
||||
Support
|
||||
-------
|
||||
|
||||
Send all support requests to jimmy@lbry.io
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
See LICENSE
|
||||
@@ -0,0 +1,52 @@
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
[](https://coveralls.io/github/lbryio/lbry)
|
||||
|
||||
# LBRY
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network.
|
||||
|
||||
This repo is a reference implementation of the [LBRY API](https://lbry.io/api).
|
||||
|
||||
It provides a daemon that can interact with the network via a json-rpc interface over HTTP.
|
||||
|
||||
## Installing
|
||||
|
||||
**Note**: This project no longer bundles a graphic interface (browser). If you want to use LBRY via a GUI, [use the LBRY App](https://github.com/lbryio/lbry-app).
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows.
|
||||
|
||||
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
## Running
|
||||
|
||||
Run `lbrynet-daemon` to launch the daemon.
|
||||
|
||||
## Using
|
||||
|
||||
By default, `lbrynet-daemon` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
|
||||
|
||||
The full API is documented [here](https://lbry.io/api).
|
||||
|
||||
## What is LBRY?
|
||||
|
||||
LBRY is a protocol providing fully decentralized network for the discovery, distribution, and payment of data.
|
||||
|
||||
It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules.
|
||||
|
||||
LBRY also provides a data network consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
|
||||
|
||||
## Contributions
|
||||
|
||||
To contribute, [join us on Slack](https://slack.lbry.io/) or contact jeremy@lbry.io. Pull requests are also welcome.
|
||||
|
||||
## Support
|
||||
|
||||
Please open an issue and describe your situation in detail. We will respond as soon as we can.
|
||||
|
||||
For private issues, contact jeremy@lbry.io.
|
||||
|
||||
## License
|
||||
|
||||
See [LICENSE](LICENSE)
|
||||
@@ -1,52 +0,0 @@
|
||||
To install LBRYnet and lbrynet-console, see INSTALL
|
||||
|
||||
lbrynet-console is a console application which makes use of the LBRYnet to share files.
|
||||
|
||||
In particular, lbrynet-console splits files into encrypted chunks of data compatible with
|
||||
LBRYnet, groups all metadata into a 'stream descriptor file' which can be sent directly to
|
||||
others wishing to obtain the file, or can be itself turned into a chunk compatible with
|
||||
LBRYnet and downloaded via LBRYnet by anyone knowing its sha384 hashsum. lbrynet-console
|
||||
also acts as a client whichreads a stream descriptor file, downloads the chunks of data
|
||||
specified by the hash sums found in the stream descriptor file, decrypts them according to
|
||||
metadata found in the stream, and reconstructs the original file. lbrynet-console features
|
||||
a server so that clients can connect to it and download the chunks and other data gotten
|
||||
from files created locally and files that have been downloaded from LBRYnet.
|
||||
|
||||
lbrynet-console also has a plugin system. There are two plugins: a live stream proof of
|
||||
concept which is currently far behind the development of the rest of the application and
|
||||
therefore will not run, and a plugin which attempts to determine which chunks on the
|
||||
network should be downloaded in order for the application to turn a profit. It will run,
|
||||
but its usefulness is extremely limited.
|
||||
|
||||
Passing '--help' to lbrynet-console will cause it to print out a quick help message
|
||||
describing other command line options to the application.
|
||||
|
||||
Once the application has been started, the user is presented with a numbered list of
|
||||
actions which looks something like this:
|
||||
|
||||
...
|
||||
[2] Toggle whether an LBRY File is running
|
||||
[3] Create an LBRY File from file
|
||||
[4] Publish a stream descriptor file to the DHT for an LBRY File
|
||||
...
|
||||
|
||||
To perform an action, type the desired number and then hit enter. For example, if you wish
|
||||
to create an LBRY file from a file as described in the beginning of this document, type 3 and
|
||||
hit enter.
|
||||
|
||||
If the application needs more input in order to for the action to be taken, the application
|
||||
will continue to print prompts for input until it has received what it needs.
|
||||
|
||||
For example, when creating an LBRY file from a file, the application needs to know which file
|
||||
it's supposed to use to create the LBRY file, so the user will be prompted for it:
|
||||
|
||||
File name:
|
||||
|
||||
The user should input the desired file name and hit enter, at which point the application
|
||||
will go about splitting the file and making it available on the network.
|
||||
|
||||
Some actions will produce sub-menus of actions, which work the same way.
|
||||
|
||||
A more detailed user guide is available at doc.lbry.io
|
||||
|
||||
Any issues may be reported to jimmy@lbry.io
|
||||
@@ -0,0 +1,34 @@
|
||||
$env:Path += ";C:\MinGW\bin\"
|
||||
|
||||
$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
|
||||
gcc --version
|
||||
mingw32-make --version
|
||||
|
||||
# build/install miniupnpc manually
|
||||
tar zxf miniupnpc-1.9.tar.gz
|
||||
cd miniupnpc-1.9
|
||||
mingw32-make -f Makefile.mingw
|
||||
python setupmingw32.py build --compiler=mingw32
|
||||
python setupmingw32.py install
|
||||
cd ..\
|
||||
Remove-Item -Recurse -Force miniupnpc-1.9
|
||||
|
||||
# copy requirements from lbry, but remove gmpy and miniupnpc (installed manually)
|
||||
Get-Content ..\requirements.txt | Select-String -Pattern 'gmpy|miniupnpc' -NotMatch | Out-File requirements_base.txt
|
||||
# add in gmpy wheel
|
||||
Add-Content requirements.txt "./gmpy-1.17-cp27-none-win32.whl"
|
||||
|
||||
python set_build.py
|
||||
|
||||
pip install -r requirements.txt
|
||||
pip install ..\.
|
||||
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
|
||||
nuget install secure-file -ExcludeVersion
|
||||
secure-file\tools\secure-file -decrypt .\lbry2.pfx.enc -secret "$env:pfx_key"
|
||||
signtool.exe sign /f .\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
|
||||
|
||||
python zip_daemon.py
|
||||
python upload_assets.py
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
|
||||
cd "$ROOT"
|
||||
BUILD_DIR="$ROOT/build"
|
||||
|
||||
FULL_BUILD="${FULL_BUILD:-false}"
|
||||
if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then
|
||||
FULL_BUILD="true"
|
||||
fi
|
||||
|
||||
[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build"
|
||||
[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# install dependencies
|
||||
$BUILD_DIR/prebuild.sh
|
||||
|
||||
VENV="$BUILD_DIR/venv"
|
||||
if [ -d "$VENV" ]; then
|
||||
rm -rf "$VENV"
|
||||
fi
|
||||
virtualenv "$VENV"
|
||||
set +u
|
||||
source "$VENV/bin/activate"
|
||||
set -u
|
||||
|
||||
# must set build before installing lbrynet. otherwise it has no effect
|
||||
python "$BUILD_DIR/set_build.py"
|
||||
fi
|
||||
|
||||
cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt"
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pip install -r requirements.txt
|
||||
)
|
||||
|
||||
(
|
||||
cd "$BUILD_DIR"
|
||||
pyinstaller -y daemon.onefile.spec
|
||||
pyinstaller -y cli.onefile.spec
|
||||
)
|
||||
|
||||
python "$BUILD_DIR/zip_daemon.py"
|
||||
|
||||
if [ "$FULL_BUILD" == "true" ]; then
|
||||
# electron-build has a publish feature, but I had a hard time getting
|
||||
# it to reliably work and it also seemed difficult to configure. Not proud of
|
||||
# this, but it seemed better to write my own.
|
||||
python "$BUILD_DIR/upload_assets.py"
|
||||
|
||||
deactivate
|
||||
fi
|
||||
|
||||
echo 'Build complete.'
|
||||
@@ -0,0 +1,129 @@
|
||||
import datetime
|
||||
import re
|
||||
|
||||
CHANGELOG_START_RE = re.compile(r'^\#\# \[Unreleased\]')
|
||||
CHANGELOG_END_RE = re.compile(r'^\#\# \[.*\] - \d{4}-\d{2}-\d{2}')
|
||||
# if we come across a section header between two release section headers
|
||||
# then we probably have an improperly formatted changelog
|
||||
CHANGELOG_ERROR_RE = re.compile(r'^\#\# ')
|
||||
SECTION_RE = re.compile(r'^\#\#\# (.*)$')
|
||||
EMPTY_RE = re.compile(r'^\w*\*\w*$')
|
||||
ENTRY_RE = re.compile(r'\* (.*)')
|
||||
VALID_SECTIONS = ['Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Security']
|
||||
|
||||
# allocate some entries to cut-down on merge conflicts
|
||||
TEMPLATE = """### Added
|
||||
*
|
||||
*
|
||||
|
||||
### Changed
|
||||
*
|
||||
*
|
||||
|
||||
### Fixed
|
||||
*
|
||||
*
|
||||
|
||||
### Deprecated
|
||||
*
|
||||
*
|
||||
|
||||
### Removed
|
||||
*
|
||||
*
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class Changelog(object):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.start = []
|
||||
self.unreleased = []
|
||||
self.rest = []
|
||||
self._parse()
|
||||
|
||||
def _parse(self):
|
||||
with open(self.path) as fp:
|
||||
lines = fp.readlines()
|
||||
|
||||
unreleased_start_found = False
|
||||
unreleased_end_found = False
|
||||
|
||||
for line in lines:
|
||||
if not unreleased_start_found:
|
||||
self.start.append(line)
|
||||
if CHANGELOG_START_RE.search(line):
|
||||
unreleased_start_found = True
|
||||
continue
|
||||
if unreleased_end_found:
|
||||
self.rest.append(line)
|
||||
continue
|
||||
if CHANGELOG_END_RE.search(line):
|
||||
self.rest.append(line)
|
||||
unreleased_end_found = True
|
||||
continue
|
||||
if CHANGELOG_ERROR_RE.search(line):
|
||||
raise Exception(
|
||||
'Failed to parse {}: {}'.format(self.path, 'unexpected section header found'))
|
||||
self.unreleased.append(line)
|
||||
|
||||
self.unreleased = self._normalize_section(self.unreleased)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_section(lines):
|
||||
"""Parse a changelog entry and output a normalized form"""
|
||||
sections = {}
|
||||
current_section_name = None
|
||||
current_section_contents = []
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or EMPTY_RE.match(line):
|
||||
continue
|
||||
match = SECTION_RE.match(line)
|
||||
if match:
|
||||
if current_section_contents:
|
||||
sections[current_section_name] = current_section_contents
|
||||
current_section_contents = []
|
||||
current_section_name = match.group(1)
|
||||
if current_section_name not in VALID_SECTIONS:
|
||||
raise ValueError("Section '{}' is not valid".format(current_section_name))
|
||||
continue
|
||||
match = ENTRY_RE.match(line)
|
||||
if match:
|
||||
current_section_contents.append(match.group(1))
|
||||
continue
|
||||
raise Exception('Something is wrong with line: {}'.format(line))
|
||||
if current_section_contents:
|
||||
sections[current_section_name] = current_section_contents
|
||||
|
||||
output = []
|
||||
for section in VALID_SECTIONS:
|
||||
if section not in sections:
|
||||
continue
|
||||
output.append('### {}'.format(section))
|
||||
for entry in sections[section]:
|
||||
output.append(' * {}'.format(entry))
|
||||
output.append("\n")
|
||||
return output
|
||||
|
||||
def get_unreleased(self):
|
||||
return '\n'.join(self.unreleased) if self.unreleased else None
|
||||
|
||||
def bump(self, version):
|
||||
if not self.unreleased:
|
||||
return
|
||||
|
||||
today = datetime.datetime.today()
|
||||
header = "## [{}] - {}\n\n".format(version, today.strftime('%Y-%m-%d'))
|
||||
|
||||
changelog_data = (
|
||||
''.join(self.start) +
|
||||
TEMPLATE +
|
||||
header +
|
||||
'\n'.join(self.unreleased) + '\n\n'
|
||||
+ ''.join(self.rest)
|
||||
)
|
||||
|
||||
with open(self.path, 'w') as fp:
|
||||
fp.write(changelog_data)
|
||||
@@ -0,0 +1,59 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
block_cipher = None
|
||||
|
||||
|
||||
a = Analysis(
|
||||
['cli.py'],
|
||||
pathex=[cwd],
|
||||
binaries=None,
|
||||
datas=[],
|
||||
hiddenimports=[],
|
||||
hookspath=[],
|
||||
runtime_hooks=[],
|
||||
excludes=[],
|
||||
win_no_prefer_redirects=False,
|
||||
win_private_assemblies=False,
|
||||
cipher=block_cipher
|
||||
)
|
||||
|
||||
pyz = PYZ(
|
||||
a.pure,
|
||||
a.zipped_data,
|
||||
cipher=block_cipher
|
||||
)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-cli',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
from lbrynet.daemon import DaemonCLI
|
||||
import logging
|
||||
|
||||
logging.basicConfig()
|
||||
|
||||
if __name__ == '__main__':
|
||||
DaemonCLI.main()
|
||||
@@ -0,0 +1,78 @@
|
||||
# -*- mode: python -*-
|
||||
import platform
|
||||
import os
|
||||
|
||||
import lbryum
|
||||
|
||||
|
||||
dir = 'build';
|
||||
cwd = os.getcwd()
|
||||
if os.path.basename(cwd) != dir:
|
||||
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
|
||||
repo_base = os.path.abspath(os.path.join(cwd, '..'))
|
||||
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
icns = os.path.join(repo_base, 'build', 'icon.icns')
|
||||
elif system == 'Linux':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
|
||||
elif system == 'Windows':
|
||||
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
|
||||
else:
|
||||
print 'Warning: System {} has no icons'.format(system)
|
||||
icns = None
|
||||
|
||||
|
||||
block_cipher = None
|
||||
|
||||
|
||||
languages = (
|
||||
'chinese_simplified.txt', 'japanese.txt', 'spanish.txt',
|
||||
'english.txt', 'portuguese.txt'
|
||||
)
|
||||
|
||||
|
||||
datas = [
|
||||
(
|
||||
os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language),
|
||||
'lbryum/wordlist'
|
||||
)
|
||||
for language in languages
|
||||
]
|
||||
|
||||
|
||||
a = Analysis(
|
||||
['daemon.py'],
|
||||
pathex=[cwd],
|
||||
binaries=None,
|
||||
datas=datas,
|
||||
hiddenimports=[],
|
||||
hookspath=[],
|
||||
runtime_hooks=[],
|
||||
excludes=[],
|
||||
win_no_prefer_redirects=False,
|
||||
win_private_assemblies=False,
|
||||
cipher=block_cipher
|
||||
)
|
||||
|
||||
|
||||
pyz = PYZ(
|
||||
a.pure, a.zipped_data,
|
||||
cipher=block_cipher
|
||||
)
|
||||
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='lbrynet-daemon',
|
||||
debug=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
icon=icns
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
from lbrynet.daemon import DaemonControl
|
||||
|
||||
if __name__ == '__main__':
|
||||
DaemonControl.start()
|
||||
|
After Width: | Height: | Size: 7.4 KiB |
|
After Width: | Height: | Size: 19 KiB |
|
After Width: | Height: | Size: 1.2 KiB |
|
After Width: | Height: | Size: 2.6 KiB |
|
After Width: | Height: | Size: 6.1 KiB |
|
After Width: | Height: | Size: 97 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 361 KiB |
|
After Width: | Height: | Size: 5.3 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 31 KiB |
@@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
|
||||
LINUX=false
|
||||
OSX=false
|
||||
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
OSX=true
|
||||
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
|
||||
LINUX=true
|
||||
else
|
||||
echo "Platform detection failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
SUDO=''
|
||||
if $LINUX && (( $EUID != 0 )); then
|
||||
SUDO='sudo'
|
||||
fi
|
||||
|
||||
cmd_exists() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
return $?
|
||||
}
|
||||
|
||||
set +eu
|
||||
GITUSERNAME=$(git config --global --get user.name)
|
||||
if [ -z "$GITUSERNAME" ]; then
|
||||
git config --global user.name "$(whoami)"
|
||||
fi
|
||||
GITEMAIL=$(git config --global --get user.email)
|
||||
if [ -z "$GITEMAIL" ]; then
|
||||
git config --global user.email "$(whoami)@lbry.io"
|
||||
fi
|
||||
set -eu
|
||||
|
||||
|
||||
if $LINUX; then
|
||||
INSTALL="$SUDO apt-get install --no-install-recommends -y"
|
||||
$INSTALL build-essential libssl-dev libffi-dev libgmp3-dev python2.7-dev
|
||||
elif $OSX && ! cmd_exists brew ; then
|
||||
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
|
||||
fi
|
||||
|
||||
|
||||
if ! cmd_exists python; then
|
||||
if $LINUX; then
|
||||
$INSTALL python2.7
|
||||
elif $OSX; then
|
||||
brew install python
|
||||
fi
|
||||
fi
|
||||
|
||||
PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
|
||||
if [ "$PYTHON_VERSION" != "2.7" ]; then
|
||||
echo "Python 2.7 required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! cmd_exists pip; then
|
||||
if $LINUX; then
|
||||
$INSTALL python-pip
|
||||
$SUDO pip install --upgrade pip
|
||||
else
|
||||
echo "Pip required"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
|
||||
#$INSTALL python-setuptools
|
||||
$SUDO pip install setuptools
|
||||
fi
|
||||
|
||||
if ! cmd_exists virtualenv; then
|
||||
$SUDO pip install virtualenv
|
||||
fi
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Bump version and create Github release
|
||||
|
||||
This script should be run locally, not on a build server.
|
||||
"""
|
||||
import argparse
|
||||
import contextlib
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import git
|
||||
import github
|
||||
|
||||
import changelog
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
|
||||
def main():
|
||||
bumpversion_parts = get_bumpversion_parts()
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("part", choices=bumpversion_parts, help="part of version to bump")
|
||||
parser.add_argument("--skip-sanity-checks", action="store_true")
|
||||
parser.add_argument("--skip-push", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--confirm", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.dry_run:
|
||||
print "DRY RUN. Nothing will be committed/pushed."
|
||||
|
||||
repo = Repo('lbry', args.part, ROOT)
|
||||
branch = 'master'
|
||||
|
||||
print 'Current version: {}'.format(repo.current_version)
|
||||
print 'New version: {}'.format(repo.new_version)
|
||||
|
||||
if not args.confirm and not confirm():
|
||||
print "Aborting"
|
||||
return 1
|
||||
|
||||
if not args.skip_sanity_checks:
|
||||
run_sanity_checks(repo, branch)
|
||||
repo.assert_new_tag_is_absent()
|
||||
|
||||
is_rc = re.search('\drc\d+$', repo.new_version) is not None
|
||||
# only have a release message for real releases, not for RCs
|
||||
release_msg = None if is_rc else repo.get_unreleased_changelog()
|
||||
if release_msg is None:
|
||||
release_msg = ''
|
||||
|
||||
if args.dry_run:
|
||||
print "rc: " + ("yes" if is_rc else "no")
|
||||
print "release message: \n" + (release_msg if not is_rc else " NO MESSAGE FOR RCs")
|
||||
return
|
||||
|
||||
gh_token = get_gh_token()
|
||||
auth = github.Github(gh_token)
|
||||
github_repo = auth.get_repo('lbryio/lbry')
|
||||
|
||||
if not is_rc:
|
||||
repo.bump_changelog()
|
||||
repo.bumpversion()
|
||||
|
||||
new_tag = repo.get_new_tag()
|
||||
github_repo.create_git_release(new_tag, new_tag, release_msg, draft=True, prerelease=is_rc)
|
||||
|
||||
if args.skip_push:
|
||||
print (
|
||||
'Skipping push; you will have to reset and delete tags if '
|
||||
'you want to run this script again.'
|
||||
)
|
||||
else:
|
||||
repo.git_repo.git.push(follow_tags=True, recurse_submodules='check')
|
||||
|
||||
|
||||
class Repo(object):
|
||||
def __init__(self, name, part, directory):
|
||||
self.name = name
|
||||
self.part = part
|
||||
if not self.part:
|
||||
raise Exception('Part required')
|
||||
self.directory = directory
|
||||
self.git_repo = git.Repo(self.directory)
|
||||
self._bumped = False
|
||||
|
||||
self.current_version = self._get_current_version()
|
||||
self.new_version = self._get_new_version()
|
||||
self._changelog = changelog.Changelog(os.path.join(self.directory, 'CHANGELOG.md'))
|
||||
|
||||
def get_new_tag(self):
|
||||
return 'v' + self.new_version
|
||||
|
||||
def get_unreleased_changelog(self):
|
||||
return self._changelog.get_unreleased()
|
||||
|
||||
def bump_changelog(self):
|
||||
self._changelog.bump(self.new_version)
|
||||
with pushd(self.directory):
|
||||
self.git_repo.git.add(os.path.basename(self._changelog.path))
|
||||
|
||||
def _get_current_version(self):
|
||||
with pushd(self.directory):
|
||||
output = subprocess.check_output(
|
||||
['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
|
||||
return re.search('^current_version=(.*)$', output, re.M).group(1)
|
||||
|
||||
def _get_new_version(self):
|
||||
with pushd(self.directory):
|
||||
output = subprocess.check_output(
|
||||
['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
|
||||
return re.search('^new_version=(.*)$', output, re.M).group(1)
|
||||
|
||||
def bumpversion(self):
|
||||
if self._bumped:
|
||||
raise Exception('Cowardly refusing to bump a repo twice')
|
||||
with pushd(self.directory):
|
||||
subprocess.check_call(['bumpversion', '--allow-dirty', self.part])
|
||||
self._bumped = True
|
||||
|
||||
def assert_new_tag_is_absent(self):
|
||||
new_tag = self.get_new_tag()
|
||||
tags = self.git_repo.git.tag()
|
||||
if new_tag in tags.split('\n'):
|
||||
raise Exception('Tag {} is already present in repo {}.'.format(new_tag, self.name))
|
||||
|
||||
def is_behind(self, branch):
|
||||
self.git_repo.remotes.origin.fetch()
|
||||
rev_list = '{branch}...origin/{branch}'.format(branch=branch)
|
||||
commits_behind = self.git_repo.git.rev_list(rev_list, right_only=True, count=True)
|
||||
commits_behind = int(commits_behind)
|
||||
return commits_behind > 0
|
||||
|
||||
|
||||
def get_bumpversion_parts():
|
||||
with pushd(ROOT):
|
||||
output = subprocess.check_output([
|
||||
'bumpversion', '--dry-run', '--list', '--allow-dirty', 'fake-part',
|
||||
])
|
||||
parse_line = re.search('^parse=(.*)$', output, re.M).group(1)
|
||||
return tuple(re.findall('<([^>]+)>', parse_line))
|
||||
|
||||
|
||||
def get_gh_token():
|
||||
if 'GH_TOKEN' in os.environ:
|
||||
return os.environ['GH_TOKEN']
|
||||
else:
|
||||
print """
|
||||
Please enter your personal access token. If you don't have one
|
||||
See https://github.com/lbryio/lbry-app/wiki/Release-Script#generate-a-personal-access-token
|
||||
for instructions on how to generate one.
|
||||
|
||||
You can also set the GH_TOKEN environment variable to avoid seeing this message
|
||||
in the future"""
|
||||
return raw_input('token: ').strip()
|
||||
|
||||
|
||||
def confirm():
|
||||
try:
|
||||
return raw_input('Is this what you want? [y/N] ').strip().lower() == 'y'
|
||||
except KeyboardInterrupt:
|
||||
return False
|
||||
|
||||
|
||||
def run_sanity_checks(repo, branch):
|
||||
if repo.git_repo.is_dirty():
|
||||
print 'Cowardly refusing to release a dirty repo'
|
||||
sys.exit(1)
|
||||
if repo.git_repo.active_branch.name != branch:
|
||||
print 'Cowardly refusing to release when not on the {} branch'.format(branch)
|
||||
sys.exit(1)
|
||||
if repo.is_behind(branch):
|
||||
print 'Cowardly refusing to release when behind origin'
|
||||
sys.exit(1)
|
||||
if not is_custom_bumpversion_version():
|
||||
print (
|
||||
'Install LBRY\'s fork of bumpversion: '
|
||||
'pip install -U git+https://github.com/lbryio/bumpversion.git'
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def is_custom_bumpversion_version():
|
||||
try:
|
||||
output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT).strip()
|
||||
if output == 'bumpversion 0.5.4-lbry':
|
||||
return True
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def pushd(new_dir):
|
||||
previous_dir = os.getcwd()
|
||||
os.chdir(new_dir)
|
||||
yield
|
||||
os.chdir(previous_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,11 @@
|
||||
# install daemon requirements (created by build script. see build.sh, build.ps1)
|
||||
-r requirements_base.txt
|
||||
|
||||
# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
|
||||
file:../.
|
||||
|
||||
# install other build requirements
|
||||
PyInstaller==3.2.1
|
||||
requests[security]==2.13.0
|
||||
uritemplate==3.0.0
|
||||
boto3==1.4.4
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
|
||||
|
||||
import os.path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
build = get_build()
|
||||
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f:
|
||||
f.write("BUILD = '{}'\n".format(build))
|
||||
|
||||
|
||||
def get_build():
|
||||
try:
|
||||
tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip()
|
||||
if re.match('v\d+\.\d+\.\d+rc\d+', tag):
|
||||
return 'rc'
|
||||
else:
|
||||
return 'release'
|
||||
except subprocess.CalledProcessError:
|
||||
# if the build doesn't have a tag
|
||||
return 'qa'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,143 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import github
|
||||
import uritemplate
|
||||
import boto3
|
||||
|
||||
|
||||
def main():
|
||||
upload_to_github_if_tagged('lbryio/lbry')
|
||||
upload_to_s3('daemon')
|
||||
|
||||
|
||||
def get_asset_filename():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
return glob.glob(this_dir + '/dist/*.zip')[0]
|
||||
|
||||
|
||||
def upload_to_s3(folder):
|
||||
tag = subprocess.check_output(['git', 'describe', '--always', 'HEAD']).strip()
|
||||
commit_date = subprocess.check_output([
|
||||
'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
bucket = 'releases.lbry.io'
|
||||
key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
|
||||
|
||||
print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
|
||||
|
||||
if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
|
||||
print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
|
||||
return 1
|
||||
|
||||
s3 = boto3.resource(
|
||||
's3',
|
||||
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
|
||||
config=boto3.session.Config(signature_version='s3v4')
|
||||
)
|
||||
s3.meta.client.upload_file(asset_path, bucket, key)
|
||||
|
||||
|
||||
def upload_to_github_if_tagged(repo_name):
|
||||
try:
|
||||
current_tag = subprocess.check_output(
|
||||
['git', 'describe', '--exact-match', 'HEAD']).strip()
|
||||
except subprocess.CalledProcessError:
|
||||
print 'Not uploading to GitHub as we are not currently on a tag'
|
||||
return 1
|
||||
|
||||
print "Current tag: " + current_tag
|
||||
|
||||
if 'GH_TOKEN' not in os.environ:
|
||||
print 'Must set GH_TOKEN in order to publish assets to a release'
|
||||
return 1
|
||||
|
||||
gh_token = os.environ['GH_TOKEN']
|
||||
auth = github.Github(gh_token)
|
||||
repo = auth.get_repo(repo_name)
|
||||
|
||||
if not check_repo_has_tag(repo, current_tag):
|
||||
print 'Tag {} is not in repo {}'.format(current_tag, repo)
|
||||
# TODO: maybe this should be an error
|
||||
return 1
|
||||
|
||||
asset_path = get_asset_filename()
|
||||
print "Uploading " + asset_path + " to Github tag " + current_tag
|
||||
release = get_github_release(repo, current_tag)
|
||||
upload_asset_to_github(release, asset_path, gh_token)
|
||||
|
||||
|
||||
def check_repo_has_tag(repo, target_tag):
|
||||
tags = repo.get_tags().get_page(0)
|
||||
for tag in tags:
|
||||
if tag.name == target_tag:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_github_release(repo, current_tag):
|
||||
for release in repo.get_releases():
|
||||
if release.tag_name == current_tag:
|
||||
return release
|
||||
raise Exception('No release for {} was found'.format(current_tag))
|
||||
|
||||
|
||||
def upload_asset_to_github(release, asset_to_upload, token):
|
||||
basename = os.path.basename(asset_to_upload)
|
||||
for asset in release.raw_data['assets']:
|
||||
if asset['name'] == basename:
|
||||
print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
|
||||
return
|
||||
|
||||
upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
|
||||
count = 0
|
||||
while count < 10:
|
||||
try:
|
||||
output = _curl_uploader(upload_uri, asset_to_upload, token)
|
||||
if 'errors' in output:
|
||||
raise Exception(output)
|
||||
else:
|
||||
print 'Successfully uploaded to {}'.format(output['browser_download_url'])
|
||||
except Exception:
|
||||
print 'Failed uploading on attempt {}'.format(count + 1)
|
||||
count += 1
|
||||
|
||||
|
||||
def _curl_uploader(upload_uri, asset_to_upload, token):
|
||||
# using requests.post fails miserably with SSL EPIPE errors. I spent
|
||||
# half a day trying to debug before deciding to switch to curl.
|
||||
#
|
||||
# TODO: actually set the content type
|
||||
print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
|
||||
cmd = [
|
||||
'curl',
|
||||
'-sS',
|
||||
'-X', 'POST',
|
||||
'-u', ':{}'.format(os.environ['GH_TOKEN']),
|
||||
'--header', 'Content-Type: application/octet-stream',
|
||||
'--data-binary', '@-',
|
||||
upload_uri
|
||||
]
|
||||
# '-d', '{"some_key": "some_value"}',
|
||||
print 'Calling curl:'
|
||||
print cmd
|
||||
print
|
||||
with open(asset_to_upload, 'rb') as fp:
|
||||
p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||
stdout, stderr = p.communicate()
|
||||
print 'curl return code:', p.returncode
|
||||
if stderr:
|
||||
print 'stderr output from curl:'
|
||||
print stderr
|
||||
print 'stdout from curl:'
|
||||
print stdout
|
||||
return json.loads(stdout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
|
||||
def main():
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
tag = subprocess.check_output(['git', 'describe']).strip()
|
||||
zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
|
||||
full_filename = os.path.join(this_dir, 'dist', zipfilename)
|
||||
executables = ['lbrynet-daemon', 'lbrynet-cli']
|
||||
ext = '.exe' if platform.system() == 'Windows' else ''
|
||||
with zipfile.ZipFile(full_filename, 'w') as myzip:
|
||||
for executable in executables:
|
||||
myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext)
|
||||
|
||||
|
||||
def get_system_label():
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
return 'macos'
|
||||
else:
|
||||
return system.lower()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,960 @@
|
||||
# LBRY Command Line Documentation
|
||||
|
||||
## blob_announce_all
|
||||
|
||||
```text
|
||||
Announce all blobs to the DHT
|
||||
|
||||
Usage:
|
||||
blob_announce_all
|
||||
|
||||
Returns:
|
||||
(str) Success/fail message
|
||||
```
|
||||
|
||||
## blob_delete
|
||||
|
||||
```text
|
||||
Delete a blob
|
||||
|
||||
Usage:
|
||||
blob_delete (<blob_hash> | --blob_hash=<blob_hash)
|
||||
|
||||
Returns:
|
||||
(str) Success/fail message
|
||||
```
|
||||
|
||||
## blob_get
|
||||
|
||||
```text
|
||||
Download and return a blob
|
||||
|
||||
Usage:
|
||||
blob_get (<blob_hash> | --blob_hash=<blob_hash>) [--timeout=<timeout>]
|
||||
[--encoding=<encoding>] [--payment_rate_manager=<payment_rate_manager>]
|
||||
|
||||
Options:
|
||||
--timeout=<timeout> : timeout in number of seconds
|
||||
--encoding=<encoding> : by default no attempt at decoding is made,
|
||||
can be set to one of the
|
||||
following decoders:
|
||||
'json'
|
||||
--payment_rate_manager=<payment_rate_manager> : if not given the default payment rate
|
||||
manager will be used.
|
||||
supported alternative rate managers:
|
||||
'only-free'
|
||||
|
||||
Returns
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
```
|
||||
|
||||
## blob_list
|
||||
|
||||
```text
|
||||
Returns blob hashes. If not given filters, returns all blobs known by the blob manager
|
||||
|
||||
Args:
|
||||
'uri' (optional): (str) filter by blobs in stream for winning claim
|
||||
'stream_hash' (optional): (str) filter by blobs in given stream hash
|
||||
'sd_hash' (optional): (str) filter by blobs in given sd hash
|
||||
'needed' (optional): (bool) only return needed blobs
|
||||
'finished' (optional): (bool) only return finished blobs
|
||||
'page_size' (optional): (int) limit number of results returned
|
||||
'page' (optional): (int) filter to page x of [page_size] results
|
||||
Returns:
|
||||
(list) List of blob hashes
|
||||
```
|
||||
|
||||
## blob_reflect_all
|
||||
|
||||
```text
|
||||
Reflects all saved blobs
|
||||
|
||||
Usage:
|
||||
blob_reflect_all
|
||||
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## block_show
|
||||
|
||||
```text
|
||||
Get contents of a block
|
||||
|
||||
Usage:
|
||||
block_show (<blockhash> | --blockhash=<blockhash>) | (<height> | --height=<height>)
|
||||
|
||||
Options:
|
||||
<blockhash>, --blockhash=<blockhash> : hash of the block to look up
|
||||
<height>, --height=<height> : height of the block to look up
|
||||
|
||||
Returns:
|
||||
(dict) Requested block
|
||||
```
|
||||
|
||||
## channel_list_mine
|
||||
|
||||
```text
|
||||
Get my channels
|
||||
|
||||
Usage:
|
||||
channel_list_mine
|
||||
|
||||
Returns:
|
||||
(list) ClaimDict
|
||||
```
|
||||
|
||||
## channel_new
|
||||
|
||||
```text
|
||||
Generate a publisher key and create a new '@' prefixed certificate claim
|
||||
|
||||
Usage:
|
||||
channel_new (<channel_name> | --channel_name=<channel_name>)
|
||||
(<amount> | --amount=<amount>)
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
'tx' : (str) hex encoded transaction
|
||||
'txid' : (str) txid of resulting claim
|
||||
'nout' : (int) nout of the resulting claim
|
||||
'fee' : (float) fee paid for the claim transaction
|
||||
'claim_id' : (str) claim ID of the resulting claim
|
||||
}
|
||||
```
|
||||
|
||||
## claim_abandon
|
||||
|
||||
```text
|
||||
Abandon a name and reclaim credits from the claim
|
||||
|
||||
Usage:
|
||||
claim_abandon (<claim_id> | --claim_id=<claim_id>)
|
||||
|
||||
Return:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
txid : (str) txid of resulting transaction
|
||||
fee : (float) fee paid for the transaction
|
||||
}
|
||||
```
|
||||
|
||||
## claim_list
|
||||
|
||||
```text
|
||||
List current claims and information about them for a given name
|
||||
|
||||
Usage:
|
||||
claim_list (<name> | --name=<name>)
|
||||
|
||||
Returns
|
||||
(dict) State of claims assigned for the name
|
||||
{
|
||||
'claims': (list) list of claims for the name
|
||||
[
|
||||
{
|
||||
'amount': (float) amount assigned to the claim
|
||||
'effective_amount': (float) total amount assigned to the claim,
|
||||
including supports
|
||||
'claim_id': (str) claim ID of the claim
|
||||
'height': (int) height of block containing the claim
|
||||
'txid': (str) txid of the claim
|
||||
'nout': (int) nout of the claim
|
||||
'supports': (list) a list of supports attached to the claim
|
||||
'value': (str) the value of the claim
|
||||
},
|
||||
]
|
||||
'supports_without_claims': (list) supports without any claims attached to them
|
||||
'last_takeover_height': (int) the height of last takeover for the name
|
||||
}
|
||||
```
|
||||
|
||||
## claim_list_by_channel
|
||||
|
||||
```text
|
||||
Get paginated claims in a channel specified by a channel uri
|
||||
|
||||
Usage:
|
||||
claim_list_by_channel (<uri> | --uri=<uri>) [<uris>...] [--page=<page>]
|
||||
[--page_size=<page_size>]
|
||||
|
||||
Options:
|
||||
--page=<page> : which page of results to return where page 1 is the first
|
||||
page, defaults to no pages
|
||||
--page_size=<page_size> : number of results in a page, default of 10
|
||||
|
||||
Returns:
|
||||
{
|
||||
resolved channel uri: {
|
||||
If there was an error:
|
||||
'error': (str) error message
|
||||
|
||||
'claims_in_channel_pages': total number of pages with <page_size> results,
|
||||
|
||||
If a page of results was requested:
|
||||
'returned_page': page number returned,
|
||||
'claims_in_channel': [
|
||||
{
|
||||
'absolute_channel_position': (int) claim index number in sorted list of
|
||||
claims which assert to be part of the
|
||||
channel
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
'has_signature': (bool) included if decoded_claim
|
||||
'name': (str) claim name,
|
||||
'supports: (list) list of supports [{'txid': txid,
|
||||
'nout': nout,
|
||||
'amount': amount}],
|
||||
'txid': (str) claim txid,
|
||||
'nout': (str) claim nout,
|
||||
'signature_is_valid': (bool), included if has_signature,
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## claim_list_mine
|
||||
|
||||
```text
|
||||
List my name claims
|
||||
|
||||
Usage:
|
||||
claim_list_mine
|
||||
|
||||
Returns
|
||||
(list) List of name claims owned by user
|
||||
[
|
||||
{
|
||||
'address': (str) address that owns the claim
|
||||
'amount': (float) amount assigned to the claim
|
||||
'blocks_to_expiration': (int) number of blocks until it expires
|
||||
'category': (str) "claim", "update" , or "support"
|
||||
'claim_id': (str) claim ID of the claim
|
||||
'confirmations': (int) number of blocks of confirmations for the claim
|
||||
'expiration_height': (int) the block height which the claim will expire
|
||||
'expired': (bool) true if expired, false otherwise
|
||||
'height': (int) height of the block containing the claim
|
||||
'is_spent': (bool) true if claim is abandoned, false otherwise
|
||||
'name': (str) name of the claim
|
||||
'txid': (str) txid of the cliam
|
||||
'nout': (int) nout of the claim
|
||||
'value': (str) value of the claim
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
## claim_new_support
|
||||
|
||||
```text
|
||||
Support a name claim
|
||||
|
||||
Usage:
|
||||
claim_new_support (<name> | --name=<name>) (<claim_id> | --claim_id=<claim_id>)
|
||||
(<amount> | --amount<amount>)
|
||||
|
||||
Return:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
txid : (str) txid of resulting support claim
|
||||
nout : (int) nout of the resulting support claim
|
||||
fee : (float) fee paid for the transaction
|
||||
}
|
||||
```
|
||||
|
||||
## claim_send_to_address
|
||||
|
||||
```text
|
||||
Send a name claim to an address
|
||||
|
||||
Usage:
|
||||
claim_send_to_address (<claim_id> | --claim_id=<claim_id>)
|
||||
(<address> | --address=<address>)
|
||||
[<amount> | --amount=<amount>]
|
||||
|
||||
Options:
|
||||
<amount> : Amount of credits to claim name for, defaults to the current amount
|
||||
on the claim
|
||||
```
|
||||
|
||||
## claim_show
|
||||
|
||||
```text
|
||||
Resolve claim info from a LBRY name
|
||||
|
||||
Usage:
|
||||
claim_show <name> [<txid> | --txid=<txid>] [<nout> | --nout=<nout>]
|
||||
[<claim_id> | --claim_id=<claim_id>]
|
||||
|
||||
Options:
|
||||
<txid>, --txid=<txid> : look for claim with this txid
|
||||
<nout>, --nout=<nout> : look for claim with this nout
|
||||
<claim_id>, --claim_id=<claim_id> : look for claim with this claim id
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary contaning claim info, (bool) false if claim is not
|
||||
resolvable
|
||||
|
||||
{
|
||||
'txid': (str) txid of claim
|
||||
'nout': (int) nout of claim
|
||||
'amount': (float) amount of claim
|
||||
'value': (str) value of claim
|
||||
'height' : (int) height of claim takeover
|
||||
'claim_id': (str) claim ID of claim
|
||||
'supports': (list) list of supports associated with claim
|
||||
}
|
||||
```
|
||||
|
||||
## cli_test_command
|
||||
|
||||
```text
|
||||
This command is only for testing the CLI argument parsing
|
||||
Usage:
|
||||
cli_test_command [-a] [-b] (<pos_arg> | --pos_arg=<pos_arg>)
|
||||
[<pos_args>...] [--pos_arg2=<pos_arg2>]
|
||||
[--pos_arg3=<pos_arg3>]
|
||||
|
||||
Options:
|
||||
-a, --a_arg : a arg
|
||||
-b, --b_arg : b arg
|
||||
<pos_arg2>, --pos_arg2=<pos_arg2> : pos arg 2
|
||||
<pos_arg3>, --pos_arg3=<pos_arg3> : pos arg 3
|
||||
Returns:
|
||||
pos args
|
||||
```
|
||||
|
||||
## commands
|
||||
|
||||
```text
|
||||
Return a list of available commands
|
||||
|
||||
Usage:
|
||||
commands
|
||||
|
||||
Returns:
|
||||
(list) list of available commands
|
||||
```
|
||||
|
||||
## daemon_stop
|
||||
|
||||
```text
|
||||
Stop lbrynet-daemon
|
||||
|
||||
Usage:
|
||||
daemon_stop
|
||||
|
||||
Returns:
|
||||
(string) Shutdown message
|
||||
```
|
||||
|
||||
## file_delete
|
||||
|
||||
```text
|
||||
Delete a LBRY file
|
||||
|
||||
Usage:
|
||||
file_delete [-f] [--delete_all] [--sd_hash=<sd_hash>] [--file_name=<file_name>]
|
||||
[--stream_hash=<stream_hash>] [--claim_id=<claim_id>]
|
||||
[--outpoint=<outpoint>] [--rowid=<rowid>]
|
||||
[--name=<name>]
|
||||
|
||||
Options:
|
||||
-f, --delete_from_download_dir : delete file from download directory,
|
||||
instead of just deleting blobs
|
||||
--delete_all : if there are multiple matching files,
|
||||
allow the deletion of multiple files.
|
||||
Otherwise do not delete anything.
|
||||
--sd_hash=<sd_hash> : delete by file sd hash
|
||||
--file_name<file_name> : delete by file name in downloads folder
|
||||
--stream_hash=<stream_hash> : delete by file stream hash
|
||||
--claim_id=<claim_id> : delete by file claim id
|
||||
--outpoint=<outpoint> : delete by file claim outpoint
|
||||
--rowid=<rowid> : delete by file row id
|
||||
--name=<name> : delete by associated name claim of file
|
||||
|
||||
Returns:
|
||||
(bool) true if deletion was successful
|
||||
```
|
||||
|
||||
## file_list
|
||||
|
||||
```text
|
||||
List files limited by optional filters
|
||||
|
||||
Usage:
|
||||
file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
|
||||
[--claim_id=<claim_id>] [--outpoint=<outpoint>] [--rowid=<rowid>]
|
||||
[--name=<name>]
|
||||
[-f]
|
||||
|
||||
Options:
|
||||
--sd_hash=<sd_hash> : get file with matching sd hash
|
||||
--file_name=<file_name> : get file with matching file name in the
|
||||
downloads folder
|
||||
--stream_hash=<stream_hash> : get file with matching stream hash
|
||||
--claim_id=<claim_id> : get file with matching claim id
|
||||
--outpoint=<outpoint> : get file with matching claim outpoint
|
||||
--rowid=<rowid> : get file with matching row id
|
||||
--name=<name> : get file with matching associated name claim
|
||||
-f : full status, populate the 'message' and 'size' fields
|
||||
|
||||
Returns:
|
||||
(list) List of files
|
||||
|
||||
[
|
||||
{
|
||||
'completed': (bool) true if download is completed,
|
||||
'file_name': (str) name of file,
|
||||
'download_directory': (str) download directory,
|
||||
'points_paid': (float) credit paid to download file,
|
||||
'stopped': (bool) true if download is stopped,
|
||||
'stream_hash': (str) stream hash of file,
|
||||
'stream_name': (str) stream name ,
|
||||
'suggested_file_name': (str) suggested file name,
|
||||
'sd_hash': (str) sd hash of file,
|
||||
'name': (str) name claim attached to file
|
||||
'outpoint': (str) claim outpoint attached to file
|
||||
'claim_id': (str) claim ID attached to file,
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes, None if full_status is false
|
||||
'written_bytes': (int) written size in bytes
|
||||
'message': (str), None if full_status is false
|
||||
'metadata': (dict) Metadata dictionary
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
## file_set_status
|
||||
|
||||
```text
|
||||
Start or stop downloading a file
|
||||
|
||||
Usage:
|
||||
file_set_status <status> [--sd_hash=<sd_hash>] [--file_name=<file_name>]
|
||||
[--stream_hash=<stream_hash>] [--claim_id=<claim_id>]
|
||||
[--outpoint=<outpoint>] [--rowid=<rowid>]
|
||||
[--name=<name>]
|
||||
|
||||
Options:
|
||||
--sd_hash=<sd_hash> : set status of file with matching sd hash
|
||||
--file_name=<file_name> : set status of file with matching file name in the
|
||||
downloads folder
|
||||
--stream_hash=<stream_hash> : set status of file with matching stream hash
|
||||
--claim_id=<claim_id> : set status of file with matching claim id
|
||||
--outpoint=<outpoint> : set status of file with matching claim outpoint
|
||||
--rowid=<rowid> : set status of file with matching row id
|
||||
--name=<name> : set status of file with matching associated name claim
|
||||
|
||||
Returns:
|
||||
(str) Confirmation message
|
||||
```
|
||||
|
||||
## get
|
||||
|
||||
```text
|
||||
Download stream from a LBRY name.
|
||||
|
||||
Usage:
|
||||
get <uri> [<file_name> | --file_name=<file_name>] [<timeout> | --timeout=<timeout>]
|
||||
[<download_directory> | --download_directory=<download_directory>]
|
||||
|
||||
Options:
|
||||
<file_name> : specified name for the downloaded file
|
||||
<timeout> : download timeout in number of seconds
|
||||
<download_directory> : path to directory where file will be saved
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary contaning information about the stream
|
||||
{
|
||||
'completed': (bool) true if download is completed,
|
||||
'file_name': (str) name of file,
|
||||
'download_directory': (str) download directory,
|
||||
'points_paid': (float) credit paid to download file,
|
||||
'stopped': (bool) true if download is stopped,
|
||||
'stream_hash': (str) stream hash of file,
|
||||
'stream_name': (str) stream name,
|
||||
'suggested_file_name': (str) suggested file name,
|
||||
'sd_hash': (str) sd hash of file,
|
||||
'name': (str) name claim attached to file
|
||||
'outpoint': (str) claim outpoint attached to file
|
||||
'claim_id': (str) claim ID attached to file,
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes, None if full_status is false
|
||||
'written_bytes': (int) written size in bytes
|
||||
'message': (str), None if full_status is false
|
||||
'metadata': (dict) Metadata dictionary
|
||||
}
|
||||
```
|
||||
|
||||
## get_availability
|
||||
|
||||
```text
|
||||
Get stream availability for lbry uri
|
||||
|
||||
Usage:
|
||||
get_availability (<uri> | --uri=<uri>) [<sd_timeout> | --sd_timeout=<sd_timeout>]
|
||||
[<peer_timeout> | --peer_timeout=<peer_timeout>]
|
||||
|
||||
Options:
|
||||
<sd_timeout>, --sd_timeout=<sd_timeout> : sd blob download timeout
|
||||
<peer_timeout>, --peer_timeout=<peer_timeout> : how long to look for peers
|
||||
|
||||
Returns:
|
||||
(float) Peers per blob / total blobs
|
||||
```
|
||||
|
||||
## help
|
||||
|
||||
```text
|
||||
Return a useful message for an API command
|
||||
|
||||
Usage:
|
||||
help [<command> | --command=<command>]
|
||||
|
||||
Options:
|
||||
<command>, --command=<command> : command to retrieve documentation for
|
||||
```
|
||||
|
||||
## peer_list
|
||||
|
||||
```text
|
||||
Get peers for blob hash
|
||||
|
||||
Usage:
|
||||
peer_list (<blob_hash> | --blob_hash=<blob_hash>) [<timeout> | --timeout=<timeout>]
|
||||
|
||||
Options:
|
||||
<timeout>, --timeout=<timeout> : peer search timeout in seconds
|
||||
|
||||
Returns:
|
||||
(list) List of contacts
|
||||
```
|
||||
|
||||
## publish
|
||||
|
||||
```text
|
||||
Make a new name claim and publish associated data to lbrynet,
|
||||
update over existing claim if user already has a claim for name.
|
||||
|
||||
Fields required in the final Metadata are:
|
||||
'title'
|
||||
'description'
|
||||
'author'
|
||||
'language'
|
||||
'license'
|
||||
'nsfw'
|
||||
|
||||
Metadata can be set by either using the metadata argument or by setting individual arguments
|
||||
fee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,
|
||||
or sources. Individual arguments will overwrite the fields specified in metadata argument.
|
||||
|
||||
Usage:
|
||||
publish (<name> | --name=<name>) (<bid> | --bid=<bid>) [--metadata=<metadata>]
|
||||
[--file_path=<file_path>] [--fee=<fee>] [--title=<title>]
|
||||
[--description=<description>] [--author=<author>] [--language=<language>]
|
||||
[--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>]
|
||||
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
|
||||
[--claim_address=<claim_address>] [--change_address=<change_address>]
|
||||
|
||||
Options:
|
||||
--metadata=<metadata> : ClaimDict to associate with the claim.
|
||||
--file_path=<file_path> : path to file to be associated with name. If provided,
|
||||
a lbry stream of this file will be used in 'sources'.
|
||||
If no path is given but a metadata dict is provided,
|
||||
the source from the given metadata will be used.
|
||||
--fee=<fee> : Dictionary representing key fee to download content:
|
||||
{
|
||||
'currency': currency_symbol,
|
||||
'amount': float,
|
||||
'address': str, optional
|
||||
}
|
||||
supported currencies: LBC, USD, BTC
|
||||
If an address is not provided a new one will be
|
||||
automatically generated. Default fee is zero.
|
||||
--title=<title> : title of the publication
|
||||
--description=<description> : description of the publication
|
||||
--author=<author> : author of the publication
|
||||
--language=<language> : language of the publication
|
||||
--license=<license> : publication license
|
||||
--license_url=<license_url> : publication license url
|
||||
--thumbnail=<thumbnail> : thumbnail url
|
||||
--preview=<preview> : preview url
|
||||
--nsfw=<nsfw> : title of the publication
|
||||
--sources=<sources> : {'lbry_sd_hash':sd_hash} specifies sd hash of file
|
||||
--channel_name=<channel_name> : name of the publisher channel name in the wallet
|
||||
--channel_id=<channel_id> : claim id of the publisher channel, does not check
|
||||
for channel claim being in the wallet. This allows
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
--claim_address=<claim_address> : address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
'tx' : (str) hex encoded transaction
|
||||
'txid' : (str) txid of resulting claim
|
||||
'nout' : (int) nout of the resulting claim
|
||||
'fee' : (float) fee paid for the claim transaction
|
||||
'claim_id' : (str) claim ID of the resulting claim
|
||||
}
|
||||
```
|
||||
|
||||
## reflect
|
||||
|
||||
```text
|
||||
Reflect a stream
|
||||
|
||||
Usage:
|
||||
reflect (<sd_hash> | --sd_hash=<sd_hash>)
|
||||
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## report_bug
|
||||
|
||||
```text
|
||||
Report a bug to slack
|
||||
|
||||
Usage:
|
||||
report_bug (<message> | --message=<message>)
|
||||
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## resolve
|
||||
|
||||
```text
|
||||
Resolve given LBRY URIs
|
||||
|
||||
Usage:
|
||||
resolve [-f] (<uri> | --uri=<uri>) [<uris>...]
|
||||
|
||||
Options:
|
||||
-f : force refresh and ignore cache
|
||||
|
||||
Returns:
|
||||
Dictionary of results, keyed by uri
|
||||
'<uri>': {
|
||||
If a resolution error occurs:
|
||||
'error': Error message
|
||||
|
||||
If the uri resolves to a channel or a claim in a channel:
|
||||
'certificate': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
'has_signature': (bool) included if decoded_claim
|
||||
'name': (str) claim name,
|
||||
'supports: (list) list of supports [{'txid': txid,
|
||||
'nout': nout,
|
||||
'amount': amount}],
|
||||
'txid': (str) claim txid,
|
||||
'nout': (str) claim nout,
|
||||
'signature_is_valid': (bool), included if has_signature,
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
|
||||
If the uri resolves to a claim:
|
||||
'claim': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
'has_signature': (bool) included if decoded_claim
|
||||
'name': (str) claim name,
|
||||
'channel_name': (str) channel name if claim is in a channel
|
||||
'supports: (list) list of supports [{'txid': txid,
|
||||
'nout': nout,
|
||||
'amount': amount}]
|
||||
'txid': (str) claim txid,
|
||||
'nout': (str) claim nout,
|
||||
'signature_is_valid': (bool), included if has_signature,
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## resolve_name
|
||||
|
||||
```text
|
||||
Resolve stream info from a LBRY name
|
||||
|
||||
Usage:
|
||||
resolve_name <name> [-f]
|
||||
|
||||
Options:
|
||||
-f : force refresh and do not check cache
|
||||
|
||||
Returns:
|
||||
(dict) Metadata dictionary from name claim, None if the name is not
|
||||
resolvable
|
||||
```
|
||||
|
||||
## send_amount_to_address
|
||||
|
||||
```text
|
||||
Queue a payment of credits to an address
|
||||
|
||||
Usage:
|
||||
send_amount_to_address (<amount> | --amount=<amount>) (<address> | --address=<address>)
|
||||
|
||||
Returns:
|
||||
(bool) true if payment successfully scheduled
|
||||
```
|
||||
|
||||
## settings_get
|
||||
|
||||
```text
|
||||
Get daemon settings
|
||||
|
||||
Usage:
|
||||
settings_get
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary of daemon settings
|
||||
See ADJUSTABLE_SETTINGS in lbrynet/conf.py for full list of settings
|
||||
```
|
||||
|
||||
## settings_set
|
||||
|
||||
```text
|
||||
Set daemon settings
|
||||
|
||||
Args:
|
||||
'run_on_startup': (bool) currently not supported
|
||||
'data_rate': (float) data rate,
|
||||
'max_key_fee': (float) maximum key fee,
|
||||
'download_directory': (str) path of where files are downloaded,
|
||||
'peer_port': (int) port through which daemon should connect,
|
||||
'max_upload': (float), currently not supported
|
||||
'max_download': (float), currently not supported
|
||||
'download_timeout': (int) download timeout in seconds
|
||||
'search_timeout': (float) search timeout in seconds
|
||||
'cache_time': (int) cache timeout in seconds
|
||||
Returns:
|
||||
(dict) Updated dictionary of daemon settings
|
||||
```
|
||||
|
||||
## status
|
||||
|
||||
```text
|
||||
Get daemon status
|
||||
|
||||
Usage:
|
||||
status [-s] [-d]
|
||||
|
||||
Options:
|
||||
-s : include session status in results
|
||||
-d : include dht network and peer status
|
||||
|
||||
Returns:
|
||||
(dict) lbrynet-daemon status
|
||||
{
|
||||
'lbry_id': lbry peer id, base58
|
||||
'installation_id': installation id, base58
|
||||
'is_running': bool
|
||||
'is_first_run': bool
|
||||
'startup_status': {
|
||||
'code': status code
|
||||
'message': status message
|
||||
},
|
||||
'connection_status': {
|
||||
'code': connection status code
|
||||
'message': connection status message
|
||||
},
|
||||
'blockchain_status': {
|
||||
'blocks': local blockchain height,
|
||||
'blocks_behind': remote_height - local_height,
|
||||
'best_blockhash': block hash of most recent block,
|
||||
},
|
||||
|
||||
If given the session status option:
|
||||
'session_status': {
|
||||
'managed_blobs': count of blobs in the blob manager,
|
||||
'managed_streams': count of streams in the file manager
|
||||
}
|
||||
|
||||
If given the dht status option:
|
||||
'dht_status': {
|
||||
'kbps_received': current kbps receiving,
|
||||
'kbps_sent': current kdps being sent,
|
||||
'total_bytes_sent': total bytes sent
|
||||
'total_bytes_received': total bytes received
|
||||
'queries_received': number of queries received per second
|
||||
'queries_sent': number of queries sent per second
|
||||
'recent_contacts': count of recently contacted peers
|
||||
'unique_contacts': count of unique peers
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## stream_cost_estimate
|
||||
|
||||
```text
|
||||
Get estimated cost for a lbry stream
|
||||
|
||||
Usage:
|
||||
stream_cost_estimate <uri> [<size> | --size=<size>]
|
||||
|
||||
Options:
|
||||
<size>, --size=<size> : stream size in bytes. if provided an sd blob won't be
|
||||
downloaded.
|
||||
|
||||
Returns:
|
||||
(float) Estimated cost in lbry credits, returns None if uri is not
|
||||
resolveable
|
||||
```
|
||||
|
||||
## transaction_list
|
||||
|
||||
```text
|
||||
List transactions belonging to wallet
|
||||
|
||||
Usage:
|
||||
transaction_list
|
||||
|
||||
Returns:
|
||||
(list) List of transactions
|
||||
```
|
||||
|
||||
## transaction_show
|
||||
|
||||
```text
|
||||
Get a decoded transaction from a txid
|
||||
|
||||
Usage:
|
||||
transaction_show (<txid> | --txid=<txid>)
|
||||
|
||||
Returns:
|
||||
(dict) JSON formatted transaction
|
||||
```
|
||||
|
||||
## version
|
||||
|
||||
```text
|
||||
Get lbry version information
|
||||
|
||||
Usage:
|
||||
version
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary of lbry version information
|
||||
{
|
||||
'build': (str) build type (e.g. "dev", "rc", "release"),
|
||||
'ip': (str) remote ip, if available,
|
||||
'lbrynet_version': (str) lbrynet_version,
|
||||
'lbryum_version': (str) lbryum_version,
|
||||
'lbryschema_version': (str) lbryschema_version,
|
||||
'os_release': (str) os release string
|
||||
'os_system': (str) os name
|
||||
'platform': (str) platform string
|
||||
'processor': (str) processor type,
|
||||
'python_version': (str) python version,
|
||||
}
|
||||
```
|
||||
|
||||
## wallet_balance
|
||||
|
||||
```text
|
||||
Return the balance of the wallet
|
||||
|
||||
Usage:
|
||||
wallet_balance [<address> | --address=<address>] [-u]
|
||||
|
||||
Options:
|
||||
<address> : If provided only the balance for this address will be given
|
||||
-u : Include unconfirmed
|
||||
|
||||
Returns:
|
||||
(float) amount of lbry credits in wallet
|
||||
```
|
||||
|
||||
## wallet_is_address_mine
|
||||
|
||||
```text
|
||||
Checks if an address is associated with the current wallet.
|
||||
|
||||
Usage:
|
||||
wallet_is_address_mine (<address> | --address=<address>)
|
||||
|
||||
Returns:
|
||||
(bool) true, if address is associated with current wallet
|
||||
```
|
||||
|
||||
## wallet_list
|
||||
|
||||
```text
|
||||
List wallet addresses
|
||||
|
||||
Usage:
|
||||
wallet_list
|
||||
|
||||
Returns:
|
||||
List of wallet addresses
|
||||
```
|
||||
|
||||
## wallet_new_address
|
||||
|
||||
```text
|
||||
Generate a new wallet address
|
||||
|
||||
Usage:
|
||||
wallet_new_address
|
||||
|
||||
Returns:
|
||||
(str) New wallet address in base58
|
||||
```
|
||||
|
||||
## wallet_public_key
|
||||
|
||||
```text
|
||||
Get public key from wallet address
|
||||
|
||||
Usage:
|
||||
wallet_public_key (<address> | --address=<address>)
|
||||
|
||||
Returns:
|
||||
(list) list of public keys associated with address.
|
||||
Could contain more than one public key if multisig.
|
||||
```
|
||||
|
||||
## wallet_unused_address
|
||||
|
||||
```text
|
||||
Return an address containing no balance, will create
|
||||
a new address if there is none.
|
||||
|
||||
Usage:
|
||||
wallet_unused_address
|
||||
|
||||
Returns:
|
||||
(str) Unused wallet address in base58
|
||||
```
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
width="80.207558mm"
|
||||
height="58.081333mm"
|
||||
viewBox="0 0 284.20001 205.8"
|
||||
id="svg3479"
|
||||
version="1.1"
|
||||
inkscape:version="0.91 r13725"
|
||||
sodipodi:docname="lbry-white-logo-only.svg">
|
||||
<defs
|
||||
id="defs3481" />
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="0.35"
|
||||
inkscape:cx="-470.0429"
|
||||
inkscape:cy="-5.6714247"
|
||||
inkscape:document-units="px"
|
||||
inkscape:current-layer="layer1"
|
||||
showgrid="false"
|
||||
inkscape:window-width="1920"
|
||||
inkscape:window-height="1056"
|
||||
inkscape:window-x="0"
|
||||
inkscape:window-y="24"
|
||||
inkscape:window-maximized="1"
|
||||
fit-margin-top="0"
|
||||
fit-margin-left="0"
|
||||
fit-margin-right="0"
|
||||
fit-margin-bottom="0" />
|
||||
<metadata
|
||||
id="metadata3484">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title></dc:title>
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Layer 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(-363.61433,-320.89078)">
|
||||
<g
|
||||
id="g3396"
|
||||
transform="translate(363.61433,320.89078)">
|
||||
<g
|
||||
id="g3398">
|
||||
<polygon
|
||||
style="fill:#ffffff"
|
||||
points="138.8,155.2 271,74 271,68.2 146.2,8 7,94.1 7,132.6 138.8,197.8 276.4,113.4 280.3,119.4 139.2,205.8 0,137 0,90.2 145.8,0 278,63.8 278,77.9 139.2,163.2 34.6,111.9 34.8,104 "
|
||||
id="polygon3400" />
|
||||
</g>
|
||||
<g
|
||||
id="g3402">
|
||||
<polygon
|
||||
style="fill:#ffffff"
|
||||
points="276.5,128.5 278.5,115.9 266.3,113.8 267.1,108.9 284.2,111.8 281.4,129.3 "
|
||||
id="polygon3404" />
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.3 KiB |
@@ -0,0 +1,758 @@
|
||||
# LBRY JSON-RPC API Documentation
|
||||
|
||||
## blob_announce_all
|
||||
|
||||
```text
|
||||
Announce all blobs to the DHT
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(str) Success/fail message
|
||||
```
|
||||
|
||||
## blob_delete
|
||||
|
||||
```text
|
||||
Delete a blob
|
||||
|
||||
Args:
|
||||
'blob_hash': (str) hash of blob to get
|
||||
Returns:
|
||||
(str) Success/fail message
|
||||
```
|
||||
|
||||
## blob_get
|
||||
|
||||
```text
|
||||
Download and return a blob
|
||||
|
||||
Args:
|
||||
'blob_hash': (str) blob hash of blob to get
|
||||
'timeout'(optional): (int) timeout in number of seconds
|
||||
'encoding'(optional): (str) by default no attempt at decoding is made,
|
||||
can be set to one of the following decoders:
|
||||
'json'
|
||||
'payment_rate_manager'(optional): if not given the default payment rate manager
|
||||
will be used. supported alternative rate managers:
|
||||
'only-free'
|
||||
|
||||
Returns
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
```
|
||||
|
||||
## blob_list
|
||||
|
||||
```text
|
||||
Returns blob hashes. If not given filters, returns all blobs known by the blob manager
|
||||
|
||||
Args:
|
||||
'uri' (optional): (str) filter by blobs in stream for winning claim
|
||||
'stream_hash' (optional): (str) filter by blobs in given stream hash
|
||||
'sd_hash' (optional): (str) filter by blobs in given sd hash
|
||||
'needed' (optional): (bool) only return needed blobs
|
||||
'finished' (optional): (bool) only return finished blobs
|
||||
'page_size' (optional): (int) limit number of results returned
|
||||
'page' (optional): (int) filter to page x of [page_size] results
|
||||
Returns:
|
||||
(list) List of blob hashes
|
||||
```
|
||||
|
||||
## blob_reflect_all
|
||||
|
||||
```text
|
||||
Reflects all saved blobs
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## block_show
|
||||
|
||||
```text
|
||||
Get contents of a block
|
||||
|
||||
Args:
|
||||
'blockhash': (str) hash of the block to look up
|
||||
Returns:
|
||||
(dict) Requested block
|
||||
```
|
||||
|
||||
## channel_list_mine
|
||||
|
||||
```text
|
||||
Get my channels
|
||||
|
||||
Returns:
|
||||
(list) ClaimDict
|
||||
```
|
||||
|
||||
## channel_new
|
||||
|
||||
```text
|
||||
Generate a publisher key and create a new certificate claim
|
||||
|
||||
Args:
|
||||
'channel_name': (str) '@' prefixed name
|
||||
'amount': (float) amount to claim name
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
'tx' : (str) hex encoded transaction
|
||||
'txid' : (str) txid of resulting claim
|
||||
'nout' : (int) nout of the resulting claim
|
||||
'fee' : (float) fee paid for the claim transaction
|
||||
'claim_id' : (str) claim ID of the resulting claim
|
||||
}
|
||||
```
|
||||
|
||||
## claim_abandon
|
||||
|
||||
```text
|
||||
Abandon a name and reclaim credits from the claim
|
||||
|
||||
Args:
|
||||
'claim_id': (str) claim_id of claim
|
||||
Return:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
txid : (str) txid of resulting transaction
|
||||
fee : (float) fee paid for the transaction
|
||||
}
|
||||
```
|
||||
|
||||
## claim_list
|
||||
|
||||
```text
|
||||
Get claims for a name
|
||||
|
||||
Args:
|
||||
'name': (str) search for claims on this name
|
||||
Returns
|
||||
(dict) State of claims assigned for the name
|
||||
{
|
||||
'claims': (list) list of claims for the name
|
||||
[
|
||||
{
|
||||
'amount': (float) amount assigned to the claim
|
||||
'effective_amount': (float) total amount assigned to the claim,
|
||||
including supports
|
||||
'claim_id': (str) claim ID of the claim
|
||||
'height': (int) height of block containing the claim
|
||||
'txid': (str) txid of the claim
|
||||
'nout': (int) nout of the claim
|
||||
'supports': (list) a list of supports attached to the claim
|
||||
'value': (str) the value of the claim
|
||||
},
|
||||
]
|
||||
'supports_without_claims': (list) supports without any claims attached to them
|
||||
'last_takeover_height': (int) the height of last takeover for the name
|
||||
}
|
||||
```
|
||||
|
||||
## claim_list_mine
|
||||
|
||||
```text
|
||||
List my name claims
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns
|
||||
(list) List of name claims owned by user
|
||||
[
|
||||
{
|
||||
'address': (str) address that owns the claim
|
||||
'amount': (float) amount assigned to the claim
|
||||
'blocks_to_expiration': (int) number of blocks until it expires
|
||||
'category': (str) "claim", "update" , or "support"
|
||||
'claim_id': (str) claim ID of the claim
|
||||
'confirmations': (int) number of blocks of confirmations for the claim
|
||||
'expiration_height': (int) the block height which the claim will expire
|
||||
'expired': (bool) true if expired, false otherwise
|
||||
'height': (int) height of the block containing the claim
|
||||
'is_spent': (bool) true if claim is abandoned, false otherwise
|
||||
'name': (str) name of the claim
|
||||
'txid': (str) txid of the cliam
|
||||
'nout': (int) nout of the claim
|
||||
'value': (str) value of the claim
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
## claim_new_support
|
||||
|
||||
```text
|
||||
Support a name claim
|
||||
|
||||
Args:
|
||||
'name': (str) name
|
||||
'claim_id': (str) claim ID of claim to support
|
||||
'amount': (float) amount to support by
|
||||
Return:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
txid : (str) txid of resulting support claim
|
||||
nout : (int) nout of the resulting support claim
|
||||
fee : (float) fee paid for the transaction
|
||||
}
|
||||
```
|
||||
|
||||
## claim_show
|
||||
|
||||
```text
|
||||
Resolve claim info from a LBRY name
|
||||
|
||||
Args:
|
||||
'name': (str) name to look up, do not include lbry:// prefix
|
||||
'txid'(optional): (str) if specified, look for claim with this txid
|
||||
'nout'(optional): (int) if specified, look for claim with this nout
|
||||
'claim_id'(optional): (str) if specified, look for claim with this claim_id
|
||||
Returns:
|
||||
(dict) Dictionary contaning claim info, (bool) false if claim is not
|
||||
resolvable
|
||||
|
||||
{
|
||||
'txid': (str) txid of claim
|
||||
'nout': (int) nout of claim
|
||||
'amount': (float) amount of claim
|
||||
'value': (str) value of claim
|
||||
'height' : (int) height of claim takeover
|
||||
'claim_id': (str) claim ID of claim
|
||||
'supports': (list) list of supports associated with claim
|
||||
}
|
||||
```
|
||||
|
||||
## commands
|
||||
|
||||
```text
|
||||
Return a list of available commands
|
||||
|
||||
Returns:
|
||||
(list) list of available commands
|
||||
```
|
||||
|
||||
## daemon_stop
|
||||
|
||||
```text
|
||||
Stop lbrynet-daemon
|
||||
|
||||
Returns:
|
||||
(string) Shutdown message
|
||||
```
|
||||
|
||||
## descriptor_get
|
||||
|
||||
```text
|
||||
Download and return a sd blob
|
||||
|
||||
Args:
|
||||
'sd_hash': (str) hash of sd blob
|
||||
'timeout'(optional): (int) timeout in number of seconds
|
||||
'payment_rate_manager'(optional): (str) if not given the default payment rate manager
|
||||
will be used. supported alternative rate managers:
|
||||
only-free
|
||||
|
||||
Returns
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
```
|
||||
|
||||
## file_delete
|
||||
|
||||
```text
|
||||
Delete a lbry file
|
||||
|
||||
Args:
|
||||
'name' (optional): (str) delete file by lbry name,
|
||||
'sd_hash' (optional): (str) delete file by sd hash,
|
||||
'file_name' (optional): (str) delete file by the name in the downloads folder,
|
||||
'stream_hash' (optional): (str) delete file by stream hash,
|
||||
'claim_id' (optional): (str) delete file by claim ID,
|
||||
'outpoint' (optional): (str) delete file by claim outpoint,
|
||||
'rowid': (optional): (int) delete file by rowid in the file manager
|
||||
'delete_target_file' (optional): (bool) delete file from downloads folder,
|
||||
defaults to true if false only the blobs and
|
||||
db entries will be deleted
|
||||
Returns:
|
||||
(bool) true if deletion was successful
|
||||
```
|
||||
|
||||
## file_list
|
||||
|
||||
```text
|
||||
List files limited by optional filters
|
||||
|
||||
Args:
|
||||
'name' (optional): (str) filter files by lbry name,
|
||||
'sd_hash' (optional): (str) filter files by sd hash,
|
||||
'file_name' (optional): (str) filter files by the name in the downloads folder,
|
||||
'stream_hash' (optional): (str) filter files by stream hash,
|
||||
'claim_id' (optional): (str) filter files by claim id,
|
||||
'outpoint' (optional): (str) filter files by claim outpoint,
|
||||
'rowid' (optional): (int) filter files by internal row id,
|
||||
'full_status': (optional): (bool) if true populate the 'message' and 'size' fields
|
||||
|
||||
Returns:
|
||||
(list) List of files
|
||||
|
||||
[
|
||||
{
|
||||
'completed': (bool) true if download is completed,
|
||||
'file_name': (str) name of file,
|
||||
'download_directory': (str) download directory,
|
||||
'points_paid': (float) credit paid to download file,
|
||||
'stopped': (bool) true if download is stopped,
|
||||
'stream_hash': (str) stream hash of file,
|
||||
'stream_name': (str) stream name ,
|
||||
'suggested_file_name': (str) suggested file name,
|
||||
'sd_hash': (str) sd hash of file,
|
||||
'name': (str) name claim attached to file
|
||||
'outpoint': (str) claim outpoint attached to file
|
||||
'claim_id': (str) claim ID attached to file,
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes, None if full_status is false
|
||||
'written_bytes': (int) written size in bytes
|
||||
'message': (str), None if full_status is false
|
||||
'metadata': (dict) Metadata dictionary
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
## file_set_status
|
||||
|
||||
```text
|
||||
Start or stop downloading a file
|
||||
|
||||
Args:
|
||||
'status': (str) "start" or "stop"
|
||||
'name' (optional): (str) start file by lbry name,
|
||||
'sd_hash' (optional): (str) start file by the hash in the name claim,
|
||||
'file_name' (optional): (str) start file by its name in the downloads folder,
|
||||
Returns:
|
||||
(str) Confirmation message
|
||||
```
|
||||
|
||||
## get
|
||||
|
||||
```text
|
||||
Download stream from a LBRY name.
|
||||
|
||||
Args:
|
||||
'uri': (str) lbry uri to download
|
||||
'file_name'(optional): (str) a user specified name for the downloaded file
|
||||
'timeout'(optional): (int) download timeout in number of seconds
|
||||
'download_directory'(optional): (str) path to directory where file will be saved
|
||||
Returns:
|
||||
(dict) Dictionary contaning information about the stream
|
||||
{
|
||||
'completed': (bool) true if download is completed,
|
||||
'file_name': (str) name of file,
|
||||
'download_directory': (str) download directory,
|
||||
'points_paid': (float) credit paid to download file,
|
||||
'stopped': (bool) true if download is stopped,
|
||||
'stream_hash': (str) stream hash of file,
|
||||
'stream_name': (str) stream name,
|
||||
'suggested_file_name': (str) suggested file name,
|
||||
'sd_hash': (str) sd hash of file,
|
||||
'name': (str) name claim attached to file
|
||||
'outpoint': (str) claim outpoint attached to file
|
||||
'claim_id': (str) claim ID attached to file,
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes, None if full_status is false
|
||||
'written_bytes': (int) written size in bytes
|
||||
'message': (str), None if full_status is false
|
||||
'metadata': (dict) Metadata dictionary
|
||||
}
|
||||
```
|
||||
|
||||
## get_availability
|
||||
|
||||
```text
|
||||
Get stream availability for lbry uri
|
||||
|
||||
Args:
|
||||
'uri' : (str) lbry uri
|
||||
'sd_timeout' (optional): (int) sd blob download timeout
|
||||
'peer_timeout' (optional): (int) how long to look for peers
|
||||
|
||||
Returns:
|
||||
(float) Peers per blob / total blobs
|
||||
```
|
||||
|
||||
## help
|
||||
|
||||
```text
|
||||
Return a useful message for an API command
|
||||
|
||||
Args:
|
||||
'command'(optional): (str) command to retrieve documentation for
|
||||
Returns:
|
||||
(str) if given a command, returns documentation about that command
|
||||
otherwise returns general help message
|
||||
```
|
||||
|
||||
## peer_list
|
||||
|
||||
```text
|
||||
Get peers for blob hash
|
||||
|
||||
Args:
|
||||
'blob_hash': (str) blob hash
|
||||
'timeout'(optional): (int) peer search timeout in seconds
|
||||
Returns:
|
||||
(list) List of contacts
|
||||
```
|
||||
|
||||
## publish
|
||||
|
||||
```text
|
||||
Make a new name claim and publish associated data to lbrynet,
|
||||
update over existing claim if user already has a claim for name.
|
||||
|
||||
Fields required in the final Metadata are:
|
||||
'title'
|
||||
'description'
|
||||
'author'
|
||||
'language'
|
||||
'license',
|
||||
'nsfw'
|
||||
|
||||
Metadata can be set by either using the metadata argument or by setting individual arguments
|
||||
fee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,
|
||||
or sources. Individual arguments will overwrite the fields specified in metadata argument.
|
||||
|
||||
Args:
|
||||
'name': (str) name to be claimed
|
||||
'bid': (float) amount of credits to commit in this claim,
|
||||
'metadata'(optional): (dict) Metadata to associate with the claim.
|
||||
'file_path'(optional): (str) path to file to be associated with name. If provided,
|
||||
a lbry stream of this file will be used in 'sources'.
|
||||
If no path is given but a metadata dict is provided, the source
|
||||
from the given metadata will be used.
|
||||
'fee'(optional): (dict) Dictionary representing key fee to download content:
|
||||
{currency_symbol: {'amount': float, 'address': str, optional}}
|
||||
supported currencies: LBC, USD, BTC
|
||||
If an address is not provided a new one will be automatically
|
||||
generated. Default fee is zero.
|
||||
'title'(optional): (str) title of the file
|
||||
'description'(optional): (str) description of the file
|
||||
'author'(optional): (str) author of the file
|
||||
'language'(optional): (str), language code
|
||||
'license'(optional): (str) license for the file
|
||||
'license_url'(optional): (str) URL to license
|
||||
'thumbnail'(optional): (str) thumbnail URL for the file
|
||||
'preview'(optional): (str) preview URL for the file
|
||||
'nsfw'(optional): (bool) True if not safe for work
|
||||
'sources'(optional): (dict){'lbry_sd_hash':sd_hash} specifies sd hash of file
|
||||
'channel_name' (optional): (str) name of the publisher channel
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
'tx' : (str) hex encoded transaction
|
||||
'txid' : (str) txid of resulting claim
|
||||
'nout' : (int) nout of the resulting claim
|
||||
'fee' : (float) fee paid for the claim transaction
|
||||
'claim_id' : (str) claim ID of the resulting claim
|
||||
}
|
||||
```
|
||||
|
||||
## reflect
|
||||
|
||||
```text
|
||||
Reflect a stream
|
||||
|
||||
Args:
|
||||
'sd_hash': (str) sd_hash of lbry file
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## report_bug
|
||||
|
||||
```text
|
||||
Report a bug to slack
|
||||
|
||||
Args:
|
||||
'message': (str) message to send
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## resolve
|
||||
|
||||
```text
|
||||
Resolve a LBRY URI
|
||||
|
||||
Args:
|
||||
'uri': (str) uri to download
|
||||
Returns:
|
||||
None if nothing can be resolved, otherwise:
|
||||
If uri resolves to a channel or a claim in a channel:
|
||||
'certificate': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
'has_signature': (bool) included if decoded_claim
|
||||
'name': (str) claim name,
|
||||
'supports: (list) list of supports [{'txid': txid,
|
||||
'nout': nout,
|
||||
'amount': amount}],
|
||||
'txid': (str) claim txid,
|
||||
'nout': (str) claim nout,
|
||||
'signature_is_valid': (bool), included if has_signature,
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
If uri resolves to a channel:
|
||||
'claims_in_channel': [
|
||||
{
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
'has_signature': (bool) included if decoded_claim
|
||||
'name': (str) claim name,
|
||||
'supports: (list) list of supports [{'txid': txid,
|
||||
'nout': nout,
|
||||
'amount': amount}],
|
||||
'txid': (str) claim txid,
|
||||
'nout': (str) claim nout,
|
||||
'signature_is_valid': (bool), included if has_signature,
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
]
|
||||
If uri resolves to a claim:
|
||||
'claim': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
'has_signature': (bool) included if decoded_claim
|
||||
'name': (str) claim name,
|
||||
'channel_name': (str) channel name if claim is in a channel
|
||||
'supports: (list) list of supports [{'txid': txid,
|
||||
'nout': nout,
|
||||
'amount': amount}]
|
||||
'txid': (str) claim txid,
|
||||
'nout': (str) claim nout,
|
||||
'signature_is_valid': (bool), included if has_signature,
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## resolve_name
|
||||
|
||||
```text
|
||||
Resolve stream info from a LBRY name
|
||||
|
||||
Args:
|
||||
'name': (str) name to look up, do not include lbry:// prefix
|
||||
Returns:
|
||||
(dict) Metadata dictionary from name claim, None if the name is not
|
||||
resolvable
|
||||
```
|
||||
|
||||
## send_amount_to_address
|
||||
|
||||
```text
|
||||
Send credits to an address
|
||||
|
||||
Args:
|
||||
'amount': (float) the amount to send
|
||||
'address': (str) the address of the recipient in base58
|
||||
Returns:
|
||||
(bool) true if payment successfully scheduled
|
||||
```
|
||||
|
||||
## settings_get
|
||||
|
||||
```text
|
||||
Get daemon settings
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary of daemon settings
|
||||
See ADJUSTABLE_SETTINGS in lbrynet/conf.py for full list of settings
|
||||
```
|
||||
|
||||
## settings_set
|
||||
|
||||
```text
|
||||
Set daemon settings
|
||||
|
||||
Args:
|
||||
'run_on_startup': (bool) currently not supported
|
||||
'data_rate': (float) data rate,
|
||||
'max_key_fee': (float) maximum key fee,
|
||||
'download_directory': (str) path of where files are downloaded,
|
||||
'peer_port': (int) port through which daemon should connect,
|
||||
'max_upload': (float), currently not supported
|
||||
'max_download': (float), currently not supported
|
||||
'download_timeout': (int) download timeout in seconds
|
||||
'search_timeout': (float) search timeout in seconds
|
||||
'cache_time': (int) cache timeout in seconds
|
||||
Returns:
|
||||
(dict) Updated dictionary of daemon settings
|
||||
```
|
||||
|
||||
## status
|
||||
|
||||
```text
|
||||
Return daemon status
|
||||
|
||||
Args:
|
||||
'session_status' (optional): (bool) true to return session status,
|
||||
default is false
|
||||
Returns:
|
||||
(dict) Daemon status dictionary
|
||||
```
|
||||
|
||||
## stream_cost_estimate
|
||||
|
||||
```text
|
||||
Get estimated cost for a lbry stream
|
||||
|
||||
Args:
|
||||
'name': (str) lbry name
|
||||
'size' (optional): (int) stream size, in bytes. if provided an sd blob
|
||||
won't be downloaded.
|
||||
Returns:
|
||||
(float) Estimated cost in lbry credits, returns None if uri is not
|
||||
resolveable
|
||||
```
|
||||
|
||||
## transaction_list
|
||||
|
||||
```text
|
||||
List transactions belonging to wallet
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(list) List of transactions
|
||||
```
|
||||
|
||||
## transaction_show
|
||||
|
||||
```text
|
||||
Get a decoded transaction from a txid
|
||||
|
||||
Args:
|
||||
'txid': (str) txid of transaction
|
||||
Returns:
|
||||
(dict) JSON formatted transaction
|
||||
```
|
||||
|
||||
## version
|
||||
|
||||
```text
|
||||
Get lbry version information
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(dict) Dictionary of lbry version information
|
||||
{
|
||||
'build': (str) build type (e.g. "dev", "rc", "release"),
|
||||
'ip': (str) remote ip, if available,
|
||||
'lbrynet_version': (str) lbrynet_version,
|
||||
'lbryum_version': (str) lbryum_version,
|
||||
'lbryschema_version': (str) lbryschema_version,
|
||||
'os_release': (str) os release string
|
||||
'os_system': (str) os name
|
||||
'platform': (str) platform string
|
||||
'processor': (str) processor type,
|
||||
'python_version': (str) python version,
|
||||
}
|
||||
```
|
||||
|
||||
## wallet_balance
|
||||
|
||||
```text
|
||||
Return the balance of the wallet
|
||||
|
||||
Args:
|
||||
'address' (optional): If address is provided only that balance will be given
|
||||
'include_unconfirmed' (optional): If set unconfirmed balance will be included in
|
||||
the only takes effect when address is also provided.
|
||||
|
||||
Returns:
|
||||
(float) amount of lbry credits in wallet
|
||||
```
|
||||
|
||||
## wallet_is_address_mine
|
||||
|
||||
```text
|
||||
Checks if an address is associated with the current wallet.
|
||||
|
||||
Args:
|
||||
'address': (str) address to check in base58
|
||||
Returns:
|
||||
(bool) true, if address is associated with current wallet
|
||||
```
|
||||
|
||||
## wallet_list
|
||||
|
||||
```text
|
||||
List wallet addresses
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
List of wallet addresses
|
||||
```
|
||||
|
||||
## wallet_new_address
|
||||
|
||||
```text
|
||||
Generate a new wallet address
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(str) New wallet address in base58
|
||||
```
|
||||
|
||||
## wallet_public_key
|
||||
|
||||
```text
|
||||
Get public key from wallet address
|
||||
|
||||
Args:
|
||||
'address': (str) wallet address in base58
|
||||
Returns:
|
||||
(list) list of public keys associated with address.
|
||||
Could contain more than one public key if multisig.
|
||||
```
|
||||
|
||||
## wallet_unused_address
|
||||
|
||||
```text
|
||||
Return an address containing no balance, will create
|
||||
a new address if there is none.
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(str) Unused wallet address in base58
|
||||
```
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
import logging
|
||||
|
||||
__version__ = "0.14.2rc2"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -0,0 +1,257 @@
|
||||
import collections
|
||||
import logging
|
||||
from twisted.internet import defer, task
|
||||
from requests import auth
|
||||
from txrequests import Session
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import looping_call_manager, utils, system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
SERVER_STARTUP_ERROR = 'Server Startup Error'
|
||||
DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
BLOB_BYTES_AVAILABLE = 'Blob Bytes Available'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
self.session_id = session_id or conf.settings.get_session_id()
|
||||
self.is_started = False
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
api = Api.new_instance(enabled)
|
||||
return cls(api)
|
||||
|
||||
# Things We Track
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
self._event(DOWNLOAD_ERRORED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_finished(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
self._event(DOWNLOAD_FINISHED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 60),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 300),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self._tracked_data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None):
|
||||
sd_hash = None
|
||||
if claim_dict:
|
||||
try:
|
||||
sd_hash = claim_dict.source_hash
|
||||
except (KeyError, TypeError, ValueError):
|
||||
log.debug('Failed to get sd_hash from %s', claim_dict, exc_info=True)
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
return {
|
||||
'app': {
|
||||
'name': 'lbrynet',
|
||||
'version': platform['lbrynet_version'],
|
||||
'python_version': platform['python_version'],
|
||||
'build': platform['build'],
|
||||
'wallet': {
|
||||
'name': wallet,
|
||||
'version': platform['lbryum_version'] if wallet == conf.LBRYUM_WALLET else None
|
||||
},
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
'library': {
|
||||
'name': 'lbrynet-analytics',
|
||||
'version': '1.0.0'
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api(object):
|
||||
def __init__(self, session, url, write_key, enabled):
|
||||
self.session = session
|
||||
self.url = url
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
assert endpoint[0] == '/'
|
||||
headers = {"Connection": "close"}
|
||||
return self.session.post(
|
||||
self.url + endpoint,
|
||||
json=data,
|
||||
auth=auth.HTTPBasicAuth(self._write_key, ''),
|
||||
headers=headers
|
||||
)
|
||||
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
session = Session()
|
||||
if enabled is None:
|
||||
enabled = conf.settings['share_usage_data']
|
||||
return cls(
|
||||
session,
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
|
||||
enabled,
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# dont touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
@@ -1,25 +1,472 @@
|
||||
"""
|
||||
Some network wide and also application specific parameters
|
||||
"""
|
||||
|
||||
|
||||
import base58
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
import envparse
|
||||
from appdirs import user_data_dir
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
ENV_NAMESPACE = 'LBRY_'
|
||||
|
||||
LBRYCRD_WALLET = 'lbrycrd'
|
||||
LBRYUM_WALLET = 'lbryum'
|
||||
PTC_WALLET = 'ptc'
|
||||
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
APP_NAME = 'LBRY'
|
||||
|
||||
LINUX = 1
|
||||
DARWIN = 2
|
||||
WINDOWS = 3
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
DEFAULT_DHT_NODES = [
|
||||
('lbrynet1.lbry.io', 4444),
|
||||
('lbrynet2.lbry.io', 4444),
|
||||
('lbrynet3.lbry.io', 4444)
|
||||
]
|
||||
|
||||
settings_decoders = {
|
||||
'.json': json.loads,
|
||||
'.yml': yaml.load
|
||||
}
|
||||
|
||||
settings_encoders = {
|
||||
'.json': json.dumps,
|
||||
'.yml': yaml.safe_dump
|
||||
}
|
||||
|
||||
def _win_path_to_bytes(path):
|
||||
"""
|
||||
Encode Windows paths to string. appdirs.user_data_dir()
|
||||
on windows will return unicode path, unlike other platforms
|
||||
which returns string. This will cause problems
|
||||
because we use strings for filenames and combining them with
|
||||
os.path.join() will result in errors.
|
||||
"""
|
||||
for encoding in ('ASCII', 'MBCS'):
|
||||
try:
|
||||
return path.encode(encoding)
|
||||
except (UnicodeEncodeError, LookupError):
|
||||
pass
|
||||
return path
|
||||
|
||||
if sys.platform.startswith('darwin'):
|
||||
platform = DARWIN
|
||||
default_download_directory = os.path.join(os.path.expanduser('~'), 'Downloads')
|
||||
default_data_dir = user_data_dir('LBRY')
|
||||
default_lbryum_dir = os.path.join(os.path.expanduser('~'), '.lbryum')
|
||||
elif sys.platform.startswith('win'):
|
||||
platform = WINDOWS
|
||||
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
|
||||
|
||||
default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
default_data_dir = os.path.join(
|
||||
get_path(FOLDERID.RoamingAppData, UserHandle.current), 'lbrynet')
|
||||
default_lbryum_dir = os.path.join(
|
||||
get_path(FOLDERID.RoamingAppData, UserHandle.current), 'lbryum')
|
||||
|
||||
default_download_directory = _win_path_to_bytes(default_download_directory)
|
||||
default_data_dir = _win_path_to_bytes(default_data_dir)
|
||||
default_lbryum_dir = _win_path_to_bytes(default_lbryum_dir)
|
||||
else:
|
||||
platform = LINUX
|
||||
default_download_directory = os.path.join(os.path.expanduser('~'), 'Downloads')
|
||||
default_data_dir = os.path.join(os.path.expanduser('~'), '.lbrynet')
|
||||
default_lbryum_dir = os.path.join(os.path.expanduser('~'), '.lbryum')
|
||||
|
||||
ICON_PATH = 'icons' if platform is WINDOWS else 'app.icns'
|
||||
|
||||
|
||||
MAX_HANDSHAKE_SIZE = 2**16
|
||||
MAX_REQUEST_SIZE = 2**16
|
||||
MAX_BLOB_REQUEST_SIZE = 2**16
|
||||
MAX_RESPONSE_INFO_SIZE = 2**16
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
BLOBFILES_DIR = ".blobfiles"
|
||||
BLOB_SIZE = 2**21
|
||||
MIN_BLOB_DATA_PAYMENT_RATE = .5 # points/megabyte
|
||||
MIN_BLOB_INFO_PAYMENT_RATE = 2.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MAX_CONNECTIONS_PER_STREAM = 5
|
||||
def server_port(server_and_port):
|
||||
server, port = server_and_port.split(':')
|
||||
return server, int(port)
|
||||
|
||||
POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
|
||||
#POINTTRADER_SERVER = 'http://127.0.0.1:2424'
|
||||
|
||||
CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
class Env(envparse.Env):
|
||||
"""An Env parser that automatically namespaces the variables with LBRY"""
|
||||
|
||||
def __init__(self, **schema):
|
||||
self.original_schema = schema
|
||||
my_schema = {
|
||||
self._convert_key(key): self._convert_value(value)
|
||||
for key, value in schema.items()
|
||||
}
|
||||
envparse.Env.__init__(self, **my_schema)
|
||||
|
||||
def __call__(self, key, *args, **kwargs):
|
||||
my_key = self._convert_key(key)
|
||||
return super(Env, self).__call__(my_key, *args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _convert_key(key):
|
||||
return ENV_NAMESPACE + key.upper()
|
||||
|
||||
@staticmethod
|
||||
def _convert_value(value):
|
||||
""" Allow value to be specified as a tuple or list.
|
||||
|
||||
If you do this, the tuple/list must be of the
|
||||
form (cast, default) or (cast, default, subcast)
|
||||
"""
|
||||
if isinstance(value, (tuple, list)):
|
||||
new_value = {'cast': value[0], 'default': value[1]}
|
||||
if len(value) == 3:
|
||||
new_value['subcast'] = value[2]
|
||||
return new_value
|
||||
return value
|
||||
|
||||
|
||||
TYPE_DEFAULT = 'default'
|
||||
TYPE_PERSISTED = 'persisted'
|
||||
TYPE_ENV = 'env'
|
||||
TYPE_CLI = 'cli'
|
||||
TYPE_RUNTIME = 'runtime'
|
||||
|
||||
FIXED_SETTINGS = {
|
||||
'ANALYTICS_ENDPOINT': 'https://api.segment.io/v1',
|
||||
'ANALYTICS_TOKEN': 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H=',
|
||||
'API_ADDRESS': 'lbryapi',
|
||||
'APP_NAME': APP_NAME,
|
||||
'BLOBFILES_DIR': 'blobfiles',
|
||||
'BLOB_SIZE': 2 * MB,
|
||||
'CRYPTSD_FILE_EXTENSION': '.cryptsd',
|
||||
'CURRENCIES': {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
},
|
||||
'DB_REVISION_FILE_NAME': 'db_revision',
|
||||
'ICON_PATH': ICON_PATH,
|
||||
'LOGGLY_TOKEN': 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv',
|
||||
'LOG_FILE_NAME': 'lbrynet.log',
|
||||
'LOG_POST_URL': 'https://lbry.io/log-upload',
|
||||
'MAX_BLOB_REQUEST_SIZE': 64 * KB,
|
||||
'MAX_HANDSHAKE_SIZE': 64 * KB,
|
||||
'MAX_REQUEST_SIZE': 64 * KB,
|
||||
'MAX_RESPONSE_INFO_SIZE': 64 * KB,
|
||||
'MAX_BLOB_INFOS_TO_REQUEST': 20,
|
||||
'PROTOCOL_PREFIX': PROTOCOL_PREFIX,
|
||||
'SLACK_WEBHOOK': ('nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='),
|
||||
'WALLET_TYPES': [LBRYUM_WALLET, LBRYCRD_WALLET],
|
||||
}
|
||||
|
||||
ADJUSTABLE_SETTINGS = {
|
||||
# By default, daemon will block all cross origin requests
|
||||
# but if this is set, this value will be used for the
|
||||
# Access-Control-Allow-Origin. For example
|
||||
# set to '*' to allow all requests, or set to 'http://localhost:8080'
|
||||
# if you're running a test UI on that port
|
||||
'allowed_origin': (str, ''),
|
||||
|
||||
# Changing this value is not-advised as it could potentially
|
||||
# expose the lbrynet daemon to the outside world which would
|
||||
# give an attacker access to your wallet and you could lose
|
||||
# all of your credits.
|
||||
'api_host': (str, 'localhost'),
|
||||
'api_port': (int, 5279),
|
||||
'cache_time': (int, 150),
|
||||
'data_dir': (str, default_data_dir),
|
||||
'data_rate': (float, .0001), # points/megabyte
|
||||
'delete_blobs_on_remove': (bool, True),
|
||||
'dht_node_port': (int, 4444),
|
||||
'download_directory': (str, default_download_directory),
|
||||
'download_timeout': (int, 180),
|
||||
'is_generous_host': (bool, True),
|
||||
'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_port),
|
||||
'lbryum_wallet_dir': (str, default_lbryum_dir),
|
||||
'max_connections_per_stream': (int, 5),
|
||||
# TODO: writing json on the cmd line is a pain, come up with a nicer
|
||||
# parser for this data structure. maybe 'USD:25'
|
||||
'max_key_fee': (json.loads, {'currency': 'USD', 'amount': 50.0}),
|
||||
'min_info_rate': (float, .02), # points/1000 infos
|
||||
'min_valuable_hash_rate': (float, .05), # points/1000 infos
|
||||
'min_valuable_info_rate': (float, .05), # points/1000 infos
|
||||
'peer_port': (int, 3333),
|
||||
'pointtrader_server': (str, 'http://127.0.0.1:2424'),
|
||||
'reflector_port': (int, 5566),
|
||||
'reflect_uploads': (bool, True),
|
||||
'reflector_servers': (list, [('reflector.lbry.io', 5566)], server_port),
|
||||
'run_reflector_server': (bool, False),
|
||||
'sd_download_timeout': (int, 3),
|
||||
'share_usage_data': (bool, True), # whether to share usage stats and diagnostic info with LBRY
|
||||
'peer_search_timeout': (int, 3),
|
||||
'use_auth_http': (bool, False),
|
||||
'use_upnp': (bool, True),
|
||||
'wallet': (str, LBRYUM_WALLET),
|
||||
}
|
||||
|
||||
|
||||
class Config(object):
|
||||
def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None,
|
||||
environment=None, cli_settings=None):
|
||||
|
||||
self._installation_id = None
|
||||
self._session_id = base58.b58encode(utils.generate_id())
|
||||
|
||||
self._fixed_defaults = fixed_defaults
|
||||
self._adjustable_defaults = adjustable_defaults
|
||||
|
||||
self._data = {
|
||||
TYPE_DEFAULT: {}, # defaults
|
||||
TYPE_PERSISTED: {}, # stored settings from daemon_settings.yml (or from a db, etc)
|
||||
TYPE_ENV: {}, # settings from environment variables
|
||||
TYPE_CLI: {}, # command-line arguments
|
||||
TYPE_RUNTIME: {}, # set during runtime (using self.set(), etc)
|
||||
}
|
||||
|
||||
# the order in which a piece of data is searched for. earlier types override later types
|
||||
self._search_order = (
|
||||
TYPE_RUNTIME, TYPE_CLI, TYPE_ENV, TYPE_PERSISTED, TYPE_DEFAULT
|
||||
)
|
||||
|
||||
self._data[TYPE_DEFAULT].update(self._fixed_defaults)
|
||||
self._data[TYPE_DEFAULT].update(
|
||||
{k: v[1] for (k, v) in self._adjustable_defaults.iteritems()})
|
||||
|
||||
if persisted_settings is None:
|
||||
persisted_settings = {}
|
||||
self._validate_settings(persisted_settings)
|
||||
self._data[TYPE_PERSISTED].update(persisted_settings)
|
||||
|
||||
env_settings = self._parse_environment(environment)
|
||||
self._validate_settings(env_settings)
|
||||
self._data[TYPE_ENV].update(env_settings)
|
||||
|
||||
if cli_settings is None:
|
||||
cli_settings = {}
|
||||
self._validate_settings(cli_settings)
|
||||
self._data[TYPE_CLI].update(cli_settings)
|
||||
|
||||
def __repr__(self):
|
||||
return self.get_current_settings_dict().__repr__()
|
||||
|
||||
def __iter__(self):
|
||||
for k in self._data[TYPE_DEFAULT].iterkeys():
|
||||
yield k
|
||||
|
||||
def __getitem__(self, name):
|
||||
return self.get(name)
|
||||
|
||||
def __setitem__(self, name, value):
|
||||
return self.set(name, value)
|
||||
|
||||
def __contains__(self, name):
|
||||
return name in self._data[TYPE_DEFAULT]
|
||||
|
||||
@staticmethod
|
||||
def _parse_environment(environment):
|
||||
env_settings = {}
|
||||
if environment is not None:
|
||||
assert isinstance(environment, Env)
|
||||
for opt in environment.original_schema:
|
||||
if environment(opt) is not None:
|
||||
env_settings[opt] = environment(opt)
|
||||
return env_settings
|
||||
|
||||
def _assert_valid_data_type(self, data_type):
|
||||
assert data_type in self._data, KeyError('{} in is not a valid data type'.format(data_type))
|
||||
|
||||
def get_valid_setting_names(self):
|
||||
return self._data[TYPE_DEFAULT].keys()
|
||||
|
||||
def _is_valid_setting(self, name):
|
||||
return name in self.get_valid_setting_names()
|
||||
|
||||
def _assert_valid_setting(self, name):
|
||||
assert self._is_valid_setting(name), \
|
||||
KeyError('{} is not a valid setting'.format(name))
|
||||
|
||||
def _validate_settings(self, data):
|
||||
invalid_settings = set(data.keys()) - set(self.get_valid_setting_names())
|
||||
if len(invalid_settings) > 0:
|
||||
raise KeyError('invalid settings: {}'.format(', '.join(invalid_settings)))
|
||||
|
||||
def _assert_editable_setting(self, name):
|
||||
self._assert_valid_setting(name)
|
||||
assert name not in self._fixed_defaults, \
|
||||
ValueError('{} is not an editable setting'.format(name))
|
||||
|
||||
def get(self, name, data_type=None):
|
||||
"""Get a config value
|
||||
|
||||
Args:
|
||||
name: the name of the value to get
|
||||
data_type: if given, get the value from a specific data set (see below)
|
||||
|
||||
Returns: the config value for the given name
|
||||
|
||||
If data_type is None, get() will search for the given name in each data set, in
|
||||
order of precedence. It will return the first value it finds. This is the "effective"
|
||||
value of a config name. For example, ENV values take precedence over DEFAULT values,
|
||||
so if a value is present in ENV and in DEFAULT, the ENV value will be returned
|
||||
"""
|
||||
self._assert_valid_setting(name)
|
||||
if data_type is not None:
|
||||
self._assert_valid_data_type(data_type)
|
||||
return self._data[data_type][name]
|
||||
for possible_data_type in self._search_order:
|
||||
if name in self._data[possible_data_type]:
|
||||
return self._data[possible_data_type][name]
|
||||
raise KeyError('{} is not a valid setting'.format(name))
|
||||
|
||||
def set(self, name, value, data_types=(TYPE_RUNTIME,)):
|
||||
"""Set a config value
|
||||
|
||||
Args:
|
||||
name: the name of the value to set
|
||||
value: the value
|
||||
data_types: what type(s) of data this is
|
||||
|
||||
Returns: None
|
||||
|
||||
By default, this sets the RUNTIME value of a config. If you wish to set other
|
||||
data types (e.g. PERSISTED values to save to a file, CLI values from parsed
|
||||
command-line options, etc), you can specify that with the data_types param
|
||||
"""
|
||||
self._assert_editable_setting(name)
|
||||
for data_type in data_types:
|
||||
self._assert_valid_data_type(data_type)
|
||||
self._data[data_type][name] = value
|
||||
|
||||
def update(self, updated_settings, data_types=(TYPE_RUNTIME,)):
|
||||
for k, v in updated_settings.iteritems():
|
||||
try:
|
||||
self.set(k, v, data_types=data_types)
|
||||
except (KeyError, AssertionError):
|
||||
pass
|
||||
|
||||
def get_current_settings_dict(self):
|
||||
current_settings = {}
|
||||
for key in self.get_valid_setting_names():
|
||||
current_settings[key] = self.get(key)
|
||||
return current_settings
|
||||
|
||||
def get_adjustable_settings_dict(self):
|
||||
return {
|
||||
key: val for key, val in self.get_current_settings_dict().iteritems()
|
||||
if key in self._adjustable_defaults
|
||||
}
|
||||
|
||||
def save_conf_file_settings(self):
|
||||
path = self.get_conf_filename()
|
||||
ext = os.path.splitext(path)[1]
|
||||
encoder = settings_encoders.get(ext, False)
|
||||
assert encoder is not False, 'Unknown settings format %s' % ext
|
||||
with open(path, 'w') as settings_file:
|
||||
settings_file.write(encoder(self._data[TYPE_PERSISTED]))
|
||||
|
||||
def load_conf_file_settings(self):
|
||||
path = self.get_conf_filename()
|
||||
ext = os.path.splitext(path)[1]
|
||||
decoder = settings_decoders.get(ext, False)
|
||||
assert decoder is not False, 'Unknown settings format %s' % ext
|
||||
try:
|
||||
with open(path, 'r') as settings_file:
|
||||
data = settings_file.read()
|
||||
decoded = self._fix_old_conf_file_settings(decoder(data))
|
||||
log.info('Loaded settings file: %s', path)
|
||||
self._validate_settings(decoded)
|
||||
self._data[TYPE_PERSISTED].update(decoded)
|
||||
except (IOError, OSError) as err:
|
||||
log.info('%s: Failed to update settings from %s', err, path)
|
||||
|
||||
def _fix_old_conf_file_settings(self, settings_dict):
|
||||
if 'API_INTERFACE' in settings_dict:
|
||||
settings_dict['api_host'] = settings_dict['API_INTERFACE']
|
||||
del settings_dict['API_INTERFACE']
|
||||
if 'startup_scripts' in settings_dict:
|
||||
del settings_dict['startup_scripts']
|
||||
if 'upload_log' in settings_dict:
|
||||
settings_dict['share_usage_data'] = settings_dict['upload_log']
|
||||
del settings_dict['upload_log']
|
||||
if 'share_debug_info' in settings_dict:
|
||||
settings_dict['share_usage_data'] = settings_dict['share_debug_info']
|
||||
del settings_dict['share_debug_info']
|
||||
for key in settings_dict.keys():
|
||||
if not self._is_valid_setting(key):
|
||||
log.warning('Ignoring invalid conf file setting: %s', key)
|
||||
del settings_dict[key]
|
||||
return settings_dict
|
||||
|
||||
def ensure_data_dir(self):
|
||||
# although there is a risk of a race condition here we don't
|
||||
# expect there to be multiple processes accessing this
|
||||
# directory so the risk can be ignored
|
||||
if not os.path.isdir(self['data_dir']):
|
||||
os.makedirs(self['data_dir'])
|
||||
return self['data_dir']
|
||||
|
||||
def get_log_filename(self):
|
||||
"""
|
||||
Return the log file for this platform.
|
||||
Also ensure the containing directory exists.
|
||||
"""
|
||||
return os.path.join(self.ensure_data_dir(), self['LOG_FILE_NAME'])
|
||||
|
||||
def get_api_connection_string(self):
|
||||
return 'http://%s:%i/%s' % (self['api_host'], self['api_port'], self['API_ADDRESS'])
|
||||
|
||||
def get_ui_address(self):
|
||||
return 'http://%s:%i' % (self['api_host'], self['api_port'])
|
||||
|
||||
def get_db_revision_filename(self):
|
||||
return os.path.join(self.ensure_data_dir(), self['DB_REVISION_FILE_NAME'])
|
||||
|
||||
def get_conf_filename(self):
|
||||
data_dir = self.ensure_data_dir()
|
||||
yml_path = os.path.join(data_dir, 'daemon_settings.yml')
|
||||
json_path = os.path.join(data_dir, 'daemon_settings.json')
|
||||
if os.path.isfile(yml_path):
|
||||
return yml_path
|
||||
elif os.path.isfile(json_path):
|
||||
return json_path
|
||||
else:
|
||||
return yml_path
|
||||
|
||||
def get_installation_id(self):
|
||||
install_id_filename = os.path.join(self.ensure_data_dir(), "install_id")
|
||||
if not self._installation_id:
|
||||
if os.path.isfile(install_id_filename):
|
||||
with open(install_id_filename, "r") as install_id_file:
|
||||
self._installation_id = install_id_file.read()
|
||||
if not self._installation_id:
|
||||
self._installation_id = base58.b58encode(utils.generate_id())
|
||||
with open(install_id_filename, "w") as install_id_file:
|
||||
install_id_file.write(self._installation_id)
|
||||
return self._installation_id
|
||||
|
||||
def get_session_id(self):
|
||||
return self._session_id
|
||||
|
||||
|
||||
# type: Config
|
||||
settings = None
|
||||
|
||||
|
||||
def get_default_env():
|
||||
env_defaults = {}
|
||||
for k, v in ADJUSTABLE_SETTINGS.iteritems():
|
||||
if len(v) == 3:
|
||||
env_defaults[k] = (v[0], None, v[2])
|
||||
else:
|
||||
env_defaults[k] = (v[0], None)
|
||||
return Env(**env_defaults)
|
||||
|
||||
|
||||
def initialize_settings(load_conf_file=True):
|
||||
global settings
|
||||
if settings is None:
|
||||
settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS,
|
||||
environment=get_default_env())
|
||||
settings.installation_id = settings.get_installation_id()
|
||||
if load_conf_file:
|
||||
settings.load_conf_file_settings()
|
||||
@@ -0,0 +1,111 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
from decimal import Decimal
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityTracker(object):
|
||||
"""
|
||||
Class to track peer counts for known blobs, and to discover new popular blobs
|
||||
|
||||
Attributes:
|
||||
availability (dict): dictionary of peers for known blobs
|
||||
"""
|
||||
|
||||
def __init__(self, blob_manager, peer_finder, dht_node):
|
||||
self.availability = {}
|
||||
self._last_mean_availability = Decimal(0.0)
|
||||
self._blob_manager = blob_manager
|
||||
self._peer_finder = peer_finder
|
||||
self._dht_node = dht_node
|
||||
self._check_popular = LoopingCall(self._update_most_popular)
|
||||
self._check_mine = LoopingCall(self._update_mine)
|
||||
|
||||
def start(self):
|
||||
log.info("Starting %s", self)
|
||||
self._check_popular.start(600)
|
||||
self._check_mine.start(600)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self._check_popular.running:
|
||||
self._check_popular.stop()
|
||||
if self._check_mine.running:
|
||||
self._check_mine.stop()
|
||||
|
||||
def get_blob_availability(self, blob, timeout=None):
|
||||
def _get_peer_count(peers):
|
||||
have_blob = sum(1 for peer in peers if peer.is_available())
|
||||
return {blob: have_blob}
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob, timeout)
|
||||
d.addCallback(_get_peer_count)
|
||||
return d
|
||||
|
||||
def get_availability_for_blobs(self, blobs, timeout=None):
|
||||
dl = [self.get_blob_availability(blob, timeout) for blob in blobs if blob]
|
||||
d = defer.DeferredList(dl)
|
||||
d.addCallback(lambda results: [val for success, val in results if success])
|
||||
return d
|
||||
|
||||
@property
|
||||
def last_mean_availability(self):
|
||||
return max(Decimal(0.01), self._last_mean_availability)
|
||||
|
||||
def _update_peers_for_blob(self, blob):
|
||||
def _save_peer_info(blob_hash, peers):
|
||||
v = {blob_hash: peers}
|
||||
self.availability.update(v)
|
||||
return v
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob)
|
||||
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
|
||||
d.addCallback(lambda peers: _save_peer_info(blob, peers))
|
||||
return d
|
||||
|
||||
def _get_most_popular(self):
|
||||
dl = []
|
||||
for (hash, _) in self._dht_node.get_most_popular_hashes(10):
|
||||
encoded = hash.encode('hex')
|
||||
dl.append(self._update_peers_for_blob(encoded))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
def _update_most_popular(self):
|
||||
d = self._get_most_popular()
|
||||
d.addCallback(lambda _: self._set_mean_peers())
|
||||
|
||||
def _update_mine(self):
|
||||
def _get_peers(blobs):
|
||||
dl = []
|
||||
for hash in blobs:
|
||||
dl.append(self._update_peers_for_blob(hash))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
def sample(blobs):
|
||||
return random.sample(blobs, min(len(blobs), 10))
|
||||
|
||||
start = time.time()
|
||||
log.debug('==> Updating the peers for my blobs')
|
||||
d = self._blob_manager.get_all_verified_blobs()
|
||||
# as far as I can tell, this only is used to set _last_mean_availability
|
||||
# which... seems like a very expensive operation for such little payoff.
|
||||
# so taking a sample should get about the same effect as querying the entire
|
||||
# list of blobs
|
||||
d.addCallback(sample)
|
||||
d.addCallback(_get_peers)
|
||||
d.addCallback(lambda _: self._set_mean_peers())
|
||||
d.addCallback(lambda _: log.debug('<== Done updating peers for my blobs. Took %s seconds',
|
||||
time.time() - start))
|
||||
# although unused, need to return or else the looping call
|
||||
# could overrun on a previous call
|
||||
return d
|
||||
|
||||
def _set_mean_peers(self):
|
||||
num_peers = [len(self.availability[blob]) for blob in self.availability]
|
||||
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
|
||||
self._last_mean_availability = mean
|
||||
@@ -15,4 +15,4 @@ class BlobInfo(object):
|
||||
def __init__(self, blob_hash, blob_num, length):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.length = length
|
||||
@@ -1,14 +1,17 @@
|
||||
import logging
|
||||
import os
|
||||
import leveldb
|
||||
import time
|
||||
import json
|
||||
from twisted.internet import threads, defer, reactor, task
|
||||
import sqlite3
|
||||
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.HashBlob import BlobFile, TempBlob, BlobFileCreator, TempBlobCreator
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.Error import NoSuchBlobError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager(DHTHashSupplier):
|
||||
@@ -20,19 +23,19 @@ class BlobManager(DHTHashSupplier):
|
||||
def setup(self):
|
||||
pass
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length):
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
pass
|
||||
|
||||
def get_blob_creator(self):
|
||||
pass
|
||||
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length):
|
||||
def _make_new_blob(self, blob_hash, length):
|
||||
pass
|
||||
|
||||
def blob_completed(self, blob, next_announce_time=None):
|
||||
pass
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
pass
|
||||
|
||||
def hashes_to_announce(self):
|
||||
@@ -44,12 +47,6 @@ class BlobManager(DHTHashSupplier):
|
||||
def delete_blob(self, blob_hash):
|
||||
pass
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
pass
|
||||
|
||||
def check_consistency(self):
|
||||
pass
|
||||
|
||||
def blob_requested(self, blob_hash):
|
||||
pass
|
||||
|
||||
@@ -62,255 +59,243 @@ class BlobManager(DHTHashSupplier):
|
||||
def blob_paid_for(self, blob_hash, amount):
|
||||
pass
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
pass
|
||||
|
||||
def add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
pass
|
||||
|
||||
def add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
pass
|
||||
|
||||
def _immediate_announce(self, blob_hashes):
|
||||
if self.hash_announcer:
|
||||
return self.hash_announcer.immediate_announce(blob_hashes)
|
||||
|
||||
|
||||
# TODO: Having different managers for different blobs breaks the
|
||||
# abstraction of a HashBlob. Why should the management of blobs
|
||||
# care what kind of Blob it has?
|
||||
class DiskBlobManager(BlobManager):
|
||||
"""This class stores blobs on the hard disk"""
|
||||
def __init__(self, hash_announcer, blob_dir, db_dir):
|
||||
BlobManager.__init__(self, hash_announcer)
|
||||
self.blob_dir = blob_dir
|
||||
self.db_dir = db_dir
|
||||
self.db = None
|
||||
self.db_file = os.path.join(db_dir, "blobs.db")
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', self.db_file, check_same_thread=False)
|
||||
self.blob_type = BlobFile
|
||||
self.blob_creator_type = BlobFileCreator
|
||||
# TODO: consider using an LRU for blobs as there could potentially
|
||||
# be thousands of blobs loaded up, many stale
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
self._next_manage_call = None
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d.addCallback(lambda _: self._manage())
|
||||
return d
|
||||
log.info("Setting up the DiskBlobManager. blob_dir: %s, db_file: %s", str(self.blob_dir),
|
||||
str(self.db_file))
|
||||
yield self._open_db()
|
||||
|
||||
def stop(self):
|
||||
if self._next_manage_call is not None and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
self.db = None
|
||||
log.info("Stopping the DiskBlobManager")
|
||||
self.db_conn.close()
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a blob that is already on the hard disk"""
|
||||
# TODO: if blob.upload_allowed and upload_allowed is False, change upload_allowed in blob and on disk
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a
|
||||
blob that is already on the hard disk
|
||||
"""
|
||||
assert length is None or isinstance(length, int)
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self._make_new_blob(blob_hash, upload_allowed, length)
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
return self.blob_creator_type(self, self.blob_dir)
|
||||
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
|
||||
blob = self.blob_type(self.blob_dir, blob_hash, upload_allowed, length)
|
||||
def _make_new_blob(self, blob_hash, length=None):
|
||||
log.debug('Making a new blob for %s', blob_hash)
|
||||
blob = self.blob_type(self.blob_dir, blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
d = threads.deferToThread(self._completed_blobs, [blob_hash])
|
||||
|
||||
def check_completed(completed_blobs):
|
||||
|
||||
def set_length(length):
|
||||
blob.length = length
|
||||
|
||||
if len(completed_blobs) == 1 and completed_blobs[0] == blob_hash:
|
||||
blob.verified = True
|
||||
inner_d = threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
inner_d.addCallback(set_length)
|
||||
inner_d.addCallback(lambda _: blob)
|
||||
else:
|
||||
inner_d = defer.succeed(blob)
|
||||
return inner_d
|
||||
|
||||
d.addCallback(check_completed)
|
||||
return d
|
||||
return defer.succeed(blob)
|
||||
|
||||
def blob_completed(self, blob, next_announce_time=None):
|
||||
if next_announce_time is None:
|
||||
next_announce_time = time.time()
|
||||
return threads.deferToThread(self._add_completed_blob, blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
next_announce_time = self.get_next_announce_time()
|
||||
d = self._add_completed_blob(blob.blob_hash, blob.length, next_announce_time)
|
||||
d.addCallback(lambda _: self._immediate_announce([blob.blob_hash]))
|
||||
return d
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
return threads.deferToThread(self._completed_blobs, blobs_to_check)
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
return self._completed_blobs(blobhashes_to_check)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_blobs_to_announce, next_announce_time)
|
||||
return self._get_blobs_to_announce()
|
||||
|
||||
def creator_finished(self, blob_creator):
|
||||
logging.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
assert blob_creator.blob_hash is not None
|
||||
assert blob_creator.blob_hash not in self.blobs
|
||||
assert blob_creator.length is not None
|
||||
new_blob = self.blob_type(self.blob_dir, blob_creator.blob_hash, True, blob_creator.length)
|
||||
new_blob.verified = True
|
||||
new_blob = self.blob_type(self.blob_dir, blob_creator.blob_hash, blob_creator.length)
|
||||
self.blobs[blob_creator.blob_hash] = new_blob
|
||||
if self.hash_announcer is not None:
|
||||
self.hash_announcer.immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
else:
|
||||
d = self.blob_completed(new_blob)
|
||||
self._immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = self.get_next_announce_time()
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
return d
|
||||
|
||||
def delete_blobs(self, blob_hashes):
|
||||
for blob_hash in blob_hashes:
|
||||
if not blob_hash in self.blob_hashes_to_delete:
|
||||
self.blob_hashes_to_delete[blob_hash] = False
|
||||
|
||||
def update_all_last_verified_dates(self, timestamp):
|
||||
return threads.deferToThread(self._update_all_last_verified_dates, timestamp)
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
d = threads.deferToThread(self._get_all_verified_blob_hashes)
|
||||
d.addCallback(self.hash_announcer.immediate_announce)
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self._immediate_announce)
|
||||
return d
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
return threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
def check_consistency(self):
|
||||
return threads.deferToThread(self._check_consistency)
|
||||
def add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
d = self._add_blob_to_download_history(blob_hash, host, rate)
|
||||
return d
|
||||
|
||||
def _manage(self):
|
||||
from twisted.internet import reactor
|
||||
def add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
d = self._add_blob_to_upload_history(blob_hash, host, rate)
|
||||
return d
|
||||
|
||||
d = self._delete_blobs_marked_for_deletion()
|
||||
|
||||
def set_next_manage_call():
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def _delete_blobs_marked_for_deletion(self):
|
||||
|
||||
def remove_from_list(b_h):
|
||||
del self.blob_hashes_to_delete[b_h]
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return err
|
||||
|
||||
def delete_from_db(result):
|
||||
b_hs = [r[1] for r in result if r[0] is True]
|
||||
if b_hs:
|
||||
d = threads.deferToThread(self._delete_blobs_from_db, b_hs)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
def delete(blob, b_h):
|
||||
d = blob.delete()
|
||||
d.addCallbacks(lambda _: remove_from_list(b_h), set_not_deleting, errbackArgs=(b_h,))
|
||||
return d
|
||||
|
||||
ds = []
|
||||
for blob_hash, being_deleted in self.blob_hashes_to_delete.items():
|
||||
if being_deleted is False:
|
||||
self.blob_hashes_to_delete[blob_hash] = True
|
||||
d = self.get_blob(blob_hash, True)
|
||||
d.addCallbacks(delete, set_not_deleting, callbackArgs=(blob_hash,), errbackArgs=(blob_hash,))
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
dl.addCallback(delete_from_db)
|
||||
return defer.DeferredList(ds)
|
||||
@defer.inlineCallbacks
|
||||
def delete_blobs(self, blob_hashes):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
try:
|
||||
blob = yield self.get_blob(blob_hash)
|
||||
yield blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
yield self._delete_blobs_from_db(bh_to_delete_from_db)
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "blobs.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
|
||||
def _add_completed_blob(self, blob_hash, length, timestamp, next_announce_time=None):
|
||||
logging.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
if next_announce_time is None:
|
||||
next_announce_time = timestamp
|
||||
self.db.Put(blob_hash, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" last_verified_time real, " +
|
||||
" next_announce_time real)")
|
||||
|
||||
def _completed_blobs(self, blobs_to_check):
|
||||
blobs = []
|
||||
for b in blobs_to_check:
|
||||
if is_valid_blobhash(b):
|
||||
try:
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(b))
|
||||
except KeyError:
|
||||
continue
|
||||
file_path = os.path.join(self.blob_dir, b)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blobs.append(b)
|
||||
return blobs
|
||||
transaction.execute("create table if not exists download (" +
|
||||
" id integer primary key autoincrement, " +
|
||||
" blob text, " +
|
||||
" host text, " +
|
||||
" rate float, " +
|
||||
" ts integer)")
|
||||
|
||||
def _get_blob_length(self, blob):
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
return length
|
||||
transaction.execute("create table if not exists upload (" +
|
||||
" id integer primary key autoincrement, " +
|
||||
" blob text, " +
|
||||
" host text, " +
|
||||
" rate float, " +
|
||||
" ts integer)")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_completed_blob(self, blob_hash, length, next_announce_time):
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
d = self.db_conn.runQuery(
|
||||
"insert into blobs (blob_hash, blob_length, next_announce_time) values (?, ?, ?)",
|
||||
(blob_hash, length, next_announce_time)
|
||||
)
|
||||
d.addErrback(lambda err: err.trap(sqlite3.IntegrityError))
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _completed_blobs(self, blobhashes_to_check):
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
|
||||
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
|
||||
defer.returnValue(blob_hashes)
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_blob_verified_timestamp(self, blob, timestamp):
|
||||
length, old_verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
self.db.Put(blob, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(blob, timestamp))
|
||||
|
||||
def _get_blobs_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
blobs = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(blob_hash, json.dumps((length, verified_time, next_announce_time)))
|
||||
blobs.append(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
return blobs
|
||||
@rerun_if_locked
|
||||
def _get_blobs_to_announce(self):
|
||||
|
||||
def _update_all_last_verified_dates(self, timestamp):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
batch.Put(blob_hash, json.dumps((length, timestamp, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select blob_hash from blobs " +
|
||||
"where next_announce_time < ? and blob_hash is not null",
|
||||
(timestamp,))
|
||||
blobs = [b for b, in r.fetchall()]
|
||||
next_announce_time = self.get_next_announce_time(len(blobs))
|
||||
transaction.execute(
|
||||
"update blobs set next_announce_time = ? where next_announce_time < ?",
|
||||
(next_announce_time, timestamp))
|
||||
log.debug("Got %s blobs to announce, next announce time is in %s seconds",
|
||||
len(blobs), next_announce_time-time.time())
|
||||
return blobs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_blobs_from_db(self, blob_hashes):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash in blob_hashes:
|
||||
batch.Delete(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
|
||||
def _check_consistency(self):
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time < os.path.getctime(file_path):
|
||||
h = get_lbry_hash_obj()
|
||||
len_so_far = 0
|
||||
f = open(file_path)
|
||||
while True:
|
||||
data = f.read(2**12)
|
||||
if not data:
|
||||
break
|
||||
h.update(data)
|
||||
len_so_far += len(data)
|
||||
if len_so_far == length and h.hexdigest() == blob_hash:
|
||||
batch.Put(blob_hash, json.dumps((length, current_time, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
def delete_blobs(transaction):
|
||||
for b in blob_hashes:
|
||||
transaction.execute("delete from blobs where blob_hash = ?", (b,))
|
||||
|
||||
return self.db_conn.runInteraction(delete_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_blob_hashes(self):
|
||||
d = self.db_conn.runQuery("select blob_hash from blobs")
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
blob_hashes = []
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blob_hashes.append(blob_hash)
|
||||
return blob_hashes
|
||||
d = self._get_all_blob_hashes()
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash, in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
ts = int(time.time())
|
||||
d = self.db_conn.runQuery(
|
||||
"insert into download values (null, ?, ?, ?, ?) ",
|
||||
(blob_hash, str(host), float(rate), ts))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
ts = int(time.time())
|
||||
d = self.db_conn.runQuery(
|
||||
"insert into upload values (null, ?, ?, ?, ?) ",
|
||||
(blob_hash, str(host), float(rate), ts))
|
||||
return d
|
||||
|
||||
|
||||
# TODO: Having different managers for different blobs breaks the
|
||||
# abstraction of a HashBlob. Why should the management of blobs
|
||||
# care what kind of Blob it has?
|
||||
class TempBlobManager(BlobManager):
|
||||
"""This class stores blobs in memory"""
|
||||
def __init__(self, hash_announcer):
|
||||
@@ -331,16 +316,16 @@ class TempBlobManager(BlobManager):
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length=None):
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self._make_new_blob(blob_hash, upload_allowed, length)
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
return self.blob_creator_type(self)
|
||||
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
|
||||
blob = self.blob_type(blob_hash, upload_allowed, length)
|
||||
def _make_new_blob(self, blob_hash, length=None):
|
||||
blob = self.blob_type(blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
return defer.succeed(blob)
|
||||
|
||||
@@ -350,14 +335,24 @@ class TempBlobManager(BlobManager):
|
||||
self.blob_next_announces[blob.blob_hash] = next_announce_time
|
||||
return defer.succeed(True)
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
blobs = [b.blob_hash for b in self.blobs.itervalues() if b.blob_hash in blobs_to_check and b.is_validated()]
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
blobs = [
|
||||
b.blob_hash for b in self.blobs.itervalues()
|
||||
if b.blob_hash in blobhashes_to_check and b.is_validated()
|
||||
]
|
||||
return defer.succeed(blobs)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self.completed_blobs(self.blobs)
|
||||
return d
|
||||
|
||||
def hashes_to_announce(self):
|
||||
now = time.time()
|
||||
blobs = [blob_hash for blob_hash, announce_time in self.blob_next_announces.iteritems() if announce_time < now]
|
||||
next_announce_time = now + self.hash_reannounce_time
|
||||
blobs = [
|
||||
blob_hash for blob_hash, announce_time in self.blob_next_announces.iteritems()
|
||||
if announce_time < now
|
||||
]
|
||||
next_announce_time = self.get_next_announce_time(len(blobs))
|
||||
for b in blobs:
|
||||
self.blob_next_announces[b] = next_announce_time
|
||||
return defer.succeed(blobs)
|
||||
@@ -366,17 +361,16 @@ class TempBlobManager(BlobManager):
|
||||
assert blob_creator.blob_hash is not None
|
||||
assert blob_creator.blob_hash not in self.blobs
|
||||
assert blob_creator.length is not None
|
||||
new_blob = self.blob_type(blob_creator.blob_hash, True, blob_creator.length)
|
||||
new_blob.verified = True
|
||||
new_blob = self.blob_type(blob_creator.blob_hash, blob_creator.length)
|
||||
# TODO: change this; its breaks the encapsulation of the
|
||||
# blob. Maybe better would be to have the blob_creator
|
||||
# produce a blob.
|
||||
new_blob.data_buffer = blob_creator.data_buffer
|
||||
new_blob.length = blob_creator.length
|
||||
new_blob._verified = True
|
||||
self.blobs[blob_creator.blob_hash] = new_blob
|
||||
if self.hash_announcer is not None:
|
||||
self.hash_announcer.immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
else:
|
||||
d = self.blob_completed(new_blob)
|
||||
self._immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = self.get_next_announce_time()
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
d.addCallback(lambda _: new_blob)
|
||||
return d
|
||||
|
||||
@@ -385,14 +379,9 @@ class TempBlobManager(BlobManager):
|
||||
if not blob_hash in self.blob_hashes_to_delete:
|
||||
self.blob_hashes_to_delete[blob_hash] = False
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
if blob_hash in self.blobs:
|
||||
if self.blobs[blob_hash].length is not None:
|
||||
return defer.succeed(self.blobs[blob_hash].length)
|
||||
return defer.fail(ValueError("No such blob hash is known"))
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
|
||||
if self.hash_announcer:
|
||||
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
|
||||
|
||||
def _manage(self):
|
||||
from twisted.internet import reactor
|
||||
@@ -400,20 +389,19 @@ class TempBlobManager(BlobManager):
|
||||
d = self._delete_blobs_marked_for_deletion()
|
||||
|
||||
def set_next_manage_call():
|
||||
logging.info("Setting the next manage call in %s", str(self))
|
||||
log.info("Setting the next manage call in %s", str(self))
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def _delete_blobs_marked_for_deletion(self):
|
||||
|
||||
def remove_from_list(b_h):
|
||||
del self.blob_hashes_to_delete[b_h]
|
||||
logging.info("Deleted blob %s", blob_hash)
|
||||
log.info("Deleted blob %s", blob_hash)
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return b_h
|
||||
|
||||
@@ -422,7 +410,7 @@ class TempBlobManager(BlobManager):
|
||||
if being_deleted is False:
|
||||
if blob_hash in self.blobs:
|
||||
self.blob_hashes_to_delete[blob_hash] = True
|
||||
logging.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
log.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
blob = self.blobs[blob_hash]
|
||||
d = blob.delete()
|
||||
|
||||
@@ -432,7 +420,7 @@ class TempBlobManager(BlobManager):
|
||||
ds.append(d)
|
||||
else:
|
||||
remove_from_list(blob_hash)
|
||||
d = defer.fail(Failure(ValueError("No such blob known")))
|
||||
logging.warning("Blob %s cannot be deleted because it is unknown")
|
||||
d = defer.fail(Failure(NoSuchBlobError(blob_hash)))
|
||||
log.warning("Blob %s cannot be deleted because it is unknown")
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds)
|
||||
return defer.DeferredList(ds)
|
||||
@@ -1,6 +1,21 @@
|
||||
class DownloadOptionChoice(object):
|
||||
"""A possible choice that can be picked for some option.
|
||||
|
||||
An option can have one or more choices that can be picked from.
|
||||
"""
|
||||
def __init__(self, value, short_description, long_description, bool_options_description=None):
|
||||
self.value = value
|
||||
self.short_description = short_description
|
||||
self.long_description = long_description
|
||||
self.bool_options_description = bool_options_description
|
||||
|
||||
|
||||
class DownloadOption(object):
|
||||
def __init__(self, option_types, long_description, short_description, default):
|
||||
"""An option for a user to select a value from several different choices."""
|
||||
def __init__(self, option_types, long_description, short_description, default_value,
|
||||
default_value_description):
|
||||
self.option_types = option_types
|
||||
self.long_description = long_description
|
||||
self.short_description = short_description
|
||||
self.default = default
|
||||
self.default_value = default_value
|
||||
self.default_value_description = default_value_description
|
||||
@@ -9,11 +9,23 @@ class DuplicateStreamHashError(Exception):
|
||||
class DownloadCanceledError(Exception):
|
||||
pass
|
||||
|
||||
class DownloadTimeoutError(Exception):
|
||||
def __init__(self, download):
|
||||
Exception.__init__(self, 'Failed to download {} within timeout'.format(download))
|
||||
self.download = download
|
||||
|
||||
class RequestCanceledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NegativeFundsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NullFundsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InsufficientFundsError(Exception):
|
||||
pass
|
||||
|
||||
@@ -22,14 +34,61 @@ class ConnectionClosedBeforeResponseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowed(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidExchangeRateResponse(Exception):
|
||||
def __init__(self, source, reason):
|
||||
Exception.__init__(self, 'Failed to get exchange rate from {}:{}'.format(source, reason))
|
||||
self.source = source
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class UnknownNameError(Exception):
|
||||
def __init__(self, name):
|
||||
Exception.__init__(self, 'Name {} is unknown'.format(name))
|
||||
self.name = name
|
||||
|
||||
|
||||
class UnknownClaimID(Exception):
|
||||
def __init__(self, claim_id):
|
||||
Exception.__init__(self, 'Claim {} is unknown'.format(claim_id))
|
||||
self.claim_id = claim_id
|
||||
|
||||
|
||||
class UnknownURI(Exception):
|
||||
def __init__(self, uri):
|
||||
Exception.__init__(self, 'URI {} cannot be resolved'.format(uri))
|
||||
self.name = uri
|
||||
|
||||
|
||||
class InvalidName(Exception):
|
||||
def __init__(self, name, invalid_characters):
|
||||
self.name = name
|
||||
self.invalid_characters = invalid_characters
|
||||
Exception.__init__(
|
||||
self, 'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
|
||||
|
||||
|
||||
class UnknownStreamTypeError(Exception):
|
||||
def __init__(self, stream_type):
|
||||
self.stream_type = stream_type
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.stream_type)
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidStreamInfoError(Exception):
|
||||
def __init__(self, name):
|
||||
def __init__(self, name, stream_info):
|
||||
msg = '{} has claim with invalid stream info: {}'.format(name, stream_info)
|
||||
Exception.__init__(self, msg)
|
||||
self.name = name
|
||||
self.stream_info = stream_info
|
||||
|
||||
|
||||
class MisbehavingPeerError(Exception):
|
||||
@@ -45,4 +104,34 @@ class NoResponseError(MisbehavingPeerError):
|
||||
|
||||
|
||||
class InvalidResponseError(MisbehavingPeerError):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchBlobError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchStreamHash(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchSDHash(Exception):
|
||||
"""
|
||||
Raised if sd hash is not known
|
||||
"""
|
||||
|
||||
|
||||
class InvalidBlobHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidHeaderError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidAuthenticationToken(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NegotiationError(Exception):
|
||||
pass
|
||||
@@ -11,5 +11,8 @@ class DummyHashAnnouncer(object):
|
||||
def add_supplier(self, *args):
|
||||
pass
|
||||
|
||||
def hash_queue_size(self):
|
||||
return 0
|
||||
|
||||
def immediate_announce(self, *args):
|
||||
pass
|
||||
pass
|
||||
@@ -8,9 +8,13 @@ from twisted.internet import interfaces, defer, threads
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
@@ -45,17 +49,26 @@ class HashBlobWriter(object):
|
||||
self.write_handle = write_handle
|
||||
self.length_getter = length_getter
|
||||
self.finished_cb = finished_cb
|
||||
self.hashsum = get_lbry_hash_obj()
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
|
||||
@property
|
||||
def blob_hash(self):
|
||||
return self._hashsum.hexdigest()
|
||||
|
||||
def write(self, data):
|
||||
self.hashsum.update(data)
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > self.length_getter():
|
||||
self.finished_cb(self, Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (str(self.len_so_far),
|
||||
str(self.length_getter())))))
|
||||
self.finished_cb(
|
||||
self,
|
||||
Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (self.len_so_far,
|
||||
self.length_getter()))))
|
||||
else:
|
||||
if self.write_handle is None:
|
||||
log.debug("Tried to write to a write_handle that was None.")
|
||||
return
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb(self)
|
||||
@@ -69,32 +82,35 @@ class HashBlobWriter(object):
|
||||
class HashBlob(object):
|
||||
"""A chunk of data available on the network which is specified by a hashsum"""
|
||||
|
||||
def __init__(self, blob_hash, upload_allowed, length=None):
|
||||
def __init__(self, blob_hash, length=None):
|
||||
assert is_valid_blobhash(blob_hash)
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.writers = {} # {Peer: writer, finished_deferred}
|
||||
self.finished_deferred = None
|
||||
self.verified = False
|
||||
self.upload_allowed = upload_allowed
|
||||
self._verified = False
|
||||
self.readers = 0
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
# protect verified from being modified by other classes
|
||||
return self._verified
|
||||
|
||||
def set_length(self, length):
|
||||
if self.length is not None and length == self.length:
|
||||
return True
|
||||
if self.length is None and 0 <= length <= BLOB_SIZE:
|
||||
if self.length is None and 0 <= length <= conf.settings['BLOB_SIZE']:
|
||||
self.length = length
|
||||
return True
|
||||
logging.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
|
||||
self.length, length)
|
||||
return False
|
||||
|
||||
def get_length(self):
|
||||
return self.length
|
||||
|
||||
def is_validated(self):
|
||||
if self.verified:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
return bool(self._verified)
|
||||
|
||||
def is_downloading(self):
|
||||
if self.writers:
|
||||
@@ -120,14 +136,15 @@ class HashBlob(object):
|
||||
def writer_finished(self, writer, err=None):
|
||||
|
||||
def fire_finished_deferred():
|
||||
self.verified = True
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
if w == writer:
|
||||
finished_deferred.callback(self)
|
||||
del self.writers[p]
|
||||
return True
|
||||
logging.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
log.warning(
|
||||
"Somehow, the writer that was accepted as being valid was already removed: %s",
|
||||
writer)
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
@@ -141,8 +158,8 @@ class HashBlob(object):
|
||||
w.cancel()
|
||||
|
||||
if err is None:
|
||||
if writer.len_so_far == self.length and writer.hashsum.hexdigest() == self.blob_hash:
|
||||
if self.verified is False:
|
||||
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
|
||||
if self._verified is False:
|
||||
d = self._save_verified_blob(writer)
|
||||
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
|
||||
d.addCallback(lambda _: cancel_other_downloads())
|
||||
@@ -152,7 +169,7 @@ class HashBlob(object):
|
||||
else:
|
||||
err_string = "length vs expected: {0}, {1}, hash vs expected: {2}, {3}"
|
||||
err_string = err_string.format(self.length, writer.len_so_far, self.blob_hash,
|
||||
writer.hashsum.hexdigest())
|
||||
writer.blob_hash)
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(True)
|
||||
else:
|
||||
@@ -184,7 +201,7 @@ class HashBlob(object):
|
||||
return self.blob_hash[:16]
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
return '<{}({})>'.format(self.__class__.__name__, str(self))
|
||||
|
||||
|
||||
class BlobFile(HashBlob):
|
||||
@@ -196,33 +213,42 @@ class BlobFile(HashBlob):
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.setting_verified_blob_lock = threading.Lock()
|
||||
self.moved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
self.set_length(os.path.getsize(self.file_path))
|
||||
# This assumes that the hash of the blob has already been
|
||||
# checked as part of the blob creation process. It might
|
||||
# be worth having a function that checks the actual hash;
|
||||
# its probably too expensive to have that check be part of
|
||||
# this call.
|
||||
self._verified = True
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
if not peer in self.writers:
|
||||
logging.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
write_file = tempfile.NamedTemporaryFile(delete=False, dir=self.blob_dir)
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(write_file, self.get_length, self.writer_finished)
|
||||
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return finished_deferred, writer.write, writer.cancel
|
||||
logging.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
if self.verified is True:
|
||||
if self._verified is True:
|
||||
file_handle = None
|
||||
try:
|
||||
file_handle = open(self.file_path, 'rb')
|
||||
self.readers += 1
|
||||
return file_handle
|
||||
except IOError:
|
||||
log.exception('Failed to open %s', self.file_path)
|
||||
self.close_read_handle(file_handle)
|
||||
return None
|
||||
|
||||
def delete(self):
|
||||
if not self.writers and not self.readers:
|
||||
self.verified = False
|
||||
self._verified = False
|
||||
self.moved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
@@ -232,13 +258,15 @@ class BlobFile(HashBlob):
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("File is currently being read or written and cannot be deleted")))
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
|
||||
def close_read_handle(self, file_handle):
|
||||
if file_handle is not None:
|
||||
@@ -247,7 +275,7 @@ class BlobFile(HashBlob):
|
||||
|
||||
def _close_writer(self, writer):
|
||||
if writer.write_handle is not None:
|
||||
logging.debug("Closing %s", str(self))
|
||||
log.debug("Closing %s", str(self))
|
||||
name = writer.write_handle.name
|
||||
writer.write_handle.close()
|
||||
threads.deferToThread(os.remove, name)
|
||||
@@ -287,17 +315,18 @@ class TempBlob(HashBlob):
|
||||
return None, None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
if self.verified is True:
|
||||
if self._verified is True:
|
||||
return StringIO(self.data_buffer)
|
||||
return None
|
||||
|
||||
def delete(self):
|
||||
if not self.writers and not self.readers:
|
||||
self.verified = False
|
||||
self._verified = False
|
||||
self.data_buffer = ''
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Blob is currently being read or written and cannot be deleted")))
|
||||
return defer.fail(Failure(
|
||||
ValueError("Blob is currently being read or written and cannot be deleted")))
|
||||
|
||||
def close_read_handle(self, file_handle):
|
||||
file_handle.close()
|
||||
@@ -320,7 +349,7 @@ class TempBlob(HashBlob):
|
||||
class HashBlobCreator(object):
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.hashsum = get_lbry_hash_obj()
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.blob_hash = None
|
||||
self.length = None
|
||||
@@ -333,9 +362,8 @@ class HashBlobCreator(object):
|
||||
if self.length == 0:
|
||||
self.blob_hash = None
|
||||
else:
|
||||
self.blob_hash = self.hashsum.hexdigest()
|
||||
self.blob_hash = self._hashsum.hexdigest()
|
||||
d = self._close()
|
||||
|
||||
if self.blob_hash is not None:
|
||||
d.addCallback(lambda _: self.blob_manager.creator_finished(self))
|
||||
d.addCallback(lambda _: self.blob_hash)
|
||||
@@ -344,7 +372,7 @@ class HashBlobCreator(object):
|
||||
return d
|
||||
|
||||
def write(self, data):
|
||||
self.hashsum.update(data)
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
self._write(data)
|
||||
|
||||
@@ -364,16 +392,11 @@ class BlobFileCreator(HashBlobCreator):
|
||||
def _close(self):
|
||||
temp_file_name = self.out_file.name
|
||||
self.out_file.close()
|
||||
|
||||
def change_file_name():
|
||||
shutil.move(temp_file_name, os.path.join(self.blob_dir, self.blob_hash))
|
||||
return True
|
||||
|
||||
if self.blob_hash is not None:
|
||||
d = threads.deferToThread(change_file_name)
|
||||
shutil.move(temp_file_name, os.path.join(self.blob_dir, self.blob_hash))
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
return d
|
||||
os.remove(temp_file_name)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _write(self, data):
|
||||
self.out_file.write(data)
|
||||
@@ -382,10 +405,11 @@ class BlobFileCreator(HashBlobCreator):
|
||||
class TempBlobCreator(HashBlobCreator):
|
||||
def __init__(self, blob_manager):
|
||||
HashBlobCreator.__init__(self, blob_manager)
|
||||
# TODO: use StringIO
|
||||
self.data_buffer = ''
|
||||
|
||||
def _close(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _write(self, data):
|
||||
self.data_buffer += data
|
||||
self.data_buffer += data
|
||||
@@ -1,468 +0,0 @@
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest
|
||||
from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
|
||||
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
|
||||
from twisted.internet import threads, reactor, defer, task
|
||||
from twisted.python.failure import Failure
|
||||
from collections import defaultdict, deque
|
||||
from zope.interface import implements
|
||||
from decimal import Decimal
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
import subprocess
|
||||
import socket
|
||||
import time
|
||||
import os
|
||||
|
||||
|
||||
class ReservedPoints(object):
|
||||
def __init__(self, identifier, amount):
|
||||
self.identifier = identifier
|
||||
self.amount = amount
|
||||
|
||||
|
||||
class LBRYcrdWallet(object):
|
||||
"""This class implements the LBRYWallet interface for the LBRYcrd payment system"""
|
||||
implements(ILBRYWallet)
|
||||
|
||||
def __init__(self, rpc_user, rpc_pass, rpc_url, rpc_port, start_lbrycrdd=False,
|
||||
wallet_dir=None, wallet_conf=None):
|
||||
self.rpc_conn_string = "http://%s:%s@%s:%s" % (rpc_user, rpc_pass, rpc_url, str(rpc_port))
|
||||
self.next_manage_call = None
|
||||
self.wallet_balance = Decimal(0.0)
|
||||
self.total_reserved_points = Decimal(0.0)
|
||||
self.peer_addresses = {} # {Peer: string}
|
||||
self.queued_payments = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.expected_balances = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.current_address_given_to_peer = {} # {Peer: address(string)}
|
||||
self.expected_balance_at_time = deque() # (Peer, address(string), amount(Decimal), time(datetime), count(int),
|
||||
# incremental_amount(float))
|
||||
self.max_expected_payment_time = datetime.timedelta(minutes=3)
|
||||
self.stopped = True
|
||||
self.start_lbrycrdd = start_lbrycrdd
|
||||
self.started_lbrycrdd = False
|
||||
self.wallet_dir = wallet_dir
|
||||
self.wallet_conf = wallet_conf
|
||||
self.lbrycrdd = None
|
||||
self.manage_running = False
|
||||
|
||||
def start(self):
|
||||
|
||||
def make_connection():
|
||||
if self.start_lbrycrdd is True:
|
||||
self._start_daemon()
|
||||
logging.info("Trying to connect to %s", self.rpc_conn_string)
|
||||
self.rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Connected!")
|
||||
|
||||
def start_manage():
|
||||
self.stopped = False
|
||||
self.manage()
|
||||
return True
|
||||
|
||||
d = threads.deferToThread(make_connection)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
self.stopped = True
|
||||
# If self.next_manage_call is None, then manage is currently running or else
|
||||
# start has not been called, so set stopped and do nothing else.
|
||||
if self.next_manage_call is not None:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
|
||||
d = self.manage()
|
||||
if self.start_lbrycrdd is True:
|
||||
d.addBoth(lambda _: self._stop_daemon())
|
||||
return d
|
||||
|
||||
def manage(self):
|
||||
logging.info("Doing manage")
|
||||
self.next_manage_call = None
|
||||
have_set_manage_running = [False]
|
||||
|
||||
def check_if_manage_running():
|
||||
|
||||
d = defer.Deferred()
|
||||
|
||||
def fire_if_not_running():
|
||||
if self.manage_running is False:
|
||||
self.manage_running = True
|
||||
have_set_manage_running[0] = True
|
||||
d.callback(True)
|
||||
else:
|
||||
task.deferLater(reactor, 1, fire_if_not_running)
|
||||
|
||||
fire_if_not_running()
|
||||
return d
|
||||
|
||||
d = check_if_manage_running()
|
||||
|
||||
d.addCallback(lambda _: self._check_expected_balances())
|
||||
|
||||
d.addCallback(lambda _: self._send_payments())
|
||||
|
||||
d.addCallback(lambda _: threads.deferToThread(self._get_wallet_balance))
|
||||
|
||||
def set_wallet_balance(balance):
|
||||
self.wallet_balance = balance
|
||||
|
||||
d.addCallback(set_wallet_balance)
|
||||
|
||||
def set_next_manage_call():
|
||||
if not self.stopped:
|
||||
self.next_manage_call = reactor.callLater(60, self.manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def log_error(err):
|
||||
logging.error("Something went wrong during manage. Error message: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
|
||||
def set_manage_not_running(arg):
|
||||
if have_set_manage_running[0] is True:
|
||||
self.manage_running = False
|
||||
return arg
|
||||
|
||||
d.addBoth(set_manage_not_running)
|
||||
return d
|
||||
|
||||
def get_info_exchanger(self):
|
||||
return LBRYcrdAddressRequester(self)
|
||||
|
||||
def get_wallet_info_query_handler_factory(self):
|
||||
return LBRYcrdAddressQueryHandlerFactory(self)
|
||||
|
||||
def get_balance(self):
|
||||
d = threads.deferToThread(self._get_wallet_balance)
|
||||
return d
|
||||
|
||||
def reserve_points(self, peer, amount):
|
||||
"""
|
||||
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
|
||||
|
||||
@param peer: The peer to which the payment will ultimately be sent
|
||||
|
||||
@param amount: The amount of points to reserve
|
||||
|
||||
@return: A ReservedPoints object which is given to send_points once the service has been rendered
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
#if peer in self.peer_addresses:
|
||||
if self.wallet_balance >= self.total_reserved_points + rounded_amount:
|
||||
self.total_reserved_points += rounded_amount
|
||||
return ReservedPoints(peer, rounded_amount)
|
||||
return None
|
||||
|
||||
def cancel_point_reservation(self, reserved_points):
|
||||
"""
|
||||
Return all of the points that were reserved previously for some ReservedPoints object
|
||||
|
||||
@param reserved_points: ReservedPoints previously returned by reserve_points
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self.total_reserved_points -= reserved_points.amount
|
||||
|
||||
def send_points(self, reserved_points, amount):
|
||||
"""
|
||||
Schedule a payment to be sent to a peer
|
||||
|
||||
@param reserved_points: ReservedPoints object previously returned by reserve_points
|
||||
|
||||
@param amount: amount of points to actually send, must be less than or equal to the
|
||||
amount reserved in reserved_points
|
||||
|
||||
@return: Deferred which fires when the payment has been scheduled
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
peer = reserved_points.identifier
|
||||
assert(rounded_amount <= reserved_points.amount)
|
||||
assert(peer in self.peer_addresses)
|
||||
self.queued_payments[self.peer_addresses[peer]] += rounded_amount
|
||||
# make any unused points available
|
||||
self.total_reserved_points -= (reserved_points.amount - rounded_amount)
|
||||
logging.info("ordering that %s points be sent to %s", str(rounded_amount),
|
||||
str(self.peer_addresses[peer]))
|
||||
peer.update_stats('points_sent', amount)
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_expected_payment(self, peer, amount):
|
||||
"""Increase the number of points expected to be paid by a peer"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
assert(peer in self.current_address_given_to_peer)
|
||||
address = self.current_address_given_to_peer[peer]
|
||||
logging.info("expecting a payment at address %s in the amount of %s", str(address), str(rounded_amount))
|
||||
self.expected_balances[address] += rounded_amount
|
||||
expected_balance = self.expected_balances[address]
|
||||
expected_time = datetime.datetime.now() + self.max_expected_payment_time
|
||||
self.expected_balance_at_time.append((peer, address, expected_balance, expected_time, 0, amount))
|
||||
peer.update_stats('expected_points', amount)
|
||||
|
||||
def update_peer_address(self, peer, address):
|
||||
self.peer_addresses[peer] = address
|
||||
|
||||
def get_new_address_for_peer(self, peer):
|
||||
def set_address_for_peer(address):
|
||||
self.current_address_given_to_peer[peer] = address
|
||||
return address
|
||||
d = threads.deferToThread(self._get_new_address)
|
||||
d.addCallback(set_address_for_peer)
|
||||
return d
|
||||
|
||||
def get_stream_info_for_name(self, name):
|
||||
|
||||
def get_stream_info_from_value(result):
|
||||
r_dict = {}
|
||||
if 'value' in result:
|
||||
value = result['value']
|
||||
try:
|
||||
value_dict = json.loads(value)
|
||||
except ValueError:
|
||||
return Failure(InvalidStreamInfoError(name))
|
||||
if 'stream_hash' in value_dict:
|
||||
r_dict['stream_hash'] = value_dict['stream_hash']
|
||||
if 'name' in value_dict:
|
||||
r_dict['name'] = value_dict['name']
|
||||
if 'description' in value_dict:
|
||||
r_dict['description'] = value_dict['description']
|
||||
return r_dict
|
||||
return Failure(UnknownNameError(name))
|
||||
|
||||
d = threads.deferToThread(self._get_value_for_name, name)
|
||||
d.addCallback(get_stream_info_from_value)
|
||||
return d
|
||||
|
||||
def claim_name(self, stream_hash, name, amount):
|
||||
value = json.dumps({"stream_hash": stream_hash})
|
||||
d = threads.deferToThread(self._claim_name, name, value, amount)
|
||||
return d
|
||||
|
||||
def get_available_balance(self):
|
||||
return float(self.wallet_balance - self.total_reserved_points)
|
||||
|
||||
def get_new_address(self):
|
||||
return threads.deferToThread(self._get_new_address)
|
||||
|
||||
def _start_daemon(self):
|
||||
|
||||
if os.name == "nt":
|
||||
si = subprocess.STARTUPINFO
|
||||
si.dwFlags = subprocess.STARTF_USESHOWWINDOW
|
||||
si.wShowWindow = subprocess.SW_HIDE
|
||||
self.lbrycrdd = subprocess.Popen(["lbrycrdd.exe", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf], startupinfo=si)
|
||||
else:
|
||||
self.lbrycrdd = subprocess.Popen(["./lbrycrdd", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf])
|
||||
self.started_lbrycrdd = True
|
||||
|
||||
tries = 0
|
||||
while tries < 5:
|
||||
try:
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.getinfo()
|
||||
break
|
||||
except (socket.error, JSONRPCException):
|
||||
tries += 1
|
||||
logging.warning("Failed to connect to lbrycrdd.")
|
||||
if tries < 5:
|
||||
time.sleep(2 ** tries)
|
||||
logging.warning("Trying again in %d seconds", 2 ** tries)
|
||||
else:
|
||||
logging.warning("Giving up.")
|
||||
else:
|
||||
self.lbrycrdd.terminate()
|
||||
raise ValueError("Couldn't open lbrycrdd")
|
||||
|
||||
def _stop_daemon(self):
|
||||
if self.lbrycrdd is not None and self.started_lbrycrdd is True:
|
||||
d = threads.deferToThread(self._rpc_stop)
|
||||
return d
|
||||
return defer.succeed(True)
|
||||
|
||||
def _check_expected_balances(self):
|
||||
now = datetime.datetime.now()
|
||||
balances_to_check = []
|
||||
try:
|
||||
while self.expected_balance_at_time[0][3] < now:
|
||||
balances_to_check.append(self.expected_balance_at_time.popleft())
|
||||
except IndexError:
|
||||
pass
|
||||
ds = []
|
||||
for balance_to_check in balances_to_check:
|
||||
d = threads.deferToThread(self._check_expected_balance, balance_to_check)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def handle_checks(results):
|
||||
from future_builtins import zip
|
||||
for balance, (success, result) in zip(balances_to_check, results):
|
||||
peer = balance[0]
|
||||
if success is True:
|
||||
if result is False:
|
||||
if balance[4] <= 1: # first or second strike, give them another chance
|
||||
new_expected_balance = (balance[0],
|
||||
balance[1],
|
||||
balance[2],
|
||||
datetime.datetime.now() + self.max_expected_payment_time,
|
||||
balance[4] + 1,
|
||||
balance[5])
|
||||
self.expected_balance_at_time.append(new_expected_balance)
|
||||
peer.update_score(-5.0)
|
||||
else:
|
||||
peer.update_score(-50.0)
|
||||
else:
|
||||
if balance[4] == 0:
|
||||
peer.update_score(balance[5])
|
||||
peer.update_stats('points_received', balance[5])
|
||||
else:
|
||||
logging.warning("Something went wrong checking a balance. Peer: %s, account: %s,"
|
||||
"expected balance: %s, expected time: %s, count: %s, error: %s",
|
||||
str(balance[0]), str(balance[1]), str(balance[2]), str(balance[3]),
|
||||
str(balance[4]), str(result.getErrorMessage()))
|
||||
|
||||
dl.addCallback(handle_checks)
|
||||
return dl
|
||||
|
||||
def _check_expected_balance(self, expected_balance):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Checking balance of address %s", str(expected_balance[1]))
|
||||
balance = rpc_conn.getreceivedbyaddress(expected_balance[1])
|
||||
logging.debug("received balance: %s", str(balance))
|
||||
logging.debug("expected balance: %s", str(expected_balance[2]))
|
||||
return balance >= expected_balance[2]
|
||||
|
||||
def _send_payments(self):
|
||||
logging.info("Trying to send payments, if there are any to be sent")
|
||||
|
||||
def do_send(payments):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.sendmany("", payments)
|
||||
|
||||
payments_to_send = {}
|
||||
for address, points in self.queued_payments.items():
|
||||
logging.info("Should be sending %s points to %s", str(points), str(address))
|
||||
payments_to_send[address] = float(points)
|
||||
self.total_reserved_points -= points
|
||||
self.wallet_balance -= points
|
||||
del self.queued_payments[address]
|
||||
if payments_to_send:
|
||||
logging.info("Creating a transaction with outputs %s", str(payments_to_send))
|
||||
return threads.deferToThread(do_send, payments_to_send)
|
||||
logging.info("There were no payments to send")
|
||||
return defer.succeed(True)
|
||||
|
||||
def _get_wallet_balance(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getbalance("")
|
||||
|
||||
def _get_new_address(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getnewaddress()
|
||||
|
||||
def _get_value_for_name(self, name):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getvalueforname(name)
|
||||
|
||||
def _claim_name(self, name, value, amount):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return str(rpc_conn.claimname(name, value, amount))
|
||||
|
||||
def _rpc_stop(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.stop()
|
||||
self.lbrycrdd.wait()
|
||||
|
||||
|
||||
class LBRYcrdAddressRequester(object):
|
||||
implements([IRequestCreator])
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self._protocols = []
|
||||
|
||||
######### IRequestCreator #########
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
|
||||
if not protocol in self._protocols:
|
||||
r = ClientRequest({'lbrycrd_address': True}, 'lbrycrd_address')
|
||||
d = protocol.add_request(r)
|
||||
d.addCallback(self._handle_address_response, peer, r, protocol)
|
||||
d.addErrback(self._request_failed, peer)
|
||||
self._protocols.append(protocol)
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
return defer.succeed(False)
|
||||
|
||||
######### internal calls #########
|
||||
|
||||
def _handle_address_response(self, response_dict, peer, request, protocol):
|
||||
assert request.response_identifier in response_dict, \
|
||||
"Expected %s in dict but did not get it" % request.response_identifier
|
||||
assert protocol in self._protocols, "Responding protocol is not in our list of protocols"
|
||||
address = response_dict[request.response_identifier]
|
||||
self.wallet.update_peer_address(peer, address)
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = LBRYcrdAddressQueryHandler(self.wallet)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'lbrycrd_address'
|
||||
|
||||
def get_description(self):
|
||||
return "LBRYcrd Address - an address for receiving payments via LBRYcrd"
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandler(object):
|
||||
implements(IQueryHandler)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = ['lbrycrd_address']
|
||||
self.address = None
|
||||
self.peer = None
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
self.peer = peer
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
|
||||
def create_response(address):
|
||||
self.address = address
|
||||
fields = {'lbrycrd_address': address}
|
||||
return fields
|
||||
|
||||
if self.query_identifiers[0] in queries:
|
||||
d = self.wallet.get_new_address_for_peer(self.peer)
|
||||
d.addCallback(create_response)
|
||||
return d
|
||||
if self.address is None:
|
||||
logging.warning("Expected a request for an address, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive an address request")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
@@ -0,0 +1,62 @@
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class Offer(object):
|
||||
"""A rate offer to download blobs from a host."""
|
||||
|
||||
RATE_ACCEPTED = "RATE_ACCEPTED"
|
||||
RATE_TOO_LOW = "RATE_TOO_LOW"
|
||||
RATE_UNSET = "RATE_UNSET"
|
||||
|
||||
def __init__(self, offer):
|
||||
self._state = None
|
||||
self.rate = None
|
||||
if isinstance(offer, Decimal):
|
||||
self.rate = round(offer, 5)
|
||||
elif isinstance(offer, float):
|
||||
self.rate = round(Decimal(offer), 5)
|
||||
if self.rate is None or self.rate < Decimal(0.0):
|
||||
self.unset()
|
||||
|
||||
@property
|
||||
def is_accepted(self):
|
||||
return self._state is Offer.RATE_ACCEPTED
|
||||
|
||||
@property
|
||||
def is_too_low(self):
|
||||
return self._state is Offer.RATE_TOO_LOW
|
||||
|
||||
@property
|
||||
def is_unset(self):
|
||||
return self._state is Offer.RATE_UNSET
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
if self.is_accepted:
|
||||
return Offer.RATE_ACCEPTED
|
||||
elif self.is_too_low:
|
||||
return Offer.RATE_TOO_LOW
|
||||
elif self.is_unset:
|
||||
return Offer.RATE_UNSET
|
||||
return None
|
||||
|
||||
def accept(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_ACCEPTED
|
||||
|
||||
def reject(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_TOO_LOW
|
||||
|
||||
def unset(self):
|
||||
self._state = Offer.RATE_UNSET
|
||||
|
||||
def handle(self, reply_message):
|
||||
if reply_message == Offer.RATE_TOO_LOW:
|
||||
self.reject()
|
||||
elif reply_message == Offer.RATE_ACCEPTED:
|
||||
self.accept()
|
||||
elif reply_message == Offer.RATE_UNSET:
|
||||
self.unset()
|
||||
else:
|
||||
raise Exception("Unknown offer reply %s" % str(reply_message))
|
||||
@@ -1,24 +1,27 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import leveldb
|
||||
import os
|
||||
import unqlite
|
||||
import time
|
||||
from Crypto.Hash import SHA512
|
||||
from Crypto.PublicKey import RSA
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet
|
||||
from lbrynet.pointtraderclient import pointtraderclient
|
||||
from twisted.internet import defer, threads
|
||||
from zope.interface import implements
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.LBRYcrdWallet import ReservedPoints
|
||||
from lbrynet.core.Wallet import ReservedPoints
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PTCWallet(object):
|
||||
"""This class sends payments to peers and also ensures that expected payments are received.
|
||||
This class is only intended to be used for testing."""
|
||||
implements(ILBRYWallet)
|
||||
implements(IWallet)
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
@@ -71,7 +74,7 @@ class PTCWallet(object):
|
||||
|
||||
def save_key(success, private_key):
|
||||
if success is True:
|
||||
threads.deferToThread(self.save_private_key, private_key.exportKey())
|
||||
self._save_private_key(private_key.exportKey())
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -95,8 +98,8 @@ class PTCWallet(object):
|
||||
def start_manage():
|
||||
self.manage()
|
||||
return True
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d.addCallback(lambda _: threads.deferToThread(self.get_wallet_private_key))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._get_wallet_private_key())
|
||||
d.addCallback(ensure_private_key_exists)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
@@ -108,14 +111,16 @@ class PTCWallet(object):
|
||||
return PointTraderKeyQueryHandlerFactory(self)
|
||||
|
||||
def reserve_points(self, peer, amount):
|
||||
"""
|
||||
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
|
||||
"""Ensure a certain amount of points are available to be sent as
|
||||
payment, before the service is rendered
|
||||
|
||||
@param peer: The peer to which the payment will ultimately be sent
|
||||
|
||||
@param amount: The amount of points to reserve
|
||||
|
||||
@return: A ReservedPoints object which is given to send_points once the service has been rendered
|
||||
@return: A ReservedPoints object which is given to send_points
|
||||
once the service has been rendered
|
||||
|
||||
"""
|
||||
if self.wallet_balance >= self.total_reserved_points + amount:
|
||||
self.total_reserved_points += amount
|
||||
@@ -154,14 +159,15 @@ class PTCWallet(object):
|
||||
ds = []
|
||||
for peer, points in self.queued_payments.items():
|
||||
if peer in self.peer_pub_keys:
|
||||
d = pointtraderclient.send_points(self.private_key, self.peer_pub_keys[peer], points)
|
||||
d = pointtraderclient.send_points(
|
||||
self.private_key, self.peer_pub_keys[peer], points)
|
||||
self.wallet_balance -= points
|
||||
self.total_reserved_points -= points
|
||||
ds.append(d)
|
||||
del self.queued_payments[peer]
|
||||
else:
|
||||
logging.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def get_balance(self):
|
||||
@@ -205,22 +211,30 @@ class PTCWallet(object):
|
||||
min_expected_balance = sum([a[0] for a in expected_payments if a[1] < expected_cutoff])
|
||||
received_balance = 0
|
||||
if self.peer_pub_keys[peer] in self.received_payments:
|
||||
received_balance = sum([a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
|
||||
received_balance = sum([
|
||||
a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
|
||||
if min_expected_balance > received_balance:
|
||||
logging.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
log.warning(
|
||||
"Account in bad standing: %s (pub_key: %s), expected amount = %s, "
|
||||
"received_amount = %s",
|
||||
peer, self.peer_pub_keys[peer], min_expected_balance, received_balance)
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
def open_db():
|
||||
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
return threads.deferToThread(open_db)
|
||||
|
||||
def save_private_key(self, private_key):
|
||||
self.db.Put("private_key", private_key)
|
||||
def _save_private_key(self, private_key):
|
||||
def save_key():
|
||||
self.db['private_key'] = private_key
|
||||
return threads.deferToThread(save_key)
|
||||
|
||||
def get_wallet_private_key(self):
|
||||
try:
|
||||
return self.db.Get("private_key")
|
||||
except KeyError:
|
||||
def _get_wallet_private_key(self):
|
||||
def get_key():
|
||||
if 'private_key' in self.db:
|
||||
return self.db['private_key']
|
||||
return None
|
||||
return threads.deferToThread(get_key)
|
||||
|
||||
|
||||
class PointTraderKeyExchanger(object):
|
||||
@@ -256,9 +270,9 @@ class PointTraderKeyExchanger(object):
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
return err
|
||||
|
||||
|
||||
class PointTraderKeyQueryHandlerFactory(object):
|
||||
@@ -277,7 +291,8 @@ class PointTraderKeyQueryHandlerFactory(object):
|
||||
return 'public_key'
|
||||
|
||||
def get_description(self):
|
||||
return "Point Trader Address - an address for receiving payments on the point trader testing network"
|
||||
return ("Point Trader Address - an address for receiving payments on the "
|
||||
"point trader testing network")
|
||||
|
||||
|
||||
class PointTraderKeyQueryHandler(object):
|
||||
@@ -301,15 +316,15 @@ class PointTraderKeyQueryHandler(object):
|
||||
try:
|
||||
RSA.importKey(new_encoded_pub_key)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
logging.warning("Client sent an invalid public key.")
|
||||
log.warning("Client sent an invalid public key.")
|
||||
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
|
||||
self.public_key = new_encoded_pub_key
|
||||
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
|
||||
logging.debug("Received the client's public key: %s", str(self.public_key))
|
||||
log.debug("Received the client's public key: %s", str(self.public_key))
|
||||
fields = {'public_key': self.wallet.encoded_public_key}
|
||||
return defer.succeed(fields)
|
||||
if self.public_key is None:
|
||||
logging.warning("Expected a public key, but did not receive one")
|
||||
log.warning("Expected a public key, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
return defer.succeed({})
|
||||
@@ -1,6 +1,13 @@
|
||||
from lbrynet.core.Strategy import get_default_strategy, OnlyFreeStrategy
|
||||
from lbrynet import conf
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class BasePaymentRateManager(object):
|
||||
def __init__(self, rate):
|
||||
self.min_blob_data_payment_rate = rate
|
||||
def __init__(self, rate=None, info_rate=None):
|
||||
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
|
||||
self.min_blob_info_payment_rate = (
|
||||
info_rate if info_rate is not None else conf.settings['min_info_rate'])
|
||||
|
||||
|
||||
class PaymentRateManager(object):
|
||||
@@ -26,4 +33,95 @@ class PaymentRateManager(object):
|
||||
return self.min_blob_data_payment_rate
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
self.points_paid += amount
|
||||
|
||||
|
||||
class NegotiatedPaymentRateManager(object):
|
||||
def __init__(self, base, availability_tracker, generous=None):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
@param availability_tracker: a BlobAvailabilityTracker
|
||||
@param rate: the min blob data payment rate
|
||||
"""
|
||||
|
||||
self.base = base
|
||||
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
|
||||
self.points_paid = 0.0
|
||||
self.blob_tracker = availability_tracker
|
||||
self.generous = generous if generous is not None else conf.settings['is_generous_host']
|
||||
self.strategy = get_default_strategy(self.blob_tracker,
|
||||
base_price=self.base.min_blob_data_payment_rate,
|
||||
is_generous=generous)
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
return (offer.is_too_low and
|
||||
round(Decimal.from_float(offer.rate), 5) >= round(self.strategy.max_rate, 5))
|
||||
return False
|
||||
|
||||
|
||||
class OnlyFreePaymentsManager(object):
|
||||
def __init__(self, **kwargs):
|
||||
"""
|
||||
A payment rate manager that will only ever accept and offer a rate of 0.0,
|
||||
Used for testing
|
||||
"""
|
||||
|
||||
self.base = BasePaymentRateManager(0.0, 0.0)
|
||||
self.points_paid = 0.0
|
||||
self.generous = True
|
||||
self.strategy = OnlyFreeStrategy()
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
if offer.rate > 0.0:
|
||||
return True
|
||||
return False
|
||||
@@ -1,19 +1,23 @@
|
||||
from collections import defaultdict
|
||||
import datetime
|
||||
from collections import defaultdict
|
||||
from lbrynet.core import utils
|
||||
|
||||
|
||||
# Do not create this object except through PeerManager
|
||||
class Peer(object):
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
# If a peer is reported down, we wait till this time till reattempting connection
|
||||
self.attempt_connection_at = None
|
||||
# Number of times this Peer has been reported to be down, resets to 0 when it is up
|
||||
self.down_count = 0
|
||||
# Number of succesful connections (with full protocol completion) to this peer
|
||||
self.success_count = 0
|
||||
self.score = 0
|
||||
self.stats = defaultdict(float) # {string stat_type, float count}
|
||||
|
||||
def is_available(self):
|
||||
if (self.attempt_connection_at is None or
|
||||
datetime.datetime.today() > self.attempt_connection_at):
|
||||
if self.attempt_connection_at is None or utils.today() > self.attempt_connection_at:
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -21,10 +25,13 @@ class Peer(object):
|
||||
self.down_count = 0
|
||||
self.attempt_connection_at = None
|
||||
|
||||
def report_success(self):
|
||||
self.success_count += 1
|
||||
|
||||
def report_down(self):
|
||||
self.down_count += 1
|
||||
timeout_time = datetime.timedelta(seconds=60 * self.down_count)
|
||||
self.attempt_connection_at = datetime.datetime.today() + timeout_time
|
||||
self.attempt_connection_at = utils.today() + timeout_time
|
||||
|
||||
def update_score(self, score_change):
|
||||
self.score += score_change
|
||||
@@ -33,4 +40,7 @@ class Peer(object):
|
||||
self.stats[stat_type] += count
|
||||
|
||||
def __str__(self):
|
||||
return self.host + ":" + str(self.port)
|
||||
return '{}:{}'.format(self.host, self.port)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Peer({!r}, {!r})'.format(self.host, self.port)
|
||||
@@ -16,4 +16,4 @@ class DummyPeerFinder(object):
|
||||
return defer.succeed([])
|
||||
|
||||
def get_most_popular_hashes(self, num_to_return):
|
||||
return []
|
||||
return []
|
||||
@@ -11,4 +11,4 @@ class PeerManager(object):
|
||||
return peer
|
||||
peer = Peer(host, port)
|
||||
self.peers.append(peer)
|
||||
return peer
|
||||
return peer
|
||||
@@ -0,0 +1,62 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.interfaces import IBlobPriceModel
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
def get_default_price_model(blob_tracker, base_price, **kwargs):
|
||||
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
|
||||
|
||||
|
||||
class ZeroPrice(object):
|
||||
def __init__(self):
|
||||
self.base_price = 0.0
|
||||
|
||||
def calculate_price(self, blob):
|
||||
return 0.0
|
||||
|
||||
|
||||
class MeanAvailabilityWeightedPrice(object):
|
||||
"""Calculate mean-blob-availability and stream-position weighted price for a blob
|
||||
|
||||
Attributes:
|
||||
base_price (float): base price
|
||||
alpha (float): constant, > 0.0 and <= 1.0, used to more highly
|
||||
value blobs at the beginning of a stream.
|
||||
alpha defaults to 1.0, which has a null effect
|
||||
blob_tracker (BlobAvailabilityTracker): blob availability tracker
|
||||
|
||||
"""
|
||||
implementer(IBlobPriceModel)
|
||||
|
||||
def __init__(self, tracker, base_price=None, alpha=1.0):
|
||||
self.blob_tracker = tracker
|
||||
base_price = base_price if base_price is not None else conf.settings['data_rate']
|
||||
self.base_price = Decimal(base_price)
|
||||
self.alpha = Decimal(alpha)
|
||||
|
||||
def calculate_price(self, blob):
|
||||
mean_availability = self.blob_tracker.last_mean_availability
|
||||
availability = self.blob_tracker.availability.get(blob, [])
|
||||
index = 0 # blob.index
|
||||
availability_mult = self._get_availability_multiplier(mean_availability, availability)
|
||||
price = self.base_price * availability_mult / self._frontload(index)
|
||||
return round(price, 5)
|
||||
|
||||
def _get_availability_multiplier(self, mean_availability, availability):
|
||||
return Decimal(max(1, mean_availability) / Decimal(max(1, len(availability))))
|
||||
|
||||
def _frontload(self, index):
|
||||
"""Get front-load multiplier, used to weight prices of blobs in a
|
||||
stream towards the front of the stream.
|
||||
|
||||
At index 0, returns 1.0
|
||||
As index increases, return value approaches 2.0
|
||||
|
||||
@param index: blob position in stream
|
||||
@return: front-load multiplier
|
||||
|
||||
"""
|
||||
|
||||
return Decimal(2.0) - (self.alpha ** index)
|
||||
@@ -1,5 +1,11 @@
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IRateLimiter
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyRateLimiter(object):
|
||||
@@ -10,22 +16,20 @@ class DummyRateLimiter(object):
|
||||
self.total_ul_bytes = 0
|
||||
self.target_dl = 0
|
||||
self.target_ul = 0
|
||||
self.ul_delay = 0.00
|
||||
self.dl_delay = 0.00
|
||||
self.next_tick = None
|
||||
self.tick_call = None
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(1)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
pass
|
||||
@@ -33,12 +37,6 @@ class DummyRateLimiter(object):
|
||||
def set_ul_limit(self, limit):
|
||||
pass
|
||||
|
||||
def ul_wait_time(self):
|
||||
return self.ul_delay
|
||||
|
||||
def dl_wait_time(self):
|
||||
return self.dl_delay
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
@@ -58,66 +56,34 @@ class RateLimiter(object):
|
||||
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
|
||||
self.max_dl_bytes = max_dl_bytes
|
||||
self.max_ul_bytes = max_ul_bytes
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.next_tick = None
|
||||
self.next_unthrottle_dl = None
|
||||
self.next_unthrottle_ul = None
|
||||
|
||||
self.next_dl_check = None
|
||||
self.next_ul_check = None
|
||||
|
||||
self.dl_check_interval = 1.0
|
||||
self.ul_check_interval = 1.0
|
||||
self.tick_call = None
|
||||
self.tick_interval = 0.1
|
||||
|
||||
self.dl_throttled = False
|
||||
self.ul_throttled = False
|
||||
|
||||
self.protocols = []
|
||||
|
||||
def start(self):
|
||||
log.info("Starting %s", self)
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(self.tick_interval)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
# happens once per second
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
if self.max_dl_bytes is not None:
|
||||
if self.dl_bytes_this_second == 0:
|
||||
self.dl_check_interval = 1.0
|
||||
else:
|
||||
self.dl_check_interval = min(1.0, self.dl_check_interval *
|
||||
self.max_dl_bytes / self.dl_bytes_this_second)
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
if self.max_ul_bytes is not None:
|
||||
if self.ul_bytes_this_second == 0:
|
||||
self.ul_check_interval = 1.0
|
||||
else:
|
||||
self.ul_check_interval = min(1.0, self.ul_check_interval *
|
||||
self.max_ul_bytes / self.ul_bytes_this_second)
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.unthrottle_dl()
|
||||
self.unthrottle_ul()
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
log.info("Stopping %s", self)
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
self.max_dl_bytes = limit
|
||||
@@ -128,28 +94,18 @@ class RateLimiter(object):
|
||||
#throttling
|
||||
|
||||
def check_dl(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_dl_check = None
|
||||
|
||||
if self.dl_bytes_this_second > self.max_dl_bytes:
|
||||
self.throttle_dl()
|
||||
else:
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
self.dl_check_interval = min(self.dl_check_interval * 2, 1.0)
|
||||
need_throttle = (self.max_dl_bytes is not None and
|
||||
self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_dl)
|
||||
|
||||
def check_ul(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_ul_check = None
|
||||
|
||||
if self.ul_bytes_this_second > self.max_ul_bytes:
|
||||
self.throttle_ul()
|
||||
else:
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.ul_check_interval = min(self.ul_check_interval * 2, 1.0)
|
||||
need_throttle = (self.max_ul_bytes is not None and
|
||||
self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_ul)
|
||||
|
||||
def throttle_dl(self):
|
||||
if self.dl_throttled is False:
|
||||
@@ -175,23 +131,17 @@ class RateLimiter(object):
|
||||
protocol.unthrottle_upload()
|
||||
self.ul_throttled = False
|
||||
|
||||
#deprecated
|
||||
|
||||
def ul_wait_time(self):
|
||||
return 0
|
||||
|
||||
def dl_wait_time(self):
|
||||
return 0
|
||||
|
||||
#called by protocols
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.dl_bytes_this_interval += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
self.check_dl()
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_second += num_bytes
|
||||
self.ul_bytes_this_interval += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
self.check_ul()
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
if protocol not in self.protocols:
|
||||
@@ -203,4 +153,4 @@ class RateLimiter(object):
|
||||
|
||||
def unregister_protocol(self, protocol):
|
||||
if protocol in self.protocols:
|
||||
self.protocols.remove(protocol)
|
||||
self.protocols.remove(protocol)
|
||||
@@ -1,6 +1,5 @@
|
||||
import logging
|
||||
import miniupnpc
|
||||
from lbrynet.core.PTCWallet import PTCWallet
|
||||
from lbrynet.core.BlobManager import DiskBlobManager, TempBlobManager
|
||||
from lbrynet.dht import node
|
||||
from lbrynet.core.PeerManager import PeerManager
|
||||
@@ -9,65 +8,96 @@ from lbrynet.core.client.DHTPeerFinder import DHTPeerFinder
|
||||
from lbrynet.core.HashAnnouncer import DummyHashAnnouncer
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager, NegotiatedPaymentRateManager
|
||||
from lbrynet.core.BlobAvailability import BlobAvailabilityTracker
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
class LBRYSession(object):
|
||||
"""This class manages all important services common to any application that uses the network:
|
||||
the hash announcer, which informs other peers that this peer is associated with some hash. Usually,
|
||||
this means this peer has a blob identified by the hash in question, but it can be used for other
|
||||
purposes.
|
||||
the peer finder, which finds peers that are associated with some hash.
|
||||
the blob manager, which keeps track of which blobs have been downloaded and provides access to them,
|
||||
the rate limiter, which attempts to ensure download and upload rates stay below a set maximum,
|
||||
and upnp, which opens holes in compatible firewalls so that remote peers can connect to this peer."""
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None, lbryid=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None, hash_announcer=None,
|
||||
blob_dir=None, blob_manager=None, peer_port=None, use_upnp=True,
|
||||
rate_limiter=None, wallet=None):
|
||||
"""
|
||||
@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
class Session(object):
|
||||
"""This class manages all important services common to any application that uses the network.
|
||||
|
||||
the hash announcer, which informs other peers that this peer is
|
||||
associated with some hash. Usually, this means this peer has a
|
||||
blob identified by the hash in question, but it can be used for
|
||||
other purposes.
|
||||
|
||||
the peer finder, which finds peers that are associated with some
|
||||
hash.
|
||||
|
||||
the blob manager, which keeps track of which blobs have been
|
||||
downloaded and provides access to them,
|
||||
|
||||
the rate limiter, which attempts to ensure download and upload
|
||||
rates stay below a set maximum
|
||||
|
||||
upnp, which opens holes in compatible firewalls so that remote
|
||||
peers can connect to this peer.
|
||||
"""
|
||||
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None,
|
||||
lbryid=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None,
|
||||
hash_announcer=None, blob_dir=None,
|
||||
blob_manager=None, peer_port=None, use_upnp=True,
|
||||
rate_limiter=None, wallet=None,
|
||||
dht_node_class=node.Node, blob_tracker_class=None,
|
||||
payment_rate_manager_class=None, is_generous=True):
|
||||
"""@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
@param db_dir: The directory in which levelDB files should be stored
|
||||
|
||||
@param lbryid: The unique ID of this node
|
||||
|
||||
@param peer_manager: An object which keeps track of all known peers. If None, a PeerManager will be created
|
||||
@param peer_manager: An object which keeps track of all known
|
||||
peers. If None, a PeerManager will be created
|
||||
|
||||
@param dht_node_port: The port on which the dht node should listen for incoming connections
|
||||
@param dht_node_port: The port on which the dht node should
|
||||
listen for incoming connections
|
||||
|
||||
@param known_dht_nodes: A list of nodes which the dht node should use to bootstrap into the dht
|
||||
@param known_dht_nodes: A list of nodes which the dht node
|
||||
should use to bootstrap into the dht
|
||||
|
||||
@param peer_finder: An object which is used to look up peers that are associated with some hash. If None,
|
||||
a DHTPeerFinder will be used, which looks for peers in the distributed hash table.
|
||||
@param peer_finder: An object which is used to look up peers
|
||||
that are associated with some hash. If None, a
|
||||
DHTPeerFinder will be used, which looks for peers in the
|
||||
distributed hash table.
|
||||
|
||||
@param hash_announcer: An object which announces to other peers that this peer is associated with some hash.
|
||||
If None, and peer_port is not None, a DHTHashAnnouncer will be used. If None and
|
||||
peer_port is None, a DummyHashAnnouncer will be used, which will not actually announce
|
||||
anything.
|
||||
@param hash_announcer: An object which announces to other
|
||||
peers that this peer is associated with some hash. If
|
||||
None, and peer_port is not None, a DHTHashAnnouncer will
|
||||
be used. If None and peer_port is None, a
|
||||
DummyHashAnnouncer will be used, which will not actually
|
||||
announce anything.
|
||||
|
||||
@param blob_dir: The directory in which blobs will be stored. If None and blob_manager is None, blobs will
|
||||
be stored in memory only.
|
||||
@param blob_dir: The directory in which blobs will be
|
||||
stored. If None and blob_manager is None, blobs will be
|
||||
stored in memory only.
|
||||
|
||||
@param blob_manager: An object which keeps track of downloaded blobs and provides access to them. If None,
|
||||
and blob_dir is not None, a DiskBlobManager will be used, with the given blob_dir.
|
||||
If None and blob_dir is None, a TempBlobManager will be used, which stores blobs in
|
||||
memory only.
|
||||
@param blob_manager: An object which keeps track of downloaded
|
||||
blobs and provides access to them. If None, and blob_dir
|
||||
is not None, a DiskBlobManager will be used, with the
|
||||
given blob_dir. If None and blob_dir is None, a
|
||||
TempBlobManager will be used, which stores blobs in memory
|
||||
only.
|
||||
|
||||
@param peer_port: The port on which other peers should connect to this peer
|
||||
@param peer_port: The port on which other peers should connect
|
||||
to this peer
|
||||
|
||||
@param use_upnp: Whether or not to try to open a hole in the firewall so that outside peers can connect to
|
||||
this peer's peer_port and dht_node_port
|
||||
@param use_upnp: Whether or not to try to open a hole in the
|
||||
firewall so that outside peers can connect to this peer's
|
||||
peer_port and dht_node_port
|
||||
|
||||
@param rate_limiter: An object which keeps track of the amount of data transferred to and from this peer,
|
||||
and can limit that rate if desired
|
||||
@param rate_limiter: An object which keeps track of the amount
|
||||
of data transferred to and from this peer, and can limit
|
||||
that rate if desired
|
||||
|
||||
@param wallet: An object which will be used to keep track of expected payments and which will pay peers.
|
||||
If None, a wallet which uses the Point Trader system will be used, which is meant for testing
|
||||
only
|
||||
@param wallet: An object which will be used to keep track of
|
||||
expected payments and which will pay peers. If None, a
|
||||
wallet which uses the Point Trader system will be used,
|
||||
which is meant for testing only
|
||||
|
||||
@return:
|
||||
"""
|
||||
self.db_dir = db_dir
|
||||
|
||||
@@ -85,6 +115,9 @@ class LBRYSession(object):
|
||||
self.blob_dir = blob_dir
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
self.blob_tracker = None
|
||||
self.blob_tracker_class = blob_tracker_class or BlobAvailabilityTracker
|
||||
|
||||
self.peer_port = peer_port
|
||||
|
||||
self.use_upnp = use_upnp
|
||||
@@ -92,24 +125,28 @@ class LBRYSession(object):
|
||||
self.rate_limiter = rate_limiter
|
||||
|
||||
self.external_ip = '127.0.0.1'
|
||||
self.upnp_handler = None
|
||||
self.upnp_redirects_set = False
|
||||
|
||||
self.upnp_redirects = []
|
||||
|
||||
self.wallet = wallet
|
||||
|
||||
self.dht_node_class = dht_node_class
|
||||
self.dht_node = None
|
||||
|
||||
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
|
||||
self.payment_rate_manager = None
|
||||
self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager
|
||||
self.is_generous = is_generous
|
||||
|
||||
def setup(self):
|
||||
"""Create the blob directory and database if necessary, start all desired services"""
|
||||
|
||||
logging.debug("Setting up the lbry session")
|
||||
log.debug("Setting up the lbry session")
|
||||
|
||||
if self.lbryid is None:
|
||||
self.lbryid = generate_id()
|
||||
|
||||
if self.wallet is None:
|
||||
from lbrynet.core.PTCWallet import PTCWallet
|
||||
self.wallet = PTCWallet(self.db_dir)
|
||||
|
||||
if self.peer_manager is None:
|
||||
@@ -124,7 +161,7 @@ class LBRYSession(object):
|
||||
d.addCallback(lambda _: self._setup_dht())
|
||||
else:
|
||||
if self.hash_announcer is None and self.peer_port is not None:
|
||||
logging.warning("The server has no way to advertise its available blobs.")
|
||||
log.warning("The server has no way to advertise its available blobs.")
|
||||
self.hash_announcer = DummyHashAnnouncer()
|
||||
|
||||
d.addCallback(lambda _: self._setup_other_components())
|
||||
@@ -132,44 +169,78 @@ class LBRYSession(object):
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop all services"""
|
||||
log.info('Shutting down %s', self)
|
||||
ds = []
|
||||
if self.blob_tracker is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_tracker.stop))
|
||||
if self.dht_node is not None:
|
||||
ds.append(defer.maybeDeferred(self.dht_node.stop))
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.upnp_redirects_set is True:
|
||||
if self.rate_limiter is not None:
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
if self.peer_finder is not None:
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
if self.hash_announcer is not None:
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
if self.wallet is not None:
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.use_upnp is True:
|
||||
ds.append(defer.maybeDeferred(self._unset_upnp))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _try_upnp(self):
|
||||
|
||||
logging.debug("In _try_upnp")
|
||||
log.debug("In _try_upnp")
|
||||
|
||||
def threaded_try_upnp():
|
||||
if self.use_upnp is False:
|
||||
logging.debug("Not using upnp")
|
||||
log.debug("Not using upnp")
|
||||
return False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
self.upnp_handler = u
|
||||
u.selectigd()
|
||||
external_ip = u.externalipaddress()
|
||||
if external_ip != '0.0.0.0':
|
||||
self.external_ip = external_ip
|
||||
if self.peer_port is not None:
|
||||
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
|
||||
if u.getspecificportmapping(self.peer_port, 'TCP') is None:
|
||||
u.addportmapping(
|
||||
self.peer_port, 'TCP', u.lanaddr, self.peer_port,
|
||||
'LBRY peer port', '')
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
log.info("Set UPnP redirect for TCP port %d", self.peer_port)
|
||||
else:
|
||||
# see comment below
|
||||
log.warning("UPnP redirect already set for TCP port %d", self.peer_port)
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
if self.dht_node_port is not None:
|
||||
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
|
||||
self.upnp_redirects_set = True
|
||||
if u.getspecificportmapping(self.dht_node_port, 'UDP') is None:
|
||||
u.addportmapping(
|
||||
self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port,
|
||||
'LBRY DHT port', '')
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
log.info("Set UPnP redirect for UDP port %d", self.dht_node_port)
|
||||
else:
|
||||
# TODO: check that the existing redirect was
|
||||
# put up by an old lbrynet session before
|
||||
# grabbing it if such a disconnected redirect
|
||||
# exists, then upnp won't work unless the
|
||||
# redirect is appended or is torn down and set
|
||||
# back up. a bad shutdown of lbrynet could
|
||||
# leave such a redirect up and cause problems
|
||||
# on the next start. this could be
|
||||
# problematic if a previous lbrynet session
|
||||
# didn't make the redirect, and it was made by
|
||||
# another application
|
||||
log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port)
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
return True
|
||||
return False
|
||||
|
||||
def upnp_failed(err):
|
||||
logging.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
return False
|
||||
|
||||
d = threads.deferToThread(threaded_try_upnp)
|
||||
@@ -180,10 +251,7 @@ class LBRYSession(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("Starting the dht")
|
||||
|
||||
def match_port(h, p):
|
||||
return h, p
|
||||
log.info("Starting DHT")
|
||||
|
||||
def join_resolved_addresses(result):
|
||||
addresses = []
|
||||
@@ -201,11 +269,14 @@ class LBRYSession(object):
|
||||
ds = []
|
||||
for host, port in self.known_dht_nodes:
|
||||
d = reactor.resolve(host)
|
||||
d.addCallback(match_port, port)
|
||||
d.addCallback(lambda h: (h, port)) # match host to port
|
||||
ds.append(d)
|
||||
|
||||
self.dht_node = node.Node(udpPort=self.dht_node_port, lbryid=self.lbryid,
|
||||
externalIP=self.external_ip)
|
||||
self.dht_node = self.dht_node_class(
|
||||
udpPort=self.dht_node_port,
|
||||
lbryid=self.lbryid,
|
||||
externalIP=self.external_ip
|
||||
)
|
||||
self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager)
|
||||
if self.hash_announcer is None:
|
||||
self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port)
|
||||
@@ -216,7 +287,7 @@ class LBRYSession(object):
|
||||
return dl
|
||||
|
||||
def _setup_other_components(self):
|
||||
logging.debug("Setting up the rest of the components")
|
||||
log.debug("Setting up the rest of the components")
|
||||
|
||||
if self.rate_limiter is None:
|
||||
self.rate_limiter = RateLimiter()
|
||||
@@ -225,21 +296,46 @@ class LBRYSession(object):
|
||||
if self.blob_dir is None:
|
||||
self.blob_manager = TempBlobManager(self.hash_announcer)
|
||||
else:
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer, self.blob_dir, self.db_dir)
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer,
|
||||
self.blob_dir,
|
||||
self.db_dir)
|
||||
|
||||
self.rate_limiter.tick()
|
||||
if self.blob_tracker is None:
|
||||
self.blob_tracker = self.blob_tracker_class(self.blob_manager,
|
||||
self.peer_finder,
|
||||
self.dht_node)
|
||||
if self.payment_rate_manager is None:
|
||||
self.payment_rate_manager = self.payment_rate_manager_class(
|
||||
self.base_payment_rate_manager,
|
||||
self.blob_tracker,
|
||||
self.is_generous)
|
||||
|
||||
self.rate_limiter.start()
|
||||
d1 = self.blob_manager.setup()
|
||||
d2 = self.wallet.start()
|
||||
return defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
|
||||
dl.addCallback(lambda _: self.blob_tracker.start())
|
||||
return dl
|
||||
|
||||
def _unset_upnp(self):
|
||||
log.info("Unsetting upnp for %s", self)
|
||||
|
||||
def threaded_unset_upnp():
|
||||
u = self.upnp_handler
|
||||
if self.peer_port is not None:
|
||||
u.deleteportmapping(self.peer_port, 'TCP')
|
||||
if self.dht_node_port is not None:
|
||||
u.deleteportmapping(self.dht_node_port, 'UDP')
|
||||
self.upnp_redirects_set = False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
for port, protocol in self.upnp_redirects:
|
||||
if u.getspecificportmapping(port, protocol) is None:
|
||||
log.warning(
|
||||
"UPnP redirect for %s %d was removed by something else.",
|
||||
protocol, port)
|
||||
else:
|
||||
u.deleteportmapping(port, protocol)
|
||||
log.info("Removed UPnP redirect for %s %d.", protocol, port)
|
||||
self.upnp_redirects = []
|
||||
|
||||
return threads.deferToThread(threaded_unset_upnp)
|
||||
d = threads.deferToThread(threaded_unset_upnp)
|
||||
d.addErrback(lambda err: str(err))
|
||||
return d
|
||||
@@ -0,0 +1,152 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.interfaces import INegotiationStrategy
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
|
||||
|
||||
|
||||
def get_default_strategy(blob_tracker, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
|
||||
|
||||
|
||||
class Strategy(object):
|
||||
"""
|
||||
Base for negotiation strategies
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
|
||||
self.price_model = price_model
|
||||
self.is_generous = (
|
||||
is_generous if is_generous is not None else conf.settings['is_generous_host'])
|
||||
self.accepted_offers = {}
|
||||
self.pending_sent_offers = {}
|
||||
self.offers_sent = {}
|
||||
self.offers_received = {}
|
||||
self.max_rate = max_rate or Decimal(self.price_model.base_price * 50)
|
||||
self.min_rate = Decimal(min_rate)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def make_offer(self, peer, blobs):
|
||||
offer_count = self.offers_sent.get(peer, 0)
|
||||
self._add_offer_sent(peer)
|
||||
if peer in self.accepted_offers:
|
||||
# if there was a previous accepted offer, use that
|
||||
offer = self.accepted_offers[peer]
|
||||
if peer in self.pending_sent_offers:
|
||||
del self.pending_sent_offers[peer]
|
||||
elif offer_count == 0 and self.is_generous:
|
||||
# Try asking for it for free
|
||||
offer = Offer(Decimal(0.0))
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
else:
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._make_rate_offer(rates, offer_count)
|
||||
offer = Offer(price)
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
return offer
|
||||
|
||||
def respond_to_offer(self, offer, peer, blobs):
|
||||
offer_count = self.offers_received.get(peer, 0)
|
||||
self._add_offer_received(peer)
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._get_response_rate(rates, offer_count)
|
||||
if peer in self.accepted_offers:
|
||||
offer = self.accepted_offers[peer]
|
||||
elif offer.rate == 0.0 and offer_count == 0 and self.is_generous:
|
||||
# give blobs away for free by default on the first request
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
elif offer.rate >= price:
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
else:
|
||||
offer.reject()
|
||||
if peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
return offer
|
||||
|
||||
def update_accepted_offers(self, peer, offer):
|
||||
if not offer.is_accepted and peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
if offer.is_accepted:
|
||||
self.accepted_offers.update({peer: offer})
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
|
||||
def _add_offer_sent(self, peer):
|
||||
turn = self.offers_sent.get(peer, 0) + 1
|
||||
self.offers_sent.update({peer: turn})
|
||||
|
||||
def _add_offer_received(self, peer):
|
||||
turn = self.offers_received.get(peer, 0) + 1
|
||||
self.offers_received.update({peer: turn})
|
||||
|
||||
def _bounded_price(self, price):
|
||||
price_for_return = Decimal(min(self.max_rate, max(price, self.min_rate)))
|
||||
return price_for_return
|
||||
|
||||
|
||||
class BasicAvailabilityWeightedStrategy(Strategy):
|
||||
"""Basic strategy to target blob prices based on supply relative to mean supply
|
||||
|
||||
Discount price target with each incoming request, and raise it
|
||||
with each outgoing from the modeled price until the rate is
|
||||
accepted or a threshold is reached
|
||||
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, blob_tracker, acceleration=1.25,
|
||||
deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0,
|
||||
is_generous=None,
|
||||
base_price=0.0001, alpha=1.0):
|
||||
price_model = MeanAvailabilityWeightedPrice(
|
||||
blob_tracker, base_price=base_price, alpha=alpha)
|
||||
Strategy.__init__(self, price_model, max_rate, min_rate, is_generous)
|
||||
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
|
||||
self._deceleration = Decimal(deceleration)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
mean_rate = Decimal(sum(rates)) / Decimal(max(len(rates), 1))
|
||||
return mean_rate
|
||||
|
||||
def _premium(self, rate, turn):
|
||||
return rate * (self._acceleration ** Decimal(turn))
|
||||
|
||||
def _discount(self, rate, turn):
|
||||
return rate * (self._deceleration ** Decimal(turn))
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
discounted = self._discount(rate, offer_count)
|
||||
rounded_price = round(discounted, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
with_premium = self._premium(rate, offer_count)
|
||||
rounded_price = round(with_premium, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
|
||||
class OnlyFreeStrategy(Strategy):
|
||||
implementer(INegotiationStrategy)
|
||||
def __init__(self, *args, **kwargs):
|
||||
price_model = ZeroPrice()
|
||||
Strategy.__init__(self, price_model, 0.0, 0.0, True)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
return 0.0
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return 0.0
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return 0.0
|
||||
@@ -3,6 +3,9 @@ from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamCreator(object):
|
||||
"""Classes which derive from this class create a 'stream', which can be any
|
||||
collection of associated blobs and associated metadata. These classes
|
||||
@@ -42,7 +45,7 @@ class StreamCreator(object):
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
logging.debug("stop has been called for StreamCreator")
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
current_blob = self.current_blob
|
||||
@@ -61,13 +64,13 @@ class StreamCreator(object):
|
||||
def _finished(self):
|
||||
pass
|
||||
|
||||
# TODO: move the stream creation process to its own thread and
|
||||
# remove the reactor from this process.
|
||||
def write(self, data):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self._write(data)
|
||||
if self.stopped is False and self.streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def _write(self, data):
|
||||
pass
|
||||
pass
|
||||
@@ -3,6 +3,10 @@ import json
|
||||
import logging
|
||||
from twisted.internet import threads
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamDescriptorReader(object):
|
||||
@@ -69,7 +73,9 @@ class StreamDescriptorWriter(object):
|
||||
return self._write_stream_descriptor(json.dumps(sd_info))
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
"""This method must be overridden by subclasses to write raw data to the stream descriptor"""
|
||||
"""This method must be overridden by subclasses to write raw data to
|
||||
the stream descriptor
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -81,7 +87,7 @@ class PlainStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
|
||||
def write_file():
|
||||
logging.debug("Writing the sd file to disk")
|
||||
log.debug("Writing the sd file to disk")
|
||||
with open(self.sd_file_name, 'w') as sd_file:
|
||||
sd_file.write(raw_data)
|
||||
return self.sd_file_name
|
||||
@@ -96,89 +102,141 @@ class BlobStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
logging.debug("Creating the new blob for the stream descriptor")
|
||||
log.debug("Creating the new blob for the stream descriptor")
|
||||
blob_creator = self.blob_manager.get_blob_creator()
|
||||
blob_creator.write(raw_data)
|
||||
logging.debug("Wrote the data to the new blob")
|
||||
log.debug("Wrote the data to the new blob")
|
||||
return blob_creator.close()
|
||||
|
||||
|
||||
class StreamMetadata(object):
|
||||
FROM_BLOB = 1
|
||||
FROM_PLAIN = 2
|
||||
|
||||
def __init__(self, validator, options, factories):
|
||||
self.validator = validator
|
||||
self.options = options
|
||||
self.factories = factories
|
||||
self.metadata_source = None
|
||||
self.source_blob_hash = None
|
||||
self.source_file = None
|
||||
|
||||
|
||||
class StreamDescriptorIdentifier(object):
|
||||
"""Tries to determine the type of stream described by the stream descriptor using the
|
||||
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
|
||||
and returns the appropriate ones based on the type of the stream descriptor given
|
||||
"""
|
||||
def __init__(self):
|
||||
self._sd_info_validators = {} # {stream_type: IStreamDescriptorValidator}
|
||||
self._stream_downloader_factories = defaultdict(list) # {stream_type: [IStreamDownloaderFactory]}
|
||||
# {stream_type: IStreamDescriptorValidator}
|
||||
self._sd_info_validators = {}
|
||||
# {stream_type: IStreamOptions
|
||||
self._stream_options = {}
|
||||
# {stream_type: [IStreamDownloaderFactory]}
|
||||
self._stream_downloader_factories = defaultdict(list)
|
||||
|
||||
def add_stream_info_validator(self, stream_type, sd_info_validator):
|
||||
"""
|
||||
This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
|
||||
"""This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
|
||||
There can only be one StreamDescriptorValidator for each type of stream.
|
||||
|
||||
@param stream_type: A string representing the type of stream descriptor. This must be unique to
|
||||
this stream descriptor.
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor. This must be unique to this stream descriptor.
|
||||
|
||||
@param sd_info_validator: A class implementing the IStreamDescriptorValidator interface. This class's
|
||||
constructor will be passed the raw metadata in the stream descriptor file and its 'validate' method
|
||||
will then be called. If the validation step fails, an exception will be thrown, preventing the stream
|
||||
descriptor from being further processed.
|
||||
@param sd_info_validator: A class implementing the
|
||||
IStreamDescriptorValidator interface. This class's
|
||||
constructor will be passed the raw metadata in the stream
|
||||
descriptor file and its 'validate' method will then be
|
||||
called. If the validation step fails, an exception will be
|
||||
thrown, preventing the stream descriptor from being
|
||||
further processed.
|
||||
|
||||
@param stream_options: A class implementing the IStreamOptions
|
||||
interface. This class's constructor will be passed the
|
||||
sd_info_validator object containing the raw metadata from
|
||||
the stream descriptor file.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._sd_info_validators[stream_type] = sd_info_validator
|
||||
self._stream_options[stream_type] = stream_options
|
||||
|
||||
def add_stream_downloader_factory(self, stream_type, factory):
|
||||
"""
|
||||
Register a stream downloader factory with the StreamDescriptorIdentifier.
|
||||
"""Register a stream downloader factory with the StreamDescriptorIdentifier.
|
||||
|
||||
This is how the StreamDescriptorIdentifier determines what factories may be used to process different stream
|
||||
descriptor files. There must be at least one factory for each type of stream added via
|
||||
"add_stream_info_validator".
|
||||
This is how the StreamDescriptorIdentifier determines what
|
||||
factories may be used to process different stream descriptor
|
||||
files. There must be at least one factory for each type of
|
||||
stream added via "add_stream_info_validator".
|
||||
|
||||
@param stream_type: A string representing the type of stream descriptor which the factory knows how to process.
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor which the factory knows how to process.
|
||||
|
||||
@param factory: An object implementing the IStreamDownloaderFactory interface.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._stream_downloader_factories[stream_type].append(factory)
|
||||
|
||||
def get_info_and_factories_for_sd_file(self, sd_path):
|
||||
def _return_metadata(self, options_validator_factories, source_type, source):
|
||||
validator, options, factories = options_validator_factories
|
||||
m = StreamMetadata(validator, options, factories)
|
||||
m.metadata_source = source_type
|
||||
if source_type == StreamMetadata.FROM_BLOB:
|
||||
m.source_blob_hash = source
|
||||
if source_type == StreamMetadata.FROM_PLAIN:
|
||||
m.source_file = source
|
||||
return m
|
||||
|
||||
def get_metadata_for_sd_file(self, sd_path):
|
||||
sd_reader = PlainStreamDescriptorReader(sd_path)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
|
||||
return d
|
||||
|
||||
def get_info_and_factories_for_sd_blob(self, sd_blob):
|
||||
def get_metadata_for_sd_blob(self, sd_blob):
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
|
||||
return d
|
||||
|
||||
def _get_factories(self, stream_type):
|
||||
assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_downloader_factories[stream_type]
|
||||
|
||||
def _get_validator(self, stream_type):
|
||||
assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._sd_info_validators:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._sd_info_validators[stream_type]
|
||||
|
||||
def _return_info_and_factories(self, sd_info):
|
||||
assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.'
|
||||
def _get_options(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_options[stream_type]
|
||||
|
||||
def _return_options_and_validator_and_factories(self, sd_info):
|
||||
if not 'stream_type' in sd_info:
|
||||
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
|
||||
stream_type = sd_info['stream_type']
|
||||
factories = self._get_factories(stream_type)
|
||||
validator = self._get_validator(stream_type)(sd_info)
|
||||
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
|
||||
|
||||
d = validator.validate()
|
||||
|
||||
d.addCallback(lambda _: (validator, factories))
|
||||
def get_options():
|
||||
options = self._get_options(stream_type)
|
||||
return validator, options, factories
|
||||
|
||||
d.addCallback(lambda _: get_options())
|
||||
return d
|
||||
|
||||
|
||||
def download_sd_blob(session, blob_hash, payment_rate_manager):
|
||||
def download_sd_blob(session, blob_hash, payment_rate_manager, timeout=None):
|
||||
"""
|
||||
Downloads a single blob from the network
|
||||
|
||||
@@ -190,6 +248,11 @@ def download_sd_blob(session, blob_hash, payment_rate_manager):
|
||||
|
||||
@return: An object of type HashBlob
|
||||
"""
|
||||
downloader = StandaloneBlobDownloader(blob_hash, session.blob_manager, session.peer_finder,
|
||||
session.rate_limiter, payment_rate_manager, session.wallet)
|
||||
return downloader.download()
|
||||
downloader = StandaloneBlobDownloader(blob_hash,
|
||||
session.blob_manager,
|
||||
session.peer_finder,
|
||||
session.rate_limiter,
|
||||
payment_rate_manager,
|
||||
session.wallet,
|
||||
timeout)
|
||||
return downloader.download()
|
||||
@@ -4,4 +4,4 @@ Classes and functions which can be used by any application wishing to make use o
|
||||
This includes classes for connecting to other peers and downloading blobs from them, listening for
|
||||
connections from peers and responding to their requests, managing locally stored blobs, sending
|
||||
and receiving payments, and locating peers in the DHT.
|
||||
"""
|
||||
"""
|
||||
@@ -1,12 +1,41 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from decimal import Decimal
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
|
||||
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
|
||||
from lbrynet.interfaces import IRequestCreator
|
||||
from lbrynet.core.Offer import Offer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_points(num_bytes, rate):
|
||||
if isinstance(rate, float):
|
||||
return 1.0 * num_bytes * rate / 2**20
|
||||
elif isinstance(rate, Decimal):
|
||||
return 1.0 * num_bytes * float(rate) / 2**20
|
||||
else:
|
||||
raise Exception("Unknown rate type")
|
||||
|
||||
|
||||
def cache(fn):
|
||||
"""Caches the function call for each instance"""
|
||||
attr = '__{}_value'.format(fn.__name__)
|
||||
|
||||
def helper(self):
|
||||
if not hasattr(self, attr):
|
||||
value = fn(self)
|
||||
setattr(self, attr, value)
|
||||
return getattr(self, attr)
|
||||
return helper
|
||||
|
||||
|
||||
class BlobRequester(object):
|
||||
@@ -17,64 +46,23 @@ class BlobRequester(object):
|
||||
self.peer_finder = peer_finder
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.download_manager = download_manager
|
||||
self._download_manager = download_manager
|
||||
self._peers = defaultdict(int) # {Peer: score}
|
||||
self._available_blobs = defaultdict(list) # {Peer: [blob_hash]}
|
||||
self._unavailable_blobs = defaultdict(list) # {Peer: [blob_hash]}}
|
||||
self._protocol_prices = {} # {ClientProtocol: price}
|
||||
self._protocol_offers = {}
|
||||
self._price_disagreements = [] # [Peer]
|
||||
self._protocol_tries = {}
|
||||
self._maxed_out_peers = []
|
||||
self._incompatible_peers = []
|
||||
|
||||
######## IRequestCreator #########
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
sent_request = False
|
||||
if self._blobs_to_download() and self._should_send_request_to(peer):
|
||||
a_r = self._get_availability_request(peer)
|
||||
d_r = self._get_download_request(peer)
|
||||
p_r = None
|
||||
|
||||
if a_r is not None or d_r is not None:
|
||||
p_r = self._get_price_request(peer, protocol)
|
||||
|
||||
if a_r is not None:
|
||||
d1 = protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, peer, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request", peer)
|
||||
sent_request = True
|
||||
if d_r is not None:
|
||||
reserved_points = self._reserve_points(peer, protocol, d_r.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
d_r.finished_deferred.addCallbacks(self._download_succeeded, self._download_failed,
|
||||
callbackArgs=(peer, d_r.blob),
|
||||
errbackArgs=(peer,))
|
||||
d_r.finished_deferred.addBoth(self._pay_or_cancel_payment, protocol, reserved_points, d_r.blob)
|
||||
d_r.finished_deferred.addErrback(self._handle_download_error, peer, d_r.blob)
|
||||
|
||||
d2 = protocol.add_blob_request(d_r)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d2.addCallback(self._handle_incoming_blob, peer, d_r)
|
||||
d2.addErrback(self._request_failed, "download request", peer)
|
||||
|
||||
sent_request = True
|
||||
else:
|
||||
d_r.cancel(InsufficientFundsError())
|
||||
return defer.fail(InsufficientFundsError())
|
||||
if sent_request is True:
|
||||
if p_r is not None:
|
||||
d3 = protocol.add_request(p_r)
|
||||
d3.addCallback(self._handle_price_response, peer, p_r, protocol)
|
||||
d3.addErrback(self._request_failed, "price request", peer)
|
||||
return defer.succeed(sent_request)
|
||||
"""Makes an availability request, download request and price request"""
|
||||
if not self.should_send_next_request(peer):
|
||||
return defer.succeed(False)
|
||||
return self._send_next_request(peer, protocol)
|
||||
|
||||
def get_new_peers(self):
|
||||
d = self._get_hash_for_peer_search()
|
||||
@@ -82,36 +70,35 @@ class BlobRequester(object):
|
||||
return d
|
||||
|
||||
######### internal calls #########
|
||||
def should_send_next_request(self, peer):
|
||||
return (
|
||||
self._blobs_to_download() and
|
||||
self._should_send_request_to(peer)
|
||||
)
|
||||
|
||||
def _download_succeeded(self, arg, peer, blob):
|
||||
logging.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
|
||||
self._update_local_score(peer, 5.0)
|
||||
peer.update_stats('blobs_downloaded', 1)
|
||||
peer.update_score(5.0)
|
||||
self.blob_manager.blob_completed(blob)
|
||||
return arg
|
||||
def _send_next_request(self, peer, protocol):
|
||||
log.debug('Sending a blob request for %s and %s', peer, protocol)
|
||||
availability = AvailabilityRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
download = DownloadRequest(self, peer, protocol, self.payment_rate_manager, self.wallet)
|
||||
price = PriceRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
|
||||
def _download_failed(self, reason, peer):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self._update_local_score(peer, -10.0)
|
||||
return reason
|
||||
sent_request = False
|
||||
if availability.can_make_request():
|
||||
availability.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if price.can_make_request():
|
||||
# TODO: document why a PriceRequest is only made if an
|
||||
# Availability or Download request was made
|
||||
price.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if download.can_make_request():
|
||||
try:
|
||||
download.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
except InsufficientFundsError as err:
|
||||
return defer.fail(err)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, protocol, reserved_points, blob):
|
||||
if blob.length != 0 and (not isinstance(arg, Failure) or arg.check(DownloadCanceledError)):
|
||||
self._pay_peer(protocol, blob.length, reserved_points)
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _handle_download_error(self, err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
logging.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
return defer.succeed(sent_request)
|
||||
|
||||
def _get_hash_for_peer_search(self):
|
||||
r = None
|
||||
@@ -123,7 +110,7 @@ class BlobRequester(object):
|
||||
else:
|
||||
blob_hash = blobs_without_sources[0].blob_hash
|
||||
r = blob_hash
|
||||
logging.debug("Blob requester peer search response: %s", str(r))
|
||||
log.debug("Blob requester peer search response: %s", str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -134,9 +121,18 @@ class BlobRequester(object):
|
||||
|
||||
def choose_best_peers(peers):
|
||||
bad_peers = self._get_bad_peers()
|
||||
return [p for p in peers if not p in bad_peers]
|
||||
without_bad_peers = [p for p in peers if not p in bad_peers]
|
||||
without_maxed_out_peers = [
|
||||
p for p in without_bad_peers if p not in self._maxed_out_peers]
|
||||
return without_maxed_out_peers
|
||||
|
||||
d.addCallback(choose_best_peers)
|
||||
|
||||
def lookup_failed(err):
|
||||
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
|
||||
return []
|
||||
|
||||
d.addErrback(lookup_failed)
|
||||
return d
|
||||
|
||||
def _should_send_request_to(self, peer):
|
||||
@@ -163,145 +159,411 @@ class BlobRequester(object):
|
||||
return False
|
||||
|
||||
def _blobs_to_download(self):
|
||||
needed_blobs = self.download_manager.needed_blobs()
|
||||
needed_blobs = self._download_manager.needed_blobs()
|
||||
return sorted(needed_blobs, key=lambda b: b.is_downloading())
|
||||
|
||||
def _blobs_without_sources(self):
|
||||
return [b for b in self.download_manager.needed_blobs() if not self._hash_available(b.blob_hash)]
|
||||
|
||||
def _get_availability_request(self, peer):
|
||||
all_needed = [b.blob_hash for b in self._blobs_to_download() if not b.blob_hash in self._available_blobs[peer]]
|
||||
# sort them so that the peer will be asked first for blobs it hasn't said it doesn't have
|
||||
to_request = sorted(all_needed, key=lambda b: b in self._unavailable_blobs[peer])[:20]
|
||||
if to_request:
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
return None
|
||||
|
||||
def _get_download_request(self, peer):
|
||||
request = None
|
||||
to_download = [b for b in self._blobs_to_download() if self._hash_available_on(b.blob_hash, peer)]
|
||||
while to_download and request is None:
|
||||
blob_to_download = to_download[0]
|
||||
to_download = to_download[1:]
|
||||
if not blob_to_download.is_validated():
|
||||
d, write_func, cancel_func = blob_to_download.open_for_writing(peer)
|
||||
|
||||
def counting_write_func(data):
|
||||
peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return write_func(data)
|
||||
|
||||
if d is not None:
|
||||
|
||||
request_dict = {'requested_blob': blob_to_download.blob_hash}
|
||||
response_identifier = 'incoming_blob'
|
||||
|
||||
request = ClientBlobRequest(request_dict, response_identifier, counting_write_func, d,
|
||||
cancel_func, blob_to_download)
|
||||
|
||||
logging.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
|
||||
return request
|
||||
return [
|
||||
b for b in self._download_manager.needed_blobs()
|
||||
if not self._hash_available(b.blob_hash)
|
||||
]
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
if protocol in self._protocol_prices:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_price_request(self, peer, protocol):
|
||||
request = None
|
||||
if not protocol in self._protocol_prices:
|
||||
self._protocol_prices[protocol] = self.payment_rate_manager.get_rate_blob_data(peer)
|
||||
request_dict = {'blob_data_payment_rate': self._protocol_prices[protocol]}
|
||||
request = ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
|
||||
|
||||
class RequestHelper(object):
|
||||
def __init__(self, requestor, peer, protocol, payment_rate_manager):
|
||||
self.requestor = requestor
|
||||
self.peer = peer
|
||||
self.protocol = protocol
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
|
||||
@property
|
||||
def protocol_prices(self):
|
||||
return self.requestor._protocol_prices
|
||||
|
||||
@property
|
||||
def protocol_offers(self):
|
||||
return self.requestor._protocol_offers
|
||||
|
||||
@property
|
||||
def available_blobs(self):
|
||||
return self.requestor._available_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def unavailable_blobs(self):
|
||||
return self.requestor._unavailable_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def maxed_out_peers(self):
|
||||
return self.requestor._maxed_out_peers
|
||||
|
||||
def update_local_score(self, score):
|
||||
self.requestor._update_local_score(self.peer, score)
|
||||
|
||||
def _request_failed(self, reason, request_type):
|
||||
if reason.check(RequestCanceledError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self.requestor._incompatible_peers.append(self.peer)
|
||||
log.warning("A request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
request_type, reason.getErrorMessage(), reason.type)
|
||||
self.update_local_score(-10.0)
|
||||
if isinstance(reason, (InvalidResponseError, NoResponseError)):
|
||||
self.peer.update_score(-10.0)
|
||||
else:
|
||||
self.peer.update_score(-2.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
|
||||
def get_rate(self):
|
||||
if self.payment_rate_manager.price_limit_reached(self.peer):
|
||||
if self.peer not in self.maxed_out_peers:
|
||||
self.maxed_out_peers.append(self.peer)
|
||||
return None
|
||||
rate = self.protocol_prices.get(self.protocol)
|
||||
if rate is None:
|
||||
if self.peer in self.payment_rate_manager.strategy.pending_sent_offers:
|
||||
pending = self.payment_rate_manager.strategy.pending_sent_offers[self.peer]
|
||||
if not pending.is_too_low and not pending.is_accepted:
|
||||
return pending.rate
|
||||
rate = self.payment_rate_manager.get_rate_blob_data(self.peer, self.available_blobs)
|
||||
return rate
|
||||
|
||||
|
||||
def _handle_incoming_blob(response_dict, peer, request):
|
||||
if request.response_identifier not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not isinstance(response_dict[request.response_identifier], dict):
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
type(response_dict[request.response_identifier]))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if 'blob_hash' not in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError(
|
||||
"Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob'])
|
||||
)
|
||||
if 'length' not in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
|
||||
|
||||
def _handle_download_error(err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
log.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
|
||||
|
||||
class AvailabilityRequest(RequestHelper):
|
||||
"""Ask a peer what blobs it has available.
|
||||
|
||||
Results are saved in `_available_blobs` and `_unavailable_blobs`
|
||||
on the parent BlobRequester.
|
||||
"""
|
||||
def can_make_request(self):
|
||||
return self.get_top_needed_blobs()
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
to_request = self.get_top_needed_blobs()
|
||||
if not to_request:
|
||||
raise Exception('Unable to make a request without available blobs')
|
||||
return self._make_request(to_request)
|
||||
|
||||
@cache
|
||||
def get_top_needed_blobs(self, limit=20):
|
||||
all_needed = [
|
||||
b.blob_hash for b in self.requestor._blobs_to_download()
|
||||
if not self.is_available(b)
|
||||
]
|
||||
# sort them so that the peer will be asked first for blobs it
|
||||
# hasn't said it doesn't have
|
||||
sorted_needed = sorted(
|
||||
all_needed,
|
||||
key=lambda b: b in self.unavailable_blobs
|
||||
)
|
||||
return sorted_needed[:limit]
|
||||
|
||||
def is_available(self, blob):
|
||||
return blob.blob_hash in self.available_blobs
|
||||
|
||||
def _make_request(self, to_request):
|
||||
log.debug('Requesting blobs: %s', to_request)
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
def _handle_request(self, a_r):
|
||||
log.debug('making an availability request')
|
||||
d1 = self.protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request")
|
||||
|
||||
def _reserve_points(self, peer, protocol, max_bytes):
|
||||
assert protocol in self._protocol_prices
|
||||
points_to_reserve = 1.0 * max_bytes * self._protocol_prices[protocol] / 2**20
|
||||
return self.wallet.reserve_points(peer, points_to_reserve)
|
||||
def _handle_availability(self, response_dict, request):
|
||||
assert request.response_identifier == 'available_blobs'
|
||||
if 'available_blobs' not in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
log.debug("Received a response to the availability request")
|
||||
# save available blobs
|
||||
blob_hashes = response_dict['available_blobs']
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
self.process_available_blob_hash(blob_hash, request)
|
||||
# everything left in the request is missing
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self.unavailable_blobs.append(blob_hash)
|
||||
return True
|
||||
|
||||
def _pay_peer(self, protocol, num_bytes, reserved_points):
|
||||
def process_available_blob_hash(self, blob_hash, request):
|
||||
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self.available_blobs.append(blob_hash)
|
||||
self.remove_from_unavailable_blobs(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
|
||||
def remove_from_unavailable_blobs(self, blob_hash):
|
||||
if blob_hash in self.unavailable_blobs:
|
||||
self.unavailable_blobs.remove(blob_hash)
|
||||
|
||||
|
||||
class PriceRequest(RequestHelper):
|
||||
"""Ask a peer if a certain price is acceptable"""
|
||||
def can_make_request(self):
|
||||
if len(self.available_blobs) and self.protocol not in self.protocol_prices:
|
||||
return self.get_rate() is not None
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_price_request()
|
||||
self._handle_price_request(request)
|
||||
|
||||
def _get_price_request(self):
|
||||
rate = self.get_rate()
|
||||
if rate is None:
|
||||
log.debug("No blobs to request from %s", self.peer)
|
||||
raise Exception('Cannot make a price request without a payment rate')
|
||||
log.debug("Offer rate %s to %s for %i blobs", rate, self.peer, len(self.available_blobs))
|
||||
|
||||
request_dict = {'blob_data_payment_rate': rate}
|
||||
assert self.protocol not in self.protocol_offers
|
||||
self.protocol_offers[self.protocol] = rate
|
||||
return ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
|
||||
def _handle_price_request(self, price_request):
|
||||
d = self.protocol.add_request(price_request)
|
||||
d.addCallback(self._handle_price_response, price_request)
|
||||
d.addErrback(self._request_failed, "price request")
|
||||
|
||||
def _handle_price_response(self, response_dict, request):
|
||||
assert request.response_identifier == 'blob_data_payment_rate'
|
||||
if 'blob_data_payment_rate' not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
offer_value = self.protocol_offers.pop(self.protocol)
|
||||
offer = Offer(offer_value)
|
||||
offer.handle(response_dict['blob_data_payment_rate'])
|
||||
self.payment_rate_manager.record_offer_reply(self.peer, offer)
|
||||
if offer.is_accepted:
|
||||
log.info("Offered rate %f/mb accepted by %s", offer.rate, self.peer.host)
|
||||
self.protocol_prices[self.protocol] = offer.rate
|
||||
return True
|
||||
elif offer.is_too_low:
|
||||
log.debug("Offered rate %f/mb rejected by %s", offer.rate, self.peer.host)
|
||||
return not self.payment_rate_manager.price_limit_reached(self.peer)
|
||||
else:
|
||||
log.warning("Price disagreement")
|
||||
self.requestor._price_disagreements.append(self.peer)
|
||||
return False
|
||||
|
||||
|
||||
class DownloadRequest(RequestHelper):
|
||||
"""Choose a blob and download it from a peer and also pay the peer for the data."""
|
||||
def __init__(self, requester, peer, protocol, payment_rate_manager, wallet):
|
||||
RequestHelper.__init__(self, requester, peer, protocol, payment_rate_manager)
|
||||
self.wallet = wallet
|
||||
|
||||
def can_make_request(self):
|
||||
if self.protocol in self.protocol_prices:
|
||||
return self.get_blob_details()
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_download_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
blob_details = self.get_blob_details()
|
||||
if not blob_details:
|
||||
raise Exception('No blobs available to download')
|
||||
return self._make_request(blob_details)
|
||||
|
||||
@cache
|
||||
def get_blob_details(self):
|
||||
"""Open a blob for writing and return the details.
|
||||
|
||||
If no blob can be opened, returns None.
|
||||
"""
|
||||
to_download = self.get_available_blobs()
|
||||
return self.find_blob(to_download)
|
||||
|
||||
def get_available_blobs(self):
|
||||
available_blobs = [
|
||||
b for b in self.requestor._blobs_to_download()
|
||||
if self.requestor._hash_available_on(b.blob_hash, self.peer)
|
||||
]
|
||||
log.debug('available blobs: %s', available_blobs)
|
||||
return available_blobs
|
||||
|
||||
def find_blob(self, to_download):
|
||||
"""Return the first blob in `to_download` that is successfully opened for write."""
|
||||
for blob in to_download:
|
||||
if blob.is_validated():
|
||||
log.debug('Skipping blob %s as its already validated', blob)
|
||||
continue
|
||||
d, write_func, cancel_func = blob.open_for_writing(self.peer)
|
||||
if d is not None:
|
||||
return BlobDownloadDetails(blob, d, write_func, cancel_func, self.peer)
|
||||
log.debug('Skipping blob %s as there was an issue opening it for writing', blob)
|
||||
return None
|
||||
|
||||
def _make_request(self, blob_details):
|
||||
blob = blob_details.blob
|
||||
request = ClientBlobRequest(
|
||||
{'requested_blob': blob.blob_hash},
|
||||
'incoming_blob',
|
||||
blob_details.counting_write_func,
|
||||
blob_details.deferred,
|
||||
blob_details.cancel_func,
|
||||
blob
|
||||
)
|
||||
log.debug("Requesting blob %s from %s", blob.blob_hash, self.peer)
|
||||
return request
|
||||
|
||||
def _handle_download_request(self, client_blob_request):
|
||||
reserved_points = self.reserve_funds_or_cancel(client_blob_request)
|
||||
self.add_callbacks_to_download_request(client_blob_request, reserved_points)
|
||||
self.create_add_blob_request(client_blob_request)
|
||||
|
||||
def reserve_funds_or_cancel(self, client_blob_request):
|
||||
reserved_points = self._reserve_points(client_blob_request.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
return reserved_points
|
||||
client_blob_request.cancel(InsufficientFundsError())
|
||||
client_blob_request.finished_deferred.addErrback(lambda _: True)
|
||||
raise InsufficientFundsError()
|
||||
|
||||
def add_callbacks_to_download_request(self, client_blob_request, reserved_points):
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
client_blob_request.finished_deferred.addCallbacks(
|
||||
self._download_succeeded,
|
||||
self._download_failed,
|
||||
callbackArgs=(client_blob_request.blob,),
|
||||
)
|
||||
client_blob_request.finished_deferred.addBoth(
|
||||
self._pay_or_cancel_payment, reserved_points, client_blob_request.blob)
|
||||
client_blob_request.finished_deferred.addErrback(
|
||||
_handle_download_error, self.peer, client_blob_request.blob)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, reserved_points, blob):
|
||||
if self._can_pay_peer(blob, arg):
|
||||
self._pay_peer(blob.length, reserved_points)
|
||||
d = self.requestor.blob_manager.add_blob_to_download_history(
|
||||
str(blob), str(self.peer.host), float(self.protocol_prices[self.protocol]))
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _can_pay_peer(self, blob, arg):
|
||||
return (
|
||||
blob.length != 0 and
|
||||
(not isinstance(arg, Failure) or arg.check(DownloadCanceledError))
|
||||
)
|
||||
|
||||
def _pay_peer(self, num_bytes, reserved_points):
|
||||
assert num_bytes != 0
|
||||
assert protocol in self._protocol_prices
|
||||
point_amount = 1.0 * num_bytes * self._protocol_prices[protocol] / 2**20
|
||||
rate = self.get_rate()
|
||||
point_amount = get_points(num_bytes, rate)
|
||||
self.wallet.send_points(reserved_points, point_amount)
|
||||
self.payment_rate_manager.record_points_paid(point_amount)
|
||||
|
||||
def _cancel_points(self, reserved_points):
|
||||
self.wallet.cancel_point_reservation(reserved_points)
|
||||
|
||||
def _handle_availability(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
logging.debug("Received a response to the availability request")
|
||||
blob_hashes = response_dict[request.response_identifier]
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
logging.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self._available_blobs[peer].append(blob_hash)
|
||||
if blob_hash in self._unavailable_blobs[peer]:
|
||||
self._unavailable_blobs[peer].remove(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self._unavailable_blobs[peer].append(blob_hash)
|
||||
return True
|
||||
def create_add_blob_request(self, client_blob_request):
|
||||
d = self.protocol.add_blob_request(client_blob_request)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d.addCallback(_handle_incoming_blob, self.peer, client_blob_request)
|
||||
d.addErrback(self._request_failed, "download request")
|
||||
|
||||
def _handle_incoming_blob(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not type(response_dict[request.response_identifier]) == dict:
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
(type(response_dict[request.response_identifier]),))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if not 'blob_hash' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError("Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob']))
|
||||
if not 'length' in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
def _reserve_points(self, num_bytes):
|
||||
# jobevers: there was an assertion here, but I don't think it
|
||||
# was a valid assertion to make. It is possible for a rate to
|
||||
# not yet been set for this protocol or for it to have been
|
||||
# removed so instead I switched it to check if a rate has been set
|
||||
# and calculate it if it has not
|
||||
rate = self.get_rate()
|
||||
points_to_reserve = get_points(num_bytes, rate)
|
||||
return self.wallet.reserve_points(self.peer, points_to_reserve)
|
||||
|
||||
def _handle_price_response(self, response_dict, peer, request, protocol):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
assert protocol in self._protocol_prices
|
||||
response = response_dict[request.response_identifier]
|
||||
if response == "RATE_ACCEPTED":
|
||||
return True
|
||||
else:
|
||||
del self._protocol_prices[protocol]
|
||||
self._price_disagreements.append(peer)
|
||||
return True
|
||||
def _download_succeeded(self, arg, blob):
|
||||
log.info("Blob %s has been successfully downloaded from %s", blob, self.peer)
|
||||
self.update_local_score(5.0)
|
||||
self.peer.update_stats('blobs_downloaded', 1)
|
||||
self.peer.update_score(5.0)
|
||||
self.requestor.blob_manager.blob_completed(blob)
|
||||
return arg
|
||||
|
||||
def _request_failed(self, reason, request_type, peer):
|
||||
if reason.check(RequestCanceledError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
str(request_type), reason.getErrorMessage(), reason.type)
|
||||
self._update_local_score(peer, -10.0)
|
||||
if isinstance(reason, InvalidResponseError):
|
||||
peer.update_score(-10.0)
|
||||
else:
|
||||
peer.update_score(-2.0)
|
||||
return reason
|
||||
def _download_failed(self, reason):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self.update_local_score(-10.0)
|
||||
return reason
|
||||
|
||||
|
||||
class BlobDownloadDetails(object):
|
||||
"""Contains the information needed to make a ClientBlobRequest from an open blob"""
|
||||
def __init__(self, blob, deferred, write_func, cancel_func, peer):
|
||||
self.blob = blob
|
||||
self.deferred = deferred
|
||||
self.write_func = write_func
|
||||
self.cancel_func = cancel_func
|
||||
self.peer = peer
|
||||
|
||||
def counting_write_func(self, data):
|
||||
self.peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return self.write_func(data)
|
||||
@@ -1,9 +1,12 @@
|
||||
import json
|
||||
import logging
|
||||
from twisted.internet import error, defer, reactor
|
||||
from decimal import Decimal
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.protocols.policies import TimeoutMixin
|
||||
from twisted.python import failure
|
||||
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
|
||||
from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
@@ -11,12 +14,22 @@ from lbrynet.interfaces import IRequestSender, IRateLimited
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
class ClientProtocol(Protocol):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_decimal(obj):
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
raise TypeError(repr(obj) + " is not JSON serializable")
|
||||
|
||||
|
||||
class ClientProtocol(Protocol, TimeoutMixin):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
######### Protocol #########
|
||||
PROTOCOL_TIMEOUT = 30
|
||||
|
||||
def connectionMade(self):
|
||||
log.debug("Connection made to %s", self.factory.peer)
|
||||
self._connection_manager = self.factory.connection_manager
|
||||
self._rate_limiter = self.factory.rate_limiter
|
||||
self.peer = self.factory.peer
|
||||
@@ -27,19 +40,23 @@ class ClientProtocol(Protocol):
|
||||
self._next_request = {}
|
||||
self.connection_closed = False
|
||||
self.connection_closing = False
|
||||
|
||||
# This needs to be set for TimeoutMixin
|
||||
self.callLater = utils.call_later
|
||||
self.peer.report_up()
|
||||
|
||||
self._ask_for_request()
|
||||
|
||||
def dataReceived(self, data):
|
||||
log.debug("Data receieved from %s", self.peer)
|
||||
self.setTimeout(None)
|
||||
self._rate_limiter.report_dl_bytes(len(data))
|
||||
if self._downloading_blob is True:
|
||||
self._blob_download_request.write(data)
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > MAX_RESPONSE_SIZE:
|
||||
logging.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
|
||||
log.warning("Response is too large from %s. Size %s",
|
||||
self.peer, len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
response, extra_data = self._get_valid_response(self._response_buff)
|
||||
if response is not None:
|
||||
@@ -48,29 +65,34 @@ class ClientProtocol(Protocol):
|
||||
if self._downloading_blob is True and len(extra_data) != 0:
|
||||
self._blob_download_request.write(extra_data)
|
||||
|
||||
def timeoutConnection(self):
|
||||
log.info("Connection timed out to %s", self.peer)
|
||||
self.peer.report_down()
|
||||
self.transport.abortConnection()
|
||||
|
||||
def connectionLost(self, reason):
|
||||
log.debug("Connection lost to %s: %s", self.peer, reason)
|
||||
self.setTimeout(None)
|
||||
self.connection_closed = True
|
||||
if reason.check(error.ConnectionDone):
|
||||
err = failure.Failure(ConnectionClosedBeforeResponseError())
|
||||
else:
|
||||
err = reason
|
||||
#if self._response_deferreds:
|
||||
# logging.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
for key, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
if self._blob_download_request is not None:
|
||||
self._blob_download_request.cancel(err)
|
||||
self._connection_manager.protocol_disconnected(self.peer, self)
|
||||
self.factory.connection_was_made_deferred.callback(True)
|
||||
|
||||
######### IRequestSender #########
|
||||
|
||||
def add_request(self, request):
|
||||
if request.response_identifier in self._response_deferreds:
|
||||
return defer.fail(failure.Failure(ValueError("There is already a request for that response active")))
|
||||
raise ValueError("There is already a request for that response active")
|
||||
self._next_request.update(request.request_dict)
|
||||
d = defer.Deferred()
|
||||
logging.debug("Adding a request. Request: %s", str(request))
|
||||
log.debug("Adding a request for %s. Request: %s", self.peer, request.request_dict)
|
||||
self._response_deferreds[request.response_identifier] = d
|
||||
return d
|
||||
|
||||
@@ -83,7 +105,7 @@ class ClientProtocol(Protocol):
|
||||
blob_request.finished_deferred.addErrback(self._handle_response_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(failure.Failure(ValueError("There is already a blob download request active")))
|
||||
raise ValueError("There is already a blob download request active")
|
||||
|
||||
def cancel_requests(self):
|
||||
self.connection_closing = True
|
||||
@@ -102,12 +124,12 @@ class ClientProtocol(Protocol):
|
||||
######### Internal request handling #########
|
||||
|
||||
def _handle_request_error(self, err):
|
||||
logging.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
log.error(
|
||||
"An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
self.peer, err.getTraceback())
|
||||
self.transport.loseConnection()
|
||||
|
||||
def _ask_for_request(self):
|
||||
|
||||
if self.connection_closed is True or self.connection_closing is True:
|
||||
return
|
||||
|
||||
@@ -117,17 +139,20 @@ class ClientProtocol(Protocol):
|
||||
self._send_request_message(request_msg)
|
||||
else:
|
||||
# The connection manager has indicated that this connection should be terminated
|
||||
logging.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
log.info(
|
||||
"Closing the connection to %s due to having no further requests to send",
|
||||
self.peer)
|
||||
self.peer.report_success()
|
||||
self.transport.loseConnection()
|
||||
|
||||
d = self._connection_manager.get_next_request(self.peer, self)
|
||||
d.addCallback(send_request_or_close)
|
||||
d.addErrback(self._handle_request_error)
|
||||
|
||||
def _send_request_message(self, request_msg):
|
||||
self.setTimeout(self.PROTOCOL_TIMEOUT)
|
||||
# TODO: compare this message to the last one. If they're the same,
|
||||
# TODO: incrementally delay this message.
|
||||
m = json.dumps(request_msg)
|
||||
m = json.dumps(request_msg, default=encode_decimal)
|
||||
self.transport.write(m)
|
||||
|
||||
def _get_valid_response(self, response_msg):
|
||||
@@ -151,20 +176,24 @@ class ClientProtocol(Protocol):
|
||||
|
||||
def _handle_response_error(self, err):
|
||||
# If an error gets to this point, log it and kill the connection.
|
||||
if not err.check(MisbehavingPeerError, ConnectionClosedBeforeResponseError, DownloadCanceledError,
|
||||
RequestCanceledError):
|
||||
logging.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
expected_errors = (MisbehavingPeerError, ConnectionClosedBeforeResponseError,
|
||||
DownloadCanceledError, RequestCanceledError)
|
||||
if not err.check(expected_errors):
|
||||
log.error("The connection to %s is closing due to an unexpected error: %s",
|
||||
self.peer, err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError):
|
||||
self.transport.loseConnection()
|
||||
# The connection manager is closing the connection, so
|
||||
# there's no need to do it here.
|
||||
return err
|
||||
|
||||
def _handle_response(self, response):
|
||||
ds = []
|
||||
logging.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
log.debug(
|
||||
"Handling a response from %s. Expected responses: %s. Actual responses: %s",
|
||||
self.peer, self._response_deferreds.keys(), response.keys())
|
||||
for key, val in response.items():
|
||||
if key in self._response_deferreds:
|
||||
d = self._response_deferreds[key]
|
||||
del self._response_deferreds[key]
|
||||
d = self._response_deferreds.pop(key)
|
||||
d.callback({key: val})
|
||||
ds.append(d)
|
||||
for k, d in self._response_deferreds.items():
|
||||
@@ -178,12 +207,31 @@ class ClientProtocol(Protocol):
|
||||
d.addErrback(self._handle_response_error)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
# TODO: are we sure we want to consume errors here
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
|
||||
dl.addCallback(lambda _: self._ask_for_request())
|
||||
def get_next_request(results):
|
||||
failed = False
|
||||
for success, result in results:
|
||||
if success is False:
|
||||
failed = True
|
||||
if not isinstance(result.value, DownloadCanceledError):
|
||||
log.info(result.value)
|
||||
log.info("The connection to %s is closing due to an error: %s",
|
||||
self.peer, result.getTraceback())
|
||||
|
||||
self.peer.report_down()
|
||||
if failed is False:
|
||||
log.debug("Asking for another request from %s", self.peer)
|
||||
self._ask_for_request()
|
||||
else:
|
||||
log.debug("Not asking for another request from %s", self.peer)
|
||||
self.transport.loseConnection()
|
||||
|
||||
dl.addCallback(get_next_request)
|
||||
|
||||
def _downloading_finished(self, arg):
|
||||
logging.debug("The blob has finished downloading")
|
||||
log.debug("The blob has finished downloading from %s", self.peer)
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return arg
|
||||
@@ -194,10 +242,8 @@ class ClientProtocol(Protocol):
|
||||
# TODO: always be this way. it's done this way now because the client has no other way
|
||||
# TODO: of telling the server it wants the download to stop. It would be great if the
|
||||
# TODO: protocol had such a mechanism.
|
||||
logging.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
#self.transport.loseConnection()
|
||||
#return True
|
||||
log.debug("Closing the connection to %s because the download of blob %s was canceled",
|
||||
self.peer, self._blob_download_request.blob)
|
||||
return err
|
||||
|
||||
######### IRateLimited #########
|
||||
@@ -223,13 +269,18 @@ class ClientProtocolFactory(ClientFactory):
|
||||
self.rate_limiter = rate_limiter
|
||||
self.connection_manager = connection_manager
|
||||
self.p = None
|
||||
# This defer fires and returns True when connection was
|
||||
# made and completed, or fires and returns False if
|
||||
# connection failed
|
||||
self.connection_was_made_deferred = defer.Deferred()
|
||||
|
||||
def clientConnectionFailed(self, connector, reason):
|
||||
log.debug("Connection failed to %s: %s", self.peer, reason)
|
||||
self.peer.report_down()
|
||||
self.connection_manager.protocol_disconnected(self.peer, connector)
|
||||
self.connection_was_made_deferred.callback(False)
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = self.protocol()
|
||||
p.factory = self
|
||||
self.p = p
|
||||
return p
|
||||
return p
|
||||
@@ -1,4 +1,4 @@
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
class ClientRequest(object):
|
||||
@@ -17,11 +17,11 @@ class ClientBlobRequest(ClientPaidRequest):
|
||||
def __init__(self, request_dict, response_identifier, write_func, finished_deferred,
|
||||
cancel_func, blob):
|
||||
if blob.length is None:
|
||||
max_pay_units = BLOB_SIZE
|
||||
max_pay_units = conf.settings['BLOB_SIZE']
|
||||
else:
|
||||
max_pay_units = blob.length
|
||||
ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units)
|
||||
self.write = write_func
|
||||
self.finished_deferred = finished_deferred
|
||||
self.cancel = cancel_func
|
||||
self.blob = blob
|
||||
self.blob = blob
|
||||
@@ -1,177 +1,227 @@
|
||||
import random
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import defer, reactor
|
||||
from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
from lbrynet.conf import MAX_CONNECTIONS_PER_STREAM
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnectionHandler(object):
|
||||
def __init__(self, request_creators, factory):
|
||||
self.request_creators = request_creators
|
||||
self.factory = factory
|
||||
self.connection = None
|
||||
|
||||
|
||||
class ConnectionManager(object):
|
||||
implements(interfaces.IConnectionManager)
|
||||
MANAGE_CALL_INTERVAL_SEC = 5
|
||||
TCP_CONNECT_TIMEOUT = 15
|
||||
|
||||
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
|
||||
def __init__(self, downloader, rate_limiter,
|
||||
primary_request_creators, secondary_request_creators):
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
self.primary_request_creators = primary_request_creators
|
||||
self.secondary_request_creators = secondary_request_creators
|
||||
self.peer_connections = {} # {Peer: {'connection': connection,
|
||||
# 'request_creators': [IRequestCreator if using this connection]}}
|
||||
self.connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self.next_manage_call = None
|
||||
self._primary_request_creators = primary_request_creators
|
||||
self._secondary_request_creators = secondary_request_creators
|
||||
self._peer_connections = {} # {Peer: PeerConnectionHandler}
|
||||
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self._next_manage_call = None
|
||||
# a deferred that gets fired when a _manage call is set
|
||||
self._manage_deferred = None
|
||||
self.stopped = True
|
||||
log.info("%s initialized", self._get_log_name())
|
||||
|
||||
# this identifies what the connection manager is for,
|
||||
# used for logging purposes only
|
||||
def _get_log_name(self):
|
||||
out = 'Connection Manager Unknown'
|
||||
if hasattr(self.downloader, 'stream_name'):
|
||||
out = 'Connection Manager '+self.downloader.stream_name
|
||||
elif hasattr(self.downloader, 'blob_hash'):
|
||||
out = 'Connection Manager '+self.downloader.blob_hash
|
||||
return out
|
||||
|
||||
def _start(self):
|
||||
self.stopped = False
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
|
||||
def start(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = reactor.callLater(0, self._manage)
|
||||
log.debug("%s starting", self._get_log_name())
|
||||
self._start()
|
||||
self._next_manage_call = utils.call_later(0, self.manage)
|
||||
return defer.succeed(True)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
closing_deferreds = []
|
||||
for peer in self.peer_connections.keys():
|
||||
log.debug("%s stopping", self._get_log_name())
|
||||
self.stopped = True
|
||||
# wait for the current manage call to finish
|
||||
if self._manage_deferred:
|
||||
yield self._manage_deferred
|
||||
# in case we stopped between manage calls, cancel the next one
|
||||
if self._next_manage_call and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
yield self._close_peers()
|
||||
|
||||
def close_connection(p):
|
||||
logging.info("Abruptly closing a connection to %s due to downloading being paused",
|
||||
str(p))
|
||||
def num_peer_connections(self):
|
||||
return len(self._peer_connections)
|
||||
|
||||
if self.peer_connections[p]['factory'].p is not None:
|
||||
d = self.peer_connections[p]['factory'].p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
def _close_peers(self):
|
||||
def disconnect_peer(p):
|
||||
d = defer.Deferred()
|
||||
self._connections_closing[p] = d
|
||||
self._peer_connections[p].connection.disconnect()
|
||||
if p in self._peer_connections:
|
||||
del self._peer_connections[p]
|
||||
return d
|
||||
|
||||
def disconnect_peer():
|
||||
self.peer_connections[p]['connection'].disconnect()
|
||||
if p in self.peer_connections:
|
||||
del self.peer_connections[p]
|
||||
d = defer.Deferred()
|
||||
self.connections_closing[p] = d
|
||||
return d
|
||||
def close_connection(p):
|
||||
log.debug("%s Abruptly closing a connection to %s due to downloading being paused",
|
||||
self._get_log_name(), p)
|
||||
if self._peer_connections[p].factory.p is not None:
|
||||
d = self._peer_connections[p].factory.p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addBoth(lambda _: disconnect_peer(p))
|
||||
return d
|
||||
|
||||
d.addBoth(lambda _: disconnect_peer())
|
||||
return d
|
||||
|
||||
closing_deferreds.append(close_connection(peer))
|
||||
closing_deferreds = [close_connection(peer) for peer in self._peer_connections.keys()]
|
||||
return defer.DeferredList(closing_deferreds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_next_request(self, peer, protocol):
|
||||
log.debug("%s Trying to get the next request for peer %s", self._get_log_name(), peer)
|
||||
if not peer in self._peer_connections or self.stopped is True:
|
||||
log.debug("%s The peer %s has already been told to shut down.",
|
||||
self._get_log_name(), peer)
|
||||
defer.returnValue(False)
|
||||
requests = yield self._send_primary_requests(peer, protocol)
|
||||
have_request = any(r[1] for r in requests if r[0] is True)
|
||||
if have_request:
|
||||
yield self._send_secondary_requests(peer, protocol)
|
||||
defer.returnValue(have_request)
|
||||
|
||||
logging.debug("Trying to get the next request for peer %s", str(peer))
|
||||
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("The peer has already been told to shut down.")
|
||||
return defer.succeed(False)
|
||||
|
||||
def _send_primary_requests(self, peer, protocol):
|
||||
def handle_error(err):
|
||||
if err.check(InsufficientFundsError):
|
||||
self.downloader.insufficient_funds()
|
||||
return False
|
||||
else:
|
||||
return err
|
||||
err.trap(InsufficientFundsError)
|
||||
self.downloader.insufficient_funds(err)
|
||||
return False
|
||||
|
||||
def check_if_request_sent(request_sent, request_creator):
|
||||
if peer not in self._peer_connections:
|
||||
# This can happen if the connection is told to close
|
||||
return False
|
||||
if request_sent is False:
|
||||
if request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].remove(request_creator)
|
||||
if request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.remove(request_creator)
|
||||
else:
|
||||
if not request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].append(request_creator)
|
||||
if not request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.append(request_creator)
|
||||
return request_sent
|
||||
|
||||
def check_requests(requests):
|
||||
have_request = True in [r[1] for r in requests if r[0] is True]
|
||||
return have_request
|
||||
|
||||
def get_secondary_requests_if_necessary(have_request):
|
||||
if have_request is True:
|
||||
ds = []
|
||||
for s_r_c in self.secondary_request_creators:
|
||||
d = s_r_c.send_next_request(peer, protocol)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
else:
|
||||
dl = defer.succeed(None)
|
||||
dl.addCallback(lambda _: have_request)
|
||||
return dl
|
||||
|
||||
ds = []
|
||||
|
||||
for p_r_c in self.primary_request_creators:
|
||||
for p_r_c in self._primary_request_creators:
|
||||
d = p_r_c.send_next_request(peer, protocol)
|
||||
d.addErrback(handle_error)
|
||||
d.addCallback(check_if_request_sent, p_r_c)
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds, fireOnOneErrback=True)
|
||||
|
||||
dl = defer.DeferredList(ds, fireOnOneErrback=True)
|
||||
dl.addCallback(check_requests)
|
||||
dl.addCallback(get_secondary_requests_if_necessary)
|
||||
return dl
|
||||
def _send_secondary_requests(self, peer, protocol):
|
||||
ds = [
|
||||
s_r_c.send_next_request(peer, protocol)
|
||||
for s_r_c in self._secondary_request_creators
|
||||
]
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def protocol_disconnected(self, peer, protocol):
|
||||
if peer in self.peer_connections:
|
||||
del self.peer_connections[peer]
|
||||
if peer in self.connections_closing:
|
||||
d = self.connections_closing[peer]
|
||||
del self.connections_closing[peer]
|
||||
d.callback(True)
|
||||
@defer.inlineCallbacks
|
||||
def manage(self, schedule_next_call=True):
|
||||
self._manage_deferred = defer.Deferred()
|
||||
if len(self._peer_connections) < conf.settings['max_connections_per_stream']:
|
||||
log.debug("%s have %d connections, looking for %d",
|
||||
self._get_log_name(), len(self._peer_connections),
|
||||
conf.settings['max_connections_per_stream'])
|
||||
ordered_request_creators = self._rank_request_creator_connections()
|
||||
peers = yield self._get_new_peers(ordered_request_creators)
|
||||
new_conns = conf.settings['max_connections_per_stream'] - len(self._peer_connections)
|
||||
peers = self._pick_best_peers(peers, new_conns)
|
||||
for peer in peers:
|
||||
self._connect_to_peer(peer)
|
||||
self._manage_deferred.callback(None)
|
||||
self._manage_deferred = None
|
||||
if not self.stopped and schedule_next_call:
|
||||
self._next_manage_call = utils.call_later(self.MANAGE_CALL_INTERVAL_SEC, self.manage)
|
||||
|
||||
def _rank_request_creator_connections(self):
|
||||
"""
|
||||
@return: an ordered list of our request creators, ranked according to which has the least number of
|
||||
connections open that it likes
|
||||
"""Returns an ordered list of our request creators, ranked according
|
||||
to which has the least number of connections open that it
|
||||
likes
|
||||
"""
|
||||
def count_peers(request_creator):
|
||||
return len([p for p in self.peer_connections.itervalues() if request_creator in p['request_creators']])
|
||||
return len([
|
||||
p for p in self._peer_connections.itervalues()
|
||||
if request_creator in p.request_creators])
|
||||
|
||||
return sorted(self._primary_request_creators, key=count_peers)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _get_new_peers(self, request_creators):
|
||||
log.debug("%s Trying to get a new peer to connect to", self._get_log_name())
|
||||
if not request_creators:
|
||||
defer.returnValue(None)
|
||||
new_peers = yield request_creators[0].get_new_peers()
|
||||
if not new_peers:
|
||||
new_peers = yield self._get_new_peers(request_creators[1:])
|
||||
defer.returnValue(new_peers)
|
||||
|
||||
def _pick_best_peers(self, peers, num_peers_to_pick):
|
||||
# TODO: Eventually rank them based on past performance/reputation. For now
|
||||
# TODO: just pick the first to which we don't have an open connection
|
||||
log.debug("%s Got a list of peers to choose from: %s",
|
||||
self._get_log_name(), peers)
|
||||
log.debug("%s Current connections: %s",
|
||||
self._get_log_name(), self._peer_connections.keys())
|
||||
log.debug("%s List of connection states: %s", self._get_log_name(),
|
||||
[p_c_h.connection.state for p_c_h in self._peer_connections.values()])
|
||||
if peers is None:
|
||||
return []
|
||||
out = [peer for peer in peers if peer not in self._peer_connections]
|
||||
random.shuffle(out)
|
||||
return out[0:num_peers_to_pick]
|
||||
|
||||
return sorted(self.primary_request_creators, key=count_peers)
|
||||
|
||||
def _connect_to_peer(self, peer):
|
||||
if peer is None or self.stopped:
|
||||
return
|
||||
|
||||
from twisted.internet import reactor
|
||||
log.debug("%s Trying to connect to %s", self._get_log_name(), peer)
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
factory.connection_was_made_deferred.addCallback(
|
||||
lambda c_was_made: self._peer_disconnected(c_was_made, peer))
|
||||
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
|
||||
factory)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory,
|
||||
timeout=self.TCP_CONNECT_TIMEOUT)
|
||||
self._peer_connections[peer].connection = connection
|
||||
|
||||
if peer is not None:
|
||||
logging.debug("Trying to connect to %s", str(peer))
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory)
|
||||
self.peer_connections[peer] = {'connection': connection,
|
||||
'request_creators': self.primary_request_creators[:],
|
||||
'factory': factory}
|
||||
def _peer_disconnected(self, connection_was_made, peer):
|
||||
log.debug("%s protocol disconnected for %s",
|
||||
self._get_log_name(), peer)
|
||||
if peer in self._peer_connections:
|
||||
del self._peer_connections[peer]
|
||||
if peer in self._connections_closing:
|
||||
d = self._connections_closing[peer]
|
||||
del self._connections_closing[peer]
|
||||
d.callback(True)
|
||||
return connection_was_made
|
||||
|
||||
def _manage(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
def get_new_peers(request_creators):
|
||||
logging.debug("Trying to get a new peer to connect to")
|
||||
if len(request_creators) > 0:
|
||||
logging.debug("Got a creator to check: %s", str(request_creators[0]))
|
||||
d = request_creators[0].get_new_peers()
|
||||
d.addCallback(lambda h: h if h is not None else get_new_peers(request_creators[1:]))
|
||||
return d
|
||||
else:
|
||||
return defer.succeed(None)
|
||||
|
||||
def pick_best_peer(peers):
|
||||
# TODO: Eventually rank them based on past performance/reputation. For now
|
||||
# TODO: just pick the first to which we don't have an open connection
|
||||
logging.debug("Got a list of peers to choose from: %s", str(peers))
|
||||
if peers is None:
|
||||
return None
|
||||
for peer in peers:
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("Got a good peer. Returning peer %s", str(peer))
|
||||
return peer
|
||||
logging.debug("Couldn't find a good peer to connect to")
|
||||
return None
|
||||
|
||||
if len(self.peer_connections) < MAX_CONNECTIONS_PER_STREAM:
|
||||
ordered_request_creators = self._rank_request_creator_connections()
|
||||
d = get_new_peers(ordered_request_creators)
|
||||
d.addCallback(pick_best_peer)
|
||||
d.addCallback(self._connect_to_peer)
|
||||
|
||||
self.next_manage_call = reactor.callLater(1, self._manage)
|
||||
@@ -1,6 +1,13 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer, reactor
|
||||
from lbrynet.interfaces import IPeerFinder
|
||||
from lbrynet.core.utils import short_hash
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTPeerFinder(object):
|
||||
@@ -14,13 +21,11 @@ class DHTPeerFinder(object):
|
||||
self.next_manage_call = None
|
||||
|
||||
def run_manage_loop(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self._manage_peers()
|
||||
self.next_manage_call = reactor.callLater(60, self.run_manage_loop)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self.next_manage_call is not None and self.next_manage_call.active():
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
@@ -28,20 +33,29 @@ class DHTPeerFinder(object):
|
||||
def _manage_peers(self):
|
||||
pass
|
||||
|
||||
def find_peers_for_blob(self, blob_hash):
|
||||
@defer.inlineCallbacks
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None):
|
||||
def _trigger_timeout():
|
||||
if not finished_deferred.called:
|
||||
log.debug("Peer search for %s timed out", short_hash(blob_hash))
|
||||
finished_deferred.cancel()
|
||||
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = self.dht_node.getPeersForBlob(bin_hash)
|
||||
|
||||
def filter_peers(peer_list):
|
||||
good_peers = []
|
||||
for host, port in peer_list:
|
||||
peer = self.peer_manager.get_peer(host, port)
|
||||
if peer.is_available() is True:
|
||||
good_peers.append(peer)
|
||||
return good_peers
|
||||
if timeout is not None:
|
||||
reactor.callLater(timeout, _trigger_timeout)
|
||||
|
||||
d = self.dht_node.getPeersForBlob(bin_hash)
|
||||
d.addCallback(filter_peers)
|
||||
return d
|
||||
peer_list = yield finished_deferred
|
||||
|
||||
peers = set(peer_list)
|
||||
good_peers = []
|
||||
for host, port in peers:
|
||||
peer = self.peer_manager.get_peer(host, port)
|
||||
if peer.is_available() is True:
|
||||
good_peers.append(peer)
|
||||
|
||||
defer.returnValue(good_peers)
|
||||
|
||||
def get_most_popular_hashes(self, num_to_return):
|
||||
return self.dht_node.get_most_popular_hashes(num_to_return)
|
||||
return self.dht_node.get_most_popular_hashes(num_to_return)
|
||||
@@ -1,22 +1,21 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted.python import failure
|
||||
from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadManager(object):
|
||||
implements(interfaces.IDownloadManager)
|
||||
|
||||
def __init__(self, blob_manager, upload_allowed):
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.upload_allowed = upload_allowed
|
||||
self.blob_requester = None
|
||||
self.blob_info_finder = None
|
||||
self.progress_manager = None
|
||||
self.blob_handler = None
|
||||
self.connection_manager = None
|
||||
|
||||
self.blobs = {}
|
||||
self.blob_infos = {}
|
||||
|
||||
@@ -24,61 +23,44 @@ class DownloadManager(object):
|
||||
|
||||
def start_downloading(self):
|
||||
d = self.blob_info_finder.get_initial_blobs()
|
||||
logging.debug("Requested the initial blobs from the info finder")
|
||||
log.debug("Requested the initial blobs from the info finder")
|
||||
d.addCallback(self.add_blobs_to_download)
|
||||
d.addCallback(lambda _: self.resume_downloading())
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def resume_downloading(self):
|
||||
yield self.connection_manager.start()
|
||||
yield self.progress_manager.start()
|
||||
defer.returnValue(True)
|
||||
|
||||
def check_start(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
d1 = self.progress_manager.start()
|
||||
d1.addBoth(check_start, "progress manager")
|
||||
d2 = self.connection_manager.start()
|
||||
d2.addBoth(check_start, "connection manager")
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
dl.addCallback(lambda xs: False not in xs)
|
||||
return dl
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop_downloading(self):
|
||||
|
||||
def check_stop(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to stop the %s: %s", manager. result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
d1 = self.progress_manager.stop()
|
||||
d1.addBoth(check_stop, "progress manager")
|
||||
d2 = self.connection_manager.stop()
|
||||
d2.addBoth(check_stop, "connection manager")
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
dl.addCallback(lambda xs: False not in xs)
|
||||
return dl
|
||||
yield self.progress_manager.stop()
|
||||
yield self.connection_manager.stop()
|
||||
defer.returnValue(True)
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
logging.debug("Adding %s to blobs", str(blob_infos))
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
logging.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
log.debug(
|
||||
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
|
||||
|
||||
def error_during_add(err):
|
||||
logging.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
log.warning(
|
||||
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
logging.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, self.upload_allowed, blob_info.length)
|
||||
log.debug(
|
||||
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
ds.append(d)
|
||||
@@ -105,11 +87,14 @@ class DownloadManager(object):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
to_be_outputted = [b for n, b in self.blobs.iteritems() if n >= self.progress_manager.last_blob_outputted]
|
||||
to_be_outputted = [
|
||||
b for n, b in self.blobs.iteritems()
|
||||
if n >= self.progress_manager.last_blob_outputted
|
||||
]
|
||||
return sum([b.length for b in to_be_outputted if b.length is not None])
|
||||
|
||||
def calculate_bytes_left_to_download(self):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
@@ -5,7 +5,13 @@ from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.Error import InvalidBlobHashError, DownloadTimeoutError
|
||||
from lbrynet.core.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleBlobMetadataHandler(object):
|
||||
@@ -18,7 +24,7 @@ class SingleBlobMetadataHandler(object):
|
||||
######## IMetadataHandler #########
|
||||
|
||||
def get_initial_blobs(self):
|
||||
logging.debug("Returning the blob info")
|
||||
log.debug("Returning the blob info")
|
||||
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
|
||||
|
||||
def final_blob_num(self):
|
||||
@@ -26,36 +32,35 @@ class SingleBlobMetadataHandler(object):
|
||||
|
||||
|
||||
class SingleProgressManager(object):
|
||||
def __init__(self, finished_callback, download_manager):
|
||||
def __init__(self, download_manager, finished_callback, timeout_callback, timeout):
|
||||
self.finished_callback = finished_callback
|
||||
self.finished = False
|
||||
self.timeout_callback = timeout_callback
|
||||
self.download_manager = download_manager
|
||||
self._next_check_if_finished = None
|
||||
|
||||
self.timeout = timeout
|
||||
self.timeout_counter = 0
|
||||
self.checker = LoopingCall(self._check_if_finished)
|
||||
|
||||
def start(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
assert self._next_check_if_finished is None
|
||||
self._next_check_if_finished = reactor.callLater(0, self._check_if_finished)
|
||||
safe_start_looping_call(self.checker, 1)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
if self._next_check_if_finished is not None:
|
||||
self._next_check_if_finished.cancel()
|
||||
self._next_check_if_finished = None
|
||||
safe_stop_looping_call(self.checker)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _check_if_finished(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self._next_check_if_finished = None
|
||||
if self.finished is False:
|
||||
if self.stream_position() == 1:
|
||||
self.blob_downloaded(self.download_manager.blobs[0], 0)
|
||||
else:
|
||||
self._next_check_if_finished = reactor.callLater(1, self._check_if_finished)
|
||||
if self.stream_position() == 1:
|
||||
blob_downloaded = self.download_manager.blobs[0]
|
||||
log.debug("The blob %s has been downloaded. Calling the finished callback",
|
||||
str(blob_downloaded))
|
||||
safe_stop_looping_call(self.checker)
|
||||
self.finished_callback(blob_downloaded)
|
||||
elif self.timeout is not None:
|
||||
self.timeout_counter += 1
|
||||
if self.timeout_counter >= self.timeout:
|
||||
safe_stop_looping_call(self.checker)
|
||||
self.timeout_callback()
|
||||
|
||||
def stream_position(self):
|
||||
blobs = self.download_manager.blobs
|
||||
@@ -68,15 +73,6 @@ class SingleProgressManager(object):
|
||||
assert len(blobs) == 1
|
||||
return [b for b in blobs.itervalues() if not b.is_validated()]
|
||||
|
||||
def blob_downloaded(self, blob, blob_num):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
if self.finished is False:
|
||||
self.finished = True
|
||||
reactor.callLater(0, self.finished_callback, blob)
|
||||
|
||||
|
||||
class DummyBlobHandler(object):
|
||||
def __init__(self):
|
||||
@@ -87,30 +83,37 @@ class DummyBlobHandler(object):
|
||||
|
||||
|
||||
class StandaloneBlobDownloader(object):
|
||||
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet):
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, wallet,
|
||||
timeout=None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.timeout = timeout
|
||||
self.download_manager = None
|
||||
self.finished_deferred = None
|
||||
|
||||
def download(self):
|
||||
if not is_valid_blobhash(self.blob_hash):
|
||||
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
|
||||
|
||||
def cancel_download(d):
|
||||
self.stop()
|
||||
|
||||
self.finished_deferred = defer.Deferred(canceller=cancel_download)
|
||||
self.download_manager = DownloadManager(self.blob_manager, True)
|
||||
self.download_manager = DownloadManager(self.blob_manager)
|
||||
self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
self.download_manager)
|
||||
self.download_manager.blob_info_finder = SingleBlobMetadataHandler(self.blob_hash,
|
||||
self.download_manager)
|
||||
self.download_manager.progress_manager = SingleProgressManager(self._blob_downloaded,
|
||||
self.download_manager)
|
||||
self.download_manager.progress_manager = SingleProgressManager(self.download_manager,
|
||||
self._blob_downloaded,
|
||||
self._download_timedout,
|
||||
self.timeout)
|
||||
self.download_manager.blob_handler = DummyBlobHandler()
|
||||
self.download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
|
||||
self.download_manager.connection_manager = ConnectionManager(
|
||||
@@ -127,7 +130,15 @@ class StandaloneBlobDownloader(object):
|
||||
|
||||
def _blob_downloaded(self, blob):
|
||||
self.stop()
|
||||
self.finished_deferred.callback(blob)
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.callback(blob)
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def _download_timedout(self):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(DownloadTimeoutError(self.blob_hash))
|
||||
|
||||
def insufficient_funds(self, err):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(err)
|
||||
@@ -4,10 +4,14 @@ from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamProgressManager(object):
|
||||
implements(IProgressManager)
|
||||
|
||||
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False):
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
self.finished_callback = finished_callback
|
||||
self.blob_manager = blob_manager
|
||||
self.delete_blob_after_finished = delete_blob_after_finished
|
||||
@@ -61,41 +65,55 @@ class StreamProgressManager(object):
|
||||
return defer.succeed(None)
|
||||
|
||||
def _finished_with_blob(self, blob_num):
|
||||
logging.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
if self.delete_blob_after_finished is True:
|
||||
logging.debug("delete_blob_after_finished is True")
|
||||
log.debug("delete_blob_after_finished is True")
|
||||
blobs = self.download_manager.blobs
|
||||
if blob_num in blobs:
|
||||
logging.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
log.debug("Telling the blob manager, %s, to delete blob %s",
|
||||
self.blob_manager, blobs[blob_num].blob_hash)
|
||||
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
|
||||
else:
|
||||
logging.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
log.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
else:
|
||||
logging.debug("delete_blob_after_finished is False")
|
||||
log.debug("delete_blob_after_finished is False")
|
||||
|
||||
|
||||
class FullStreamProgressManager(StreamProgressManager):
|
||||
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False):
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
StreamProgressManager.__init__(self, finished_callback, blob_manager, download_manager,
|
||||
delete_blob_after_finished)
|
||||
self.outputting_d = None
|
||||
|
||||
######### IProgressManager #########
|
||||
|
||||
def _done(self, i, blobs):
|
||||
"""Return true if `i` is a blob number we don't have"""
|
||||
return (
|
||||
i not in blobs or
|
||||
(
|
||||
not blobs[i].is_validated() and
|
||||
i not in self.provided_blob_nums
|
||||
)
|
||||
)
|
||||
|
||||
def stream_position(self):
|
||||
blobs = self.download_manager.blobs
|
||||
if not blobs:
|
||||
return 0
|
||||
else:
|
||||
for i in xrange(max(blobs.iterkeys())):
|
||||
if not i in blobs or (not blobs[i].is_validated() and not i in self.provided_blob_nums):
|
||||
if self._done(i, blobs):
|
||||
return i
|
||||
return max(blobs.iterkeys()) + 1
|
||||
|
||||
def needed_blobs(self):
|
||||
blobs = self.download_manager.blobs
|
||||
return [b for n, b in blobs.iteritems() if not b.is_validated() and not n in self.provided_blob_nums]
|
||||
return [
|
||||
b for n, b in blobs.iteritems()
|
||||
if not b.is_validated() and not n in self.provided_blob_nums
|
||||
]
|
||||
|
||||
######### internal #########
|
||||
|
||||
@@ -115,6 +133,8 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
|
||||
def finished_outputting_blob():
|
||||
self.last_blob_outputted += 1
|
||||
|
||||
def check_if_finished():
|
||||
final_blob_num = self.download_manager.final_blob_num()
|
||||
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
|
||||
self._finished_outputting()
|
||||
@@ -126,16 +146,20 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
current_blob_num = self.last_blob_outputted + 1
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
log.debug("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
self.provided_blob_nums.append(self.last_blob_outputted + 1)
|
||||
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
d.addCallback(lambda _: check_if_finished())
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
log.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
if self.outputting_d is not None and not self.outputting_d.called:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
|
||||
d.addErrback(log_error)
|
||||
else:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
self.outputting_d = None
|
||||
@@ -15,4 +15,4 @@ def sign_with_pass_phrase(m, pass_phrase):
|
||||
|
||||
|
||||
def verify_signature(m, signature, pub_key):
|
||||
return seccure.verify(m, signature, pub_key, curve="brainpoolp384r1")
|
||||
return seccure.verify(m, signature, pub_key, curve="brainpoolp384r1")
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_read_handle(path):
|
||||
"""
|
||||
Get os independent read handle for a file
|
||||
"""
|
||||
|
||||
if os.name == "nt":
|
||||
file_mode = 'rb'
|
||||
else:
|
||||
file_mode = 'r'
|
||||
read_handle = open(path, file_mode)
|
||||
yield read_handle
|
||||
read_handle.close()
|
||||
@@ -0,0 +1,376 @@
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from txrequests import Session
|
||||
import twisted.python.log
|
||||
|
||||
from lbrynet import __version__ as lbrynet_version, build_type, conf
|
||||
from lbrynet.core import utils
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
#####
|
||||
|
||||
|
||||
TRACE = 5
|
||||
|
||||
|
||||
def bg_cb(sess, resp):
|
||||
""" Don't do anything with the response """
|
||||
pass
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None, session=None):
|
||||
logging.Handler.__init__(self)
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.session = session if session is not None else Session()
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
payload = self.format(record)
|
||||
self.session.post(self.url, data=payload, background_callback=bg_cb)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except:
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
for handler in log.handlers:
|
||||
if handler.name == handler_name:
|
||||
log.removeHandler(handler)
|
||||
|
||||
|
||||
def _log_decorator(fn):
|
||||
"""Configure a logging handler.
|
||||
|
||||
`fn` is a function that returns a logging handler. The returned
|
||||
handler has its log-level set and is attached to the specified
|
||||
logger or the root logger.
|
||||
"""
|
||||
|
||||
def helper(*args, **kwargs):
|
||||
log = kwargs.pop('log', logging.getLogger())
|
||||
level = kwargs.pop('level', logging.INFO)
|
||||
if not isinstance(level, int):
|
||||
# despite the name, getLevelName returns
|
||||
# the numeric level when passed a text level
|
||||
level = logging.getLevelName(level)
|
||||
handler = fn(*args, **kwargs)
|
||||
configure_handler(handler, log, level)
|
||||
return handler
|
||||
|
||||
return helper
|
||||
|
||||
|
||||
def configure_handler(handler, log, level):
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
return handler
|
||||
|
||||
|
||||
def disable_third_party_loggers():
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
logging.getLogger('urllib3').setLevel(logging.WARNING)
|
||||
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
|
||||
logging.getLogger('lbryum').setLevel(logging.WARNING)
|
||||
logging.getLogger('twisted').setLevel(logging.WARNING)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_console(**kwargs):
|
||||
"""Convenience function to configure a log-handler that outputs to stdout"""
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'console'
|
||||
return handler
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_file_handler(file_name, **kwargs):
|
||||
"""Convenience function to configure a log-handler that writes to `file_name`"""
|
||||
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'file'
|
||||
return handler
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
|
||||
version = version or lbrynet_version
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
def configure_loggly_handler(*args, **kwargs):
|
||||
if build_type.BUILD == 'dev':
|
||||
return
|
||||
level = kwargs.pop('level', logging.WARNING)
|
||||
_configure_loggly_handler(*args, level=level, **kwargs)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def _configure_loggly_handler(url=None, **kwargs):
|
||||
url = url or get_loggly_url()
|
||||
formatter = JsonFormatter(**kwargs)
|
||||
handler = HTTPSHandler(url)
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
|
||||
def failure(failure, log, msg, *args):
|
||||
"""Log a failure message from a deferred.
|
||||
|
||||
Args:
|
||||
failure: twisted.python.failure.Failure
|
||||
log: a python logger instance
|
||||
msg: the message to log. Can use normal logging string interpolation.
|
||||
the last argument will be set to the error message from the failure.
|
||||
args: values to substitute into `msg`
|
||||
"""
|
||||
args += (failure.getErrorMessage(),)
|
||||
exc_info = (failure.type, failure.value, failure.getTracebackObject())
|
||||
log.error(msg, *args, exc_info=exc_info)
|
||||
|
||||
|
||||
def convert_verbose(verbose):
|
||||
"""Convert the results of the --verbose flag into a list of logger names
|
||||
|
||||
if --verbose is not provided, args.verbose will be None and logging
|
||||
should be at the info level.
|
||||
if --verbose is provided, but not followed by any arguments, then
|
||||
args.verbose = [] and debug logging should be enabled for all of lbrynet
|
||||
along with info logging on lbryum.
|
||||
if --verbose is provided and followed by arguments, those arguments
|
||||
will be in a list
|
||||
"""
|
||||
if verbose is None:
|
||||
return []
|
||||
if verbose == []:
|
||||
return ['lbrynet', 'lbryum']
|
||||
return verbose
|
||||
|
||||
|
||||
def configure_logging(file_name, console, verbose=None):
|
||||
"""Apply the default logging configuration.
|
||||
|
||||
Enables two log-handlers at the INFO level: a file logger and a loggly logger.
|
||||
Optionally turns on a console logger that defaults to the INFO level, with
|
||||
specified loggers being set to the DEBUG level.
|
||||
|
||||
Args:
|
||||
file_name: the file to which logs should be saved
|
||||
console: If true, enable a console logger
|
||||
verbose: a list of loggers to set to debug level.
|
||||
See `convert_verbose` for more details.
|
||||
"""
|
||||
verbose = convert_verbose(verbose)
|
||||
configure_twisted()
|
||||
configure_file_handler(file_name)
|
||||
configure_loggly_handler()
|
||||
disable_third_party_loggers()
|
||||
if console:
|
||||
# if there are some loggers at the debug level, we need
|
||||
# to enable the console to allow debug. Otherwise, only
|
||||
# allow info.
|
||||
level = 'DEBUG' if verbose else 'INFO'
|
||||
handler = configure_console(level=level)
|
||||
if 'lbryum' in verbose:
|
||||
# TODO: this enables lbryum logging on the other handlers
|
||||
# too which isn't consistent with how verbose logging
|
||||
# happens with other loggers. Should change the configuration
|
||||
# so that its only logging at the INFO level for the console.
|
||||
logging.getLogger('lbryum').setLevel(logging.INFO)
|
||||
verbose.remove('lbryum')
|
||||
if verbose:
|
||||
handler.addFilter(LoggerNameFilter(verbose))
|
||||
|
||||
|
||||
def configure_twisted():
|
||||
"""Setup twisted logging to output events to the python stdlib logger"""
|
||||
# I tried using the new logging api
|
||||
# https://twistedmatrix.com/documents/current/core/howto/logger.html#compatibility-with-standard-library-logging
|
||||
# and it simply didn't work
|
||||
observer = twisted.python.log.PythonLoggingObserver()
|
||||
observer.start()
|
||||
|
||||
|
||||
class LoggerNameFilter(object):
|
||||
"""Filter a log record based on its name.
|
||||
|
||||
Allows all info level and higher records to pass thru.
|
||||
Debug records pass if the log record name (or a parent) match
|
||||
the input list of logger names.
|
||||
"""
|
||||
|
||||
def __init__(self, logger_names):
|
||||
self.logger_names = logger_names
|
||||
|
||||
def filter(self, record):
|
||||
if record.levelno >= logging.INFO:
|
||||
return True
|
||||
name = record.name
|
||||
while name:
|
||||
if name in self.logger_names:
|
||||
return True
|
||||
name = get_parent(name)
|
||||
return False
|
||||
|
||||
|
||||
def get_parent(logger_name):
|
||||
names = logger_name.split('.')
|
||||
if len(names) == 1:
|
||||
return ''
|
||||
names = names[:-1]
|
||||
return '.'.join(names)
|
||||
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to subtitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
@@ -0,0 +1,20 @@
|
||||
class LoopingCallManager(object):
|
||||
def __init__(self, calls=None):
|
||||
self.calls = calls or {}
|
||||
|
||||
def register_looping_call(self, name, call):
|
||||
assert name not in self.calls, '{} is already registered'.format(name)
|
||||
self.calls[name] = call
|
||||
|
||||
def start(self, name, *args):
|
||||
lcall = self.calls[name]
|
||||
if not lcall.running:
|
||||
lcall.start(*args)
|
||||
|
||||
def stop(self, name):
|
||||
self.calls[name].stop()
|
||||
|
||||
def shutdown(self):
|
||||
for lcall in self.calls.itervalues():
|
||||
if lcall.running:
|
||||
lcall.stop()
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -37,10 +40,11 @@ class BlobAvailabilityHandler(object):
|
||||
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
logging.debug("Received the client's list of requested blobs")
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
@@ -52,4 +56,4 @@ class BlobAvailabilityHandler(object):
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
return d
|
||||
return d
|
||||
@@ -1,23 +1,31 @@
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet import analytics
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequestHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager):
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = BlobRequestHandler(self.blob_manager, self.wallet, self.payment_rate_manager)
|
||||
q_h = BlobRequestHandler(
|
||||
self.blob_manager, self.wallet, self.payment_rate_manager, self.analytics_manager)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
@@ -29,18 +37,23 @@ class BlobRequestHandlerFactory(object):
|
||||
|
||||
class BlobRequestHandler(object):
|
||||
implements(IQueryHandler, IBlobSender)
|
||||
PAYMENT_RATE_QUERY = 'blob_data_payment_rate'
|
||||
BLOB_QUERY = 'requested_blob'
|
||||
AVAILABILITY_QUERY = 'requested_blobs'
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager):
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = ['blob_data_payment_rate', 'requested_blob']
|
||||
self.query_identifiers = [self.PAYMENT_RATE_QUERY, self.BLOB_QUERY, self.AVAILABILITY_QUERY]
|
||||
self.analytics_manager = analytics_manager
|
||||
self.peer = None
|
||||
self.blob_data_payment_rate = None
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
self.blob_bytes_uploaded = 0
|
||||
self._blobs_requested = []
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
@@ -50,44 +63,22 @@ class BlobRequestHandler(object):
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = {}
|
||||
if self.query_identifiers[0] in queries:
|
||||
if not self.handle_blob_data_payment_rate(queries[self.query_identifiers[0]]):
|
||||
response['blob_data_payment_rate'] = "RATE_TOO_LOW"
|
||||
else:
|
||||
response['blob_data_payment_rate'] = 'RATE_ACCEPTED'
|
||||
response = defer.succeed({})
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
logging.debug("Received the client's request to send a blob")
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response_fields['error'] = "RATE_UNSET"
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
|
||||
d = self.blob_manager.get_blob(queries[self.query_identifiers[1]], True)
|
||||
|
||||
def open_blob_for_reading(blob):
|
||||
if blob.is_validated():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
logging.debug("Sending %s to client", str(blob))
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
return response
|
||||
logging.debug("We can not send %s", str(blob))
|
||||
response_fields['error'] = "BLOB_UNAVAILABLE"
|
||||
return response
|
||||
|
||||
d.addCallback(open_blob_for_reading)
|
||||
|
||||
return d
|
||||
else:
|
||||
return defer.succeed(response)
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
|
||||
@@ -105,12 +96,92 @@ class BlobRequestHandler(object):
|
||||
|
||||
######### internal #########
|
||||
|
||||
def handle_blob_data_payment_rate(self, requested_payment_rate):
|
||||
if not self.payment_rate_manager.accept_rate_blob_data(self.peer, requested_payment_rate):
|
||||
return False
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
log.debug("Offered rate %f LBC/mb for %i blobs", offer.rate, len(blobs))
|
||||
reply = self.payment_rate_manager.reply_to_offer(self.peer, blobs, offer)
|
||||
if reply.is_accepted:
|
||||
self.blob_data_payment_rate = offer.rate
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_ACCEPTED"
|
||||
log.debug("Accepted rate: %f", offer.rate)
|
||||
elif reply.is_too_low:
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_TOO_LOW"
|
||||
log.debug("Reject rate: %f", offer.rate)
|
||||
elif reply.is_unset:
|
||||
log.warning("Rate unset")
|
||||
request['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
log.debug("Returning rate query result: %s", str(request))
|
||||
|
||||
return request
|
||||
|
||||
def _handle_blob_query(self, response, query):
|
||||
log.debug("Received the client's request to send a blob")
|
||||
response['incoming_blob'] = {}
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response['incoming_blob'] = {'error': "RATE_UNSET"}
|
||||
return response
|
||||
else:
|
||||
self.blob_data_payment_rate = requested_payment_rate
|
||||
return True
|
||||
return self._send_blob(response, query)
|
||||
|
||||
def _send_blob(self, response, query):
|
||||
d = self.blob_manager.get_blob(query)
|
||||
d.addCallback(self.open_blob_for_reading, response)
|
||||
return d
|
||||
|
||||
def open_blob_for_reading(self, blob, response):
|
||||
response_fields = {}
|
||||
d = defer.succeed(None)
|
||||
if blob.is_validated():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
log.info("Sending %s to %s", str(blob), self.peer)
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
response['incoming_blob'] = response_fields
|
||||
d.addCallback(lambda _: self.record_transaction(blob))
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
log.debug("We can not send %s", str(blob))
|
||||
response['incoming_blob'] = {'error': 'BLOB_UNAVAILABLE'}
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
|
||||
def record_transaction(self, blob):
|
||||
d = self.blob_manager.add_blob_to_upload_history(
|
||||
str(blob), self.peer.host, self.blob_data_payment_rate)
|
||||
return d
|
||||
|
||||
def _reply_to_send_request(self, response, incoming):
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
log.debug("Rate not set yet")
|
||||
response['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
log.debug("Requested blob: %s", str(incoming))
|
||||
d = self.blob_manager.get_blob(incoming)
|
||||
d.addCallback(lambda blob: self.open_blob_for_reading(blob, response))
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
@@ -123,23 +194,31 @@ class BlobRequestHandler(object):
|
||||
return inner_d
|
||||
|
||||
def count_bytes(data):
|
||||
self.blob_bytes_uploaded += len(data)
|
||||
self.peer.update_stats('blob_bytes_uploaded', len(data))
|
||||
uploaded = len(data)
|
||||
self.blob_bytes_uploaded += uploaded
|
||||
self.peer.update_stats('blob_bytes_uploaded', uploaded)
|
||||
if self.analytics_manager is not None:
|
||||
self.analytics_manager.add_observation(analytics.BLOB_BYTES_UPLOADED, uploaded)
|
||||
return data
|
||||
|
||||
def start_transfer():
|
||||
self.file_sender = FileSender()
|
||||
logging.info("Starting the file upload")
|
||||
assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer"
|
||||
log.debug("Starting the file upload")
|
||||
assert self.read_handle is not None, \
|
||||
"self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
|
||||
return d
|
||||
|
||||
def set_expected_payment():
|
||||
logging.info("Setting expected payment")
|
||||
if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None:
|
||||
log.debug("Setting expected payment")
|
||||
if (
|
||||
self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None
|
||||
and self.blob_data_payment_rate > 0
|
||||
):
|
||||
# TODO: explain why 2**20
|
||||
self.wallet.add_expected_payment(self.peer,
|
||||
self.currently_uploading.length * 1.0 *
|
||||
self.blob_data_payment_rate / 2**20)
|
||||
self.blob_data_payment_rate / 2 ** 20)
|
||||
self.blob_bytes_uploaded = 0
|
||||
self.peer.update_stats('blobs_uploaded', 1)
|
||||
return None
|
||||
@@ -151,6 +230,6 @@ class BlobRequestHandler(object):
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
if reason is not None and isinstance(reason, Failure):
|
||||
logging.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
return _send_file()
|
||||
return _send_file()
|
||||
@@ -1,9 +1,18 @@
|
||||
import binascii
|
||||
from twisted.internet import defer, task, reactor
|
||||
import collections
|
||||
import logging
|
||||
import time
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTHashAnnouncer(object):
|
||||
ANNOUNCE_CHECK_INTERVAL = 60
|
||||
CONCURRENT_ANNOUNCERS = 5
|
||||
|
||||
"""This class announces to the DHT that this peer has certain blobs"""
|
||||
def __init__(self, dht_node, peer_port):
|
||||
self.dht_node = dht_node
|
||||
@@ -14,14 +23,12 @@ class DHTHashAnnouncer(object):
|
||||
self._concurrent_announcers = 0
|
||||
|
||||
def run_manage_loop(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.peer_port is not None:
|
||||
self._announce_available_hashes()
|
||||
self.next_manage_call = reactor.callLater(60, self.run_manage_loop)
|
||||
self.next_manage_call = utils.call_later(self.ANNOUNCE_CHECK_INTERVAL, self.run_manage_loop)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self.next_manage_call is not None:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
@@ -31,11 +38,15 @@ class DHTHashAnnouncer(object):
|
||||
|
||||
def immediate_announce(self, blob_hashes):
|
||||
if self.peer_port is not None:
|
||||
return self._announce_hashes(blob_hashes)
|
||||
return self._announce_hashes(blob_hashes, immediate=True)
|
||||
else:
|
||||
return defer.succeed(False)
|
||||
|
||||
def hash_queue_size(self):
|
||||
return len(self.hash_queue)
|
||||
|
||||
def _announce_available_hashes(self):
|
||||
log.debug('Announcing available hashes')
|
||||
ds = []
|
||||
for supplier in self.suppliers:
|
||||
d = supplier.hashes_to_announce()
|
||||
@@ -44,38 +55,75 @@ class DHTHashAnnouncer(object):
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
|
||||
def _announce_hashes(self, hashes):
|
||||
|
||||
def _announce_hashes(self, hashes, immediate=False):
|
||||
if not hashes:
|
||||
return
|
||||
log.debug('Announcing %s hashes', len(hashes))
|
||||
# TODO: add a timeit decorator
|
||||
start = time.time()
|
||||
ds = []
|
||||
|
||||
for h in hashes:
|
||||
announce_deferred = defer.Deferred()
|
||||
ds.append(announce_deferred)
|
||||
self.hash_queue.append((h, announce_deferred))
|
||||
if immediate:
|
||||
self.hash_queue.appendleft((h, announce_deferred))
|
||||
else:
|
||||
self.hash_queue.append((h, announce_deferred))
|
||||
log.debug('There are now %s hashes remaining to be announced', self.hash_queue_size())
|
||||
|
||||
def announce():
|
||||
if len(self.hash_queue):
|
||||
h, announce_deferred = self.hash_queue.popleft()
|
||||
log.debug('Announcing blob %s to dht', h)
|
||||
d = self.dht_node.announceHaveBlob(binascii.unhexlify(h), self.peer_port)
|
||||
d.chainDeferred(announce_deferred)
|
||||
d.addBoth(lambda _: reactor.callLater(0, announce))
|
||||
d.addBoth(lambda _: utils.call_later(0, announce))
|
||||
else:
|
||||
self._concurrent_announcers -= 1
|
||||
|
||||
for i in range(self._concurrent_announcers, 5):
|
||||
# TODO: maybe make the 5 configurable
|
||||
for i in range(self._concurrent_announcers, self.CONCURRENT_ANNOUNCERS):
|
||||
self._concurrent_announcers += 1
|
||||
announce()
|
||||
return defer.DeferredList(ds)
|
||||
d = defer.DeferredList(ds)
|
||||
d.addCallback(lambda _: log.debug('Took %s seconds to announce %s hashes',
|
||||
time.time() - start, len(hashes)))
|
||||
return d
|
||||
|
||||
|
||||
class DHTHashSupplier(object):
|
||||
# 1 hour is the min time hash will be reannounced
|
||||
MIN_HASH_REANNOUNCE_TIME = 60*60
|
||||
# conservative assumption of the time it takes to announce
|
||||
# a single hash
|
||||
SINGLE_HASH_ANNOUNCE_DURATION = 1
|
||||
|
||||
"""Classes derived from this class give hashes to a hash announcer"""
|
||||
def __init__(self, announcer):
|
||||
if announcer is not None:
|
||||
announcer.add_supplier(self)
|
||||
self.hash_announcer = announcer
|
||||
self.hash_reannounce_time = 60 * 60 # 1 hour
|
||||
|
||||
def hashes_to_announce(self):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
def get_next_announce_time(self, num_hashes_to_announce=1):
|
||||
"""
|
||||
Hash reannounce time is set to current time + MIN_HASH_REANNOUNCE_TIME,
|
||||
unless we are announcing a lot of hashes at once which could cause the
|
||||
the announce queue to pile up. To prevent pile up, reannounce
|
||||
only after a conservative estimate of when it will finish
|
||||
to announce all the hashes.
|
||||
|
||||
Args:
|
||||
num_hashes_to_announce: number of hashes that will be added to the queue
|
||||
Returns:
|
||||
timestamp for next announce time
|
||||
"""
|
||||
queue_size = self.hash_announcer.hash_queue_size()+num_hashes_to_announce
|
||||
reannounce = max(self.MIN_HASH_REANNOUNCE_TIME,
|
||||
queue_size*self.SINGLE_HASH_ANNOUNCE_DURATION)
|
||||
return time.time() + reannounce
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ from zope.interface import implements
|
||||
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerProtocol(Protocol):
|
||||
"""ServerProtocol needs to:
|
||||
|
||||
@@ -26,15 +29,14 @@ class ServerProtocol(Protocol):
|
||||
#Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
logging.debug("Got a connection")
|
||||
log.debug("Got a connection")
|
||||
peer_info = self.transport.getPeer()
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.request_handler = ServerRequestHandler(self)
|
||||
for query_handler_factory, enabled in self.factory.query_handler_factories.iteritems():
|
||||
if enabled is True:
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
logging.debug("Setting the request handler")
|
||||
for query_handler_factory in self.factory.query_handler_factories.values():
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
log.debug("Setting the request handler")
|
||||
self.factory.rate_limiter.register_protocol(self)
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
@@ -42,10 +44,10 @@ class ServerProtocol(Protocol):
|
||||
self.request_handler.stopProducing()
|
||||
self.factory.rate_limiter.unregister_protocol(self)
|
||||
if not reason.check(error.ConnectionDone):
|
||||
logging.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
def dataReceived(self, data):
|
||||
logging.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
self.factory.rate_limiter.report_dl_bytes(len(data))
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.data_received(data)
|
||||
@@ -53,7 +55,7 @@ class ServerProtocol(Protocol):
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
logging.debug("Registering the producer")
|
||||
log.debug("Registering the producer")
|
||||
assert streaming is True
|
||||
|
||||
def unregisterProducer(self):
|
||||
@@ -61,7 +63,7 @@ class ServerProtocol(Protocol):
|
||||
self.transport.loseConnection()
|
||||
|
||||
def write(self, data):
|
||||
logging.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
log.trace("Writing %s bytes of data to the transport", len(data))
|
||||
self.transport.write(data)
|
||||
self.factory.rate_limiter.report_ul_bytes(len(data))
|
||||
|
||||
@@ -88,4 +90,4 @@ class ServerProtocolFactory(ServerFactory):
|
||||
def __init__(self, rate_limiter, query_handler_factories, peer_manager):
|
||||
self.rate_limiter = rate_limiter
|
||||
self.query_handler_factories = query_handler_factories
|
||||
self.peer_manager = peer_manager
|
||||
self.peer_manager = peer_manager
|
||||
@@ -5,10 +5,14 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IRequestHandler
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and return request for information about
|
||||
more blobs that are associated with streams"""
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and
|
||||
return request for information about more blobs that are
|
||||
associated with streams.
|
||||
"""
|
||||
implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
|
||||
|
||||
def __init__(self, consumer):
|
||||
@@ -48,18 +52,19 @@ class ServerRequestHandler(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.production_paused is False:
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk != '':
|
||||
logging.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
if self.production_paused:
|
||||
return
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk == '':
|
||||
return
|
||||
log.trace("writing %s bytes to the client", len(chunk))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
#assert self.file_sender == producer
|
||||
self.producer = producer
|
||||
assert streaming is False
|
||||
producer.resumeProducing()
|
||||
@@ -76,7 +81,7 @@ class ServerRequestHandler(object):
|
||||
|
||||
def get_more_data():
|
||||
if self.producer is not None:
|
||||
logging.debug("Requesting more data from the producer")
|
||||
log.trace("Requesting more data from the producer")
|
||||
self.producer.resumeProducing()
|
||||
|
||||
reactor.callLater(0, get_more_data)
|
||||
@@ -84,22 +89,30 @@ class ServerRequestHandler(object):
|
||||
#From Protocol
|
||||
|
||||
def data_received(self, data):
|
||||
logging.debug("Received data")
|
||||
logging.debug("%s", str(data))
|
||||
log.debug("Received data")
|
||||
log.debug("%s", str(data))
|
||||
if self.request_received is False:
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg is not None:
|
||||
self.request_buff = ''
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender is not None:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
else:
|
||||
logging.info("Request buff not a valid json message")
|
||||
logging.info("Request buff: %s", str(self.request_buff))
|
||||
return self._parse_data_and_maybe_send_blob(data)
|
||||
else:
|
||||
logging.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
log.warning(
|
||||
"The client sent data when we were uploading a file. This should not happen")
|
||||
|
||||
def _parse_data_and_maybe_send_blob(self, data):
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg:
|
||||
self.request_buff = ''
|
||||
self._process_msg(msg)
|
||||
else:
|
||||
log.debug("Request buff not a valid json message")
|
||||
log.debug("Request buff: %s", self.request_buff)
|
||||
|
||||
def _process_msg(self, msg):
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
|
||||
|
||||
######### IRequestHandler #########
|
||||
|
||||
@@ -112,7 +125,7 @@ class ServerRequestHandler(object):
|
||||
#response handling
|
||||
|
||||
def request_failure_handler(self, err):
|
||||
logging.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
self.stopProducing()
|
||||
return err
|
||||
|
||||
@@ -122,15 +135,15 @@ class ServerRequestHandler(object):
|
||||
|
||||
def send_response(self, msg):
|
||||
m = json.dumps(msg)
|
||||
logging.info("Sending a response of length %s", str(len(m)))
|
||||
logging.debug("Response: %s", str(m))
|
||||
log.debug("Sending a response of length %s", str(len(m)))
|
||||
log.debug("Response: %s", str(m))
|
||||
self.response_buff = self.response_buff + m
|
||||
self._produce_more()
|
||||
return True
|
||||
|
||||
def handle_request(self, msg):
|
||||
logging.debug("Handling a request")
|
||||
logging.debug(str(msg))
|
||||
log.debug("Handling a request")
|
||||
log.debug(str(msg))
|
||||
|
||||
def create_response_message(results):
|
||||
response = {}
|
||||
@@ -140,11 +153,13 @@ class ServerRequestHandler(object):
|
||||
else:
|
||||
# result is a Failure
|
||||
return result
|
||||
logging.debug("Finished making the response message. Response: %s", str(response))
|
||||
log.debug("Finished making the response message. Response: %s", str(response))
|
||||
return response
|
||||
|
||||
def log_errors(err):
|
||||
logging.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
log.warning(
|
||||
"An error occurred handling a client request. Error message: %s",
|
||||
err.getErrorMessage())
|
||||
return err
|
||||
|
||||
def send_response(response):
|
||||
@@ -168,4 +183,4 @@ class ServerRequestHandler(object):
|
||||
msg = json.loads(request_buff)
|
||||
return msg
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -0,0 +1,23 @@
|
||||
import sqlite3
|
||||
from twisted.internet import task, reactor
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
|
||||
def rerun(err, *args, **kwargs):
|
||||
if err.check(sqlite3.OperationalError) and err.value.message == "database is locked":
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
return task.deferLater(reactor, 0, wrapper, *args, **kwargs)
|
||||
return err
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addErrback(rerun, *args, **kwargs)
|
||||
return d
|
||||
|
||||
return wrapper
|
||||
@@ -0,0 +1,46 @@
|
||||
import platform
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
from urllib2 import urlopen
|
||||
from lbryschema import __version__ as lbryschema_version
|
||||
from lbryum import __version__ as LBRYUM_VERSION
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from lbrynet.conf import ROOT_DIR
|
||||
|
||||
|
||||
def get_lbrynet_version():
|
||||
if build_type.BUILD == "dev":
|
||||
try:
|
||||
with open(os.devnull, 'w') as devnull:
|
||||
git_dir = ROOT_DIR + '/.git'
|
||||
return subprocess.check_output(
|
||||
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
|
||||
stderr=devnull
|
||||
).strip().lstrip('v')
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
print "failed to get version from git"
|
||||
return lbrynet_version
|
||||
|
||||
|
||||
def get_platform(get_ip=True):
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": get_lbrynet_version(),
|
||||
"lbryum_version": LBRYUM_VERSION,
|
||||
"lbryschema_version": lbryschema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
|
||||
if get_ip:
|
||||
try:
|
||||
p['ip'] = json.load(urlopen('http://jsonip.com'))['ip']
|
||||
except:
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
@@ -1,10 +1,62 @@
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
import base64
|
||||
import datetime
|
||||
import logging
|
||||
import random
|
||||
import socket
|
||||
import string
|
||||
import json
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from lbryschema.claim import ClaimDict
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2 # digest_size is in bytes, and blob hashes are hex encoded
|
||||
# defining these time functions here allows for easier overriding in testing
|
||||
def now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def utcnow():
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def isonow():
|
||||
"""Return utc now in isoformat with timezone"""
|
||||
return utcnow().isoformat() + 'Z'
|
||||
|
||||
|
||||
def today():
|
||||
return datetime.datetime.today()
|
||||
|
||||
|
||||
def timedelta(**kwargs):
|
||||
return datetime.timedelta(**kwargs)
|
||||
|
||||
|
||||
def datetime_obj(*args, **kwargs):
|
||||
return datetime.datetime(*args, **kwargs)
|
||||
|
||||
|
||||
def call_later(delay, func, *args, **kwargs):
|
||||
# Import here to ensure that it gets called after installing a reactor
|
||||
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
|
||||
from twisted.internet import reactor
|
||||
return reactor.callLater(delay, func, *args, **kwargs)
|
||||
|
||||
def safe_start_looping_call(looping_call, interval_sec):
|
||||
if not looping_call.running:
|
||||
looping_call.start(interval_sec)
|
||||
|
||||
def safe_stop_looping_call(looping_call):
|
||||
if looping_call.running:
|
||||
looping_call.stop()
|
||||
|
||||
def generate_id(num=None):
|
||||
h = get_lbry_hash_obj()
|
||||
if num is not None:
|
||||
@@ -14,15 +66,64 @@ def generate_id(num=None):
|
||||
return h.digest()
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: Whether the blobhash is the correct length and contains only valid characters (0-9, a-f)
|
||||
@return: True/False
|
||||
"""
|
||||
if len(blobhash) != blobhash_length:
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def version_is_greater_than(a, b):
|
||||
"""Returns True if version a is more recent than version b"""
|
||||
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
|
||||
|
||||
|
||||
def deobfuscate(obfustacated):
|
||||
return base64.b64decode(obfustacated.decode('rot13'))
|
||||
|
||||
|
||||
def obfuscate(plain):
|
||||
return base64.b64encode(plain).encode('rot13')
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80):
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
try:
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
host = socket.gethostbyname(server)
|
||||
s = socket.create_connection((host, port), 2)
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except Exception as ex:
|
||||
log.info(
|
||||
"Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port, exc_info=True)
|
||||
return False
|
||||
for l in blobhash:
|
||||
if l not in "0123456789abcdef":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
|
||||
def short_hash(hash_str):
|
||||
return hash_str[:6]
|
||||
|
||||
|
||||
def get_sd_hash(stream_info):
|
||||
if not stream_info:
|
||||
return None
|
||||
if isinstance(stream_info, ClaimDict):
|
||||
return stream_info.source_hash
|
||||
return stream_info['stream']['source']['source']
|
||||
|
||||
|
||||
def json_dumps_pretty(obj, **kwargs):
|
||||
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
|
||||
@@ -1,10 +1,13 @@
|
||||
import binascii
|
||||
import logging
|
||||
from Crypto.Cipher import AES
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
@@ -13,6 +16,15 @@ class CryptBlobInfo(BlobInfo):
|
||||
|
||||
class StreamBlobDecryptor(object):
|
||||
def __init__(self, blob, key, iv, length):
|
||||
"""
|
||||
This class decrypts blob
|
||||
|
||||
blob - object which implements read() function.
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
length = length in bytes of blob
|
||||
"""
|
||||
self.blob = blob
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
@@ -22,6 +34,12 @@ class StreamBlobDecryptor(object):
|
||||
self.cipher = AES.new(self.key, AES.MODE_CBC, self.iv)
|
||||
|
||||
def decrypt(self, write_func):
|
||||
"""
|
||||
Decrypt blob and write its content useing write_func
|
||||
|
||||
write_func - function that takes decrypted string as
|
||||
arugment and writes it somewhere
|
||||
"""
|
||||
|
||||
def remove_padding(data):
|
||||
pad_len = ord(data[-1])
|
||||
@@ -32,8 +50,8 @@ class StreamBlobDecryptor(object):
|
||||
|
||||
def write_bytes():
|
||||
if self.len_read < self.length:
|
||||
num_bytes_to_decrypt = (len(self.buff) // self.cipher.block_size) * self.cipher.block_size
|
||||
data_to_decrypt, self.buff = self.buff[:num_bytes_to_decrypt], self.buff[num_bytes_to_decrypt:]
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), self.cipher.block_size)
|
||||
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
|
||||
write_func(self.cipher.decrypt(data_to_decrypt))
|
||||
|
||||
def finish_decrypt():
|
||||
@@ -45,7 +63,6 @@ class StreamBlobDecryptor(object):
|
||||
self.buff += data
|
||||
self.len_read += len(data)
|
||||
write_bytes()
|
||||
#write_func(remove_padding(self.cipher.decrypt(self.buff)))
|
||||
|
||||
d = self.blob.read(decrypt_bytes)
|
||||
d.addCallback(lambda _: finish_decrypt())
|
||||
@@ -53,8 +70,16 @@ class StreamBlobDecryptor(object):
|
||||
|
||||
|
||||
class CryptStreamBlobMaker(object):
|
||||
"""This class encrypts data and writes it to a new blob"""
|
||||
def __init__(self, key, iv, blob_num, blob):
|
||||
"""
|
||||
This class encrypts data and writes it to a new blob
|
||||
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
blob = object which implements write(), close() function , close() function must
|
||||
be a deferred. (Will generally be of HashBlobCreator type)
|
||||
"""
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
self.blob_num = blob_num
|
||||
@@ -64,7 +89,15 @@ class CryptStreamBlobMaker(object):
|
||||
self.length = 0
|
||||
|
||||
def write(self, data):
|
||||
max_bytes_to_write = BLOB_SIZE - self.length - 1
|
||||
"""
|
||||
encrypt and write string data
|
||||
|
||||
Returns:
|
||||
tuple (done, num_bytes_to_write) where done is True if
|
||||
max bytes are written. num_bytes_to_write is the number
|
||||
of bytes that will be written from data in this call
|
||||
"""
|
||||
max_bytes_to_write = conf.settings['BLOB_SIZE'] - self.length - 1
|
||||
done = False
|
||||
if max_bytes_to_write <= len(data):
|
||||
num_bytes_to_write = max_bytes_to_write
|
||||
@@ -78,17 +111,17 @@ class CryptStreamBlobMaker(object):
|
||||
return done, num_bytes_to_write
|
||||
|
||||
def close(self):
|
||||
logging.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self._close_buffer()
|
||||
d = self.blob.close()
|
||||
d.addCallback(self._return_info)
|
||||
logging.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
return d
|
||||
|
||||
def _write_buffer(self):
|
||||
num_bytes_to_encrypt = (len(self.buff) // AES.block_size) * AES.block_size
|
||||
data_to_encrypt, self.buff = self.buff[:num_bytes_to_encrypt], self.buff[num_bytes_to_encrypt:]
|
||||
data_to_encrypt, self.buff = split(self.buff, num_bytes_to_encrypt)
|
||||
encrypted_data = self.cipher.encrypt(data_to_encrypt)
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
@@ -103,4 +136,13 @@ class CryptStreamBlobMaker(object):
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
def _return_info(self, blob_hash):
|
||||
return CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
return CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
|
||||
|
||||
def greatest_multiple(a, b):
|
||||
"""return the largest value `c`, that is a multiple of `b` and is <= `a`"""
|
||||
return (a // b) * b
|
||||
|
||||
|
||||
def split(buff, cutoff):
|
||||
return buff[:cutoff], buff[cutoff:]
|
||||
@@ -12,25 +12,29 @@ from lbrynet.core.StreamCreator import StreamCreator
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(StreamCreator):
|
||||
"""Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
Each blob is encrypted with the same key, but each blob has its own initialization vector
|
||||
which is associated with the blob when the blob is associated with the stream."""
|
||||
Each blob is encrypted with the same key, but each blob has its
|
||||
own initialization vector which is associated with the blob when
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""
|
||||
@param blob_manager: Object that stores and provides access to blobs.
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@type blob_manager: BlobManager
|
||||
|
||||
@param name: the name of the stream, which will be presented to the user
|
||||
@type name: string
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
|
||||
be generated.
|
||||
@param key: the raw AES key which will be used to encrypt the
|
||||
blobs. If None, a random key will be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
|
||||
once for each blob.
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: None
|
||||
@@ -57,22 +61,20 @@ class CryptStreamCreator(StreamCreator):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _finalize(self):
|
||||
logging.debug("_finalize has been called")
|
||||
log.debug("_finalize has been called")
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
final_blob_creator = self.blob_manager.get_blob_creator()
|
||||
logging.debug("Created the finished_deferred")
|
||||
log.debug("Created the finished_deferred")
|
||||
final_blob = self._get_blob_maker(iv, final_blob_creator)
|
||||
logging.debug("Created the final blob")
|
||||
logging.debug("Calling close on final blob")
|
||||
log.debug("Created the final blob")
|
||||
log.debug("Calling close on final blob")
|
||||
d = final_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
self.finished_deferreds.append(d)
|
||||
logging.debug("called close on final blob, returning from make_final_blob")
|
||||
return d
|
||||
log.debug("called close on final blob, returning from make_final_blob")
|
||||
|
||||
def _write(self, data):
|
||||
|
||||
def close_blob(blob):
|
||||
d = blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
@@ -91,4 +93,4 @@ class CryptStreamCreator(StreamCreator):
|
||||
self.current_blob = None
|
||||
|
||||
def _get_blob_maker(self, iv, blob_creator):
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
@@ -5,4 +5,4 @@ Crypt Streams are encrypted blobs and metadata tying those blobs together. At le
|
||||
metadata is generally stored in a Stream Descriptor File, for example containing a public key
|
||||
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
|
||||
may or may not be present.
|
||||
"""
|
||||
"""
|
||||
@@ -14,6 +14,7 @@ class CryptBlobHandler(object):
|
||||
######## IBlobHandler #########
|
||||
|
||||
def handle_blob(self, blob, blob_info):
|
||||
blob_decryptor = StreamBlobDecryptor(blob, self.key, binascii.unhexlify(blob_info.iv), blob_info.length)
|
||||
blob_decryptor = StreamBlobDecryptor(
|
||||
blob, self.key, binascii.unhexlify(blob_info.iv), blob_info.length)
|
||||
d = blob_decryptor.decrypt(self.write_func)
|
||||
return d
|
||||
return d
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IStreamDownloader
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
@@ -9,6 +10,9 @@ from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartFailedError(Exception):
|
||||
pass
|
||||
|
||||
@@ -34,21 +38,22 @@ class CryptStreamDownloader(object):
|
||||
implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, upload_allowed):
|
||||
"""
|
||||
Initialize a CryptStreamDownloader
|
||||
payment_rate_manager, wallet):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@param peer_finder: An object which implements the IPeerFinder interface. Used to look up peers by a hashsum.
|
||||
@param peer_finder: An object which implements the IPeerFinder
|
||||
interface. Used to look up peers by a hashsum.
|
||||
|
||||
@param rate_limiter: An object which implements the IRateLimiter interface
|
||||
|
||||
@param blob_manager: A BlobManager object
|
||||
|
||||
@param payment_rate_manager: A PaymentRateManager object
|
||||
@param payment_rate_manager: A NegotiatedPaymentRateManager object
|
||||
|
||||
@param wallet: An object which implements the ILBRYWallet interface
|
||||
@param wallet: An object which implements the IWallet interface
|
||||
|
||||
@return:
|
||||
|
||||
"""
|
||||
|
||||
self.peer_finder = peer_finder
|
||||
@@ -56,20 +61,19 @@ class CryptStreamDownloader(object):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.upload_allowed = upload_allowed
|
||||
|
||||
self.key = None
|
||||
self.stream_name = None
|
||||
|
||||
self.completed = False
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
self.starting = False
|
||||
|
||||
self.download_manager = None
|
||||
self.finished_deferred = None
|
||||
|
||||
self.points_paid = 0.0
|
||||
self.blob_requester = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stream_name)
|
||||
|
||||
def toggle_running(self):
|
||||
if self.stopped is True:
|
||||
@@ -78,11 +82,6 @@ class CryptStreamDownloader(object):
|
||||
return self.stop()
|
||||
|
||||
def start(self):
|
||||
|
||||
def set_finished_deferred():
|
||||
self.finished_deferred = defer.Deferred()
|
||||
return self.finished_deferred
|
||||
|
||||
if self.starting is True:
|
||||
raise CurrentlyStartingError()
|
||||
if self.stopping is True:
|
||||
@@ -92,29 +91,25 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is None
|
||||
self.starting = True
|
||||
self.completed = False
|
||||
self.finished_deferred = defer.Deferred()
|
||||
d = self._start()
|
||||
d.addCallback(lambda _: set_finished_deferred())
|
||||
d.addCallback(lambda _: self.finished_deferred)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
|
||||
def check_if_stop_succeeded(success):
|
||||
self.stopping = False
|
||||
if success is True:
|
||||
self.stopped = True
|
||||
self._remove_download_manager()
|
||||
return success
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None):
|
||||
if self.stopped is True:
|
||||
raise AlreadyStoppedError()
|
||||
if self.stopping is True:
|
||||
raise CurrentlyStoppingError()
|
||||
assert self.download_manager is not None
|
||||
self.stopping = True
|
||||
d = self.download_manager.stop_downloading()
|
||||
self._fire_completed_deferred()
|
||||
d.addCallback(check_if_stop_succeeded)
|
||||
return d
|
||||
success = yield self.download_manager.stop_downloading()
|
||||
self.stopping = False
|
||||
if success is True:
|
||||
self.stopped = True
|
||||
self._remove_download_manager()
|
||||
yield self._fire_completed_deferred(err)
|
||||
|
||||
def _start_failed(self):
|
||||
|
||||
@@ -149,28 +144,32 @@ class CryptStreamDownloader(object):
|
||||
return d
|
||||
|
||||
def _get_download_manager(self):
|
||||
download_manager = DownloadManager(self.blob_manager, self.upload_allowed)
|
||||
assert self.blob_requester is None
|
||||
download_manager = DownloadManager(self.blob_manager)
|
||||
# TODO: can we get rid of these circular references. I'm not
|
||||
# smart enough to handle thinking about the interactions
|
||||
# between them and have hope that there is a simpler way
|
||||
# to accomplish what we want
|
||||
download_manager.blob_info_finder = self._get_metadata_handler(download_manager)
|
||||
download_manager.blob_requester = self._get_blob_requester(download_manager)
|
||||
download_manager.progress_manager = self._get_progress_manager(download_manager)
|
||||
download_manager.blob_handler = self._get_blob_handler(download_manager)
|
||||
download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
|
||||
# blob_requester needs to be set before the connection manager is setup
|
||||
self.blob_requester = self._get_blob_requester(download_manager)
|
||||
download_manager.connection_manager = self._get_connection_manager(download_manager)
|
||||
#return DownloadManager(self.blob_manager, self.blob_requester, self.metadata_handler,
|
||||
# self.progress_manager, self.blob_handler, self.connection_manager)
|
||||
return download_manager
|
||||
|
||||
def _remove_download_manager(self):
|
||||
self.download_manager.blob_info_finder = None
|
||||
self.download_manager.blob_requester = None
|
||||
self.download_manager.progress_manager = None
|
||||
self.download_manager.blob_handler = None
|
||||
self.download_manager.wallet_info_exchanger = None
|
||||
self.blob_requester = None
|
||||
self.download_manager.connection_manager = None
|
||||
self.download_manager = None
|
||||
|
||||
def _get_primary_request_creators(self, download_manager):
|
||||
return [download_manager.blob_requester]
|
||||
return [self.blob_requester]
|
||||
|
||||
def _get_secondary_request_creators(self, download_manager):
|
||||
return [download_manager.wallet_info_exchanger]
|
||||
@@ -179,11 +178,13 @@ class CryptStreamDownloader(object):
|
||||
pass
|
||||
|
||||
def _get_blob_requester(self, download_manager):
|
||||
return BlobRequester(self.blob_manager, self.peer_finder, self.payment_rate_manager, self.wallet,
|
||||
return BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
download_manager)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _get_write_func(self):
|
||||
pass
|
||||
@@ -196,10 +197,16 @@ class CryptStreamDownloader(object):
|
||||
self._get_primary_request_creators(download_manager),
|
||||
self._get_secondary_request_creators(download_manager))
|
||||
|
||||
def _fire_completed_deferred(self):
|
||||
def _fire_completed_deferred(self, err=None):
|
||||
self.finished_deferred, d = None, self.finished_deferred
|
||||
if d is not None:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
if err is not None:
|
||||
d.errback(err)
|
||||
else:
|
||||
value = self._get_finished_deferred_callback_value()
|
||||
d.callback(value)
|
||||
else:
|
||||
log.debug("Not firing the completed deferred because d is None")
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
return None
|
||||
@@ -209,5 +216,5 @@ class CryptStreamDownloader(object):
|
||||
self.completed = True
|
||||
return self.stop()
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop(err=err)
|
||||