Compare commits
2241
Commits
alpha
...
v0.9.2rc24
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b11521868 | ||
|
|
c26f83d81f | ||
|
|
e6f92a2dc7 | ||
|
|
4698910ab7 | ||
|
|
8c361aef1a | ||
|
|
2c8b99df76 | ||
|
|
10c72dcfd2 | ||
|
|
10dcc429f3 | ||
|
|
31b5026570 | ||
|
|
f38120fdfc | ||
|
|
06cd14e0ff | ||
|
|
3820401fce | ||
|
|
ddf907231d | ||
|
|
2df1092e1a | ||
|
|
d1414bc03a | ||
|
|
c9e3f1fd02 | ||
|
|
33c754f5a6 | ||
|
|
9669e06279 | ||
|
|
389ee2ad70 | ||
|
|
c87c8beeb4 | ||
|
|
f53cde824e | ||
|
|
63142df865 | ||
|
|
023f81674c | ||
|
|
b644125e93 | ||
|
|
c5699db30a | ||
|
|
c1525760d8 | ||
|
|
b773cbbec6 | ||
|
|
043e2d0ab9 | ||
|
|
16221d08a9 | ||
|
|
6ccab8bcb8 | ||
|
|
a87834bacc | ||
|
|
105fa0b947 | ||
|
|
8db7c37fa7 | ||
|
|
54c29d4a8d | ||
|
|
a2eb0cad33 | ||
|
|
4c122f4ab0 | ||
|
|
a43127fffc | ||
|
|
66ab1386c5 | ||
|
|
22381f9427 | ||
|
|
5efbafec39 | ||
|
|
0ff308e3eb | ||
|
|
9531bdf396 | ||
|
|
978139a631 | ||
|
|
86117c431b | ||
|
|
585a17c830 | ||
|
|
04c119543f | ||
|
|
33c4856f00 | ||
|
|
ead32f32d8 | ||
|
|
e54dc9a01d | ||
|
|
c0ccfe2046 | ||
|
|
40374ec202 | ||
|
|
6ed2c5831e | ||
|
|
dc0699b929 | ||
|
|
eb1bdcc312 | ||
|
|
e9cfbea75f | ||
|
|
ccc94a0db9 | ||
|
|
bcf7a28fc8 | ||
|
|
1880f64da2 | ||
|
|
45b3837638 | ||
|
|
ee7ca0978f | ||
|
|
fc3573f823 | ||
|
|
8e15eddf9b | ||
|
|
880b974b84 | ||
|
|
5b78215063 | ||
|
|
bf7882d393 | ||
|
|
7a798616d6 | ||
|
|
6f909f9654 | ||
|
|
528ce42821 | ||
|
|
931038d478 | ||
|
|
60d6666abf | ||
|
|
ca041b5dc4 | ||
|
|
f43b38a5ea | ||
|
|
769329c90e | ||
|
|
23596b434a | ||
|
|
22f57f6490 | ||
|
|
99e4f9b00b | ||
|
|
bee67117c8 | ||
|
|
502e2227b5 | ||
|
|
d3173b157e | ||
|
|
c0a40605ad | ||
|
|
c20ba29c7a | ||
|
|
850f51140e | ||
|
|
5ec891c9ac | ||
|
|
5e936ef3cc | ||
|
|
d23e3cf066 | ||
|
|
bacc3a0b3a | ||
|
|
32e2ee8bf5 | ||
|
|
beeaec986a | ||
|
|
e32aa928ac | ||
|
|
1a562d70aa | ||
|
|
8dbadbd75b | ||
|
|
370a91fdaa | ||
|
|
1916b990b7 | ||
|
|
84d9461deb | ||
|
|
9e5ac6f73e | ||
|
|
5dbe2fdcec | ||
|
|
eabf9f1c4a | ||
|
|
bb4bf620f4 | ||
|
|
7abfbc4c5f | ||
|
|
b2d0c53bbe | ||
|
|
ddd8fd1a77 | ||
|
|
3af5e9dc5c | ||
|
|
23f1cf9c82 | ||
|
|
26f40b62f9 | ||
|
|
b15fba9c34 | ||
|
|
b0fd6e57df | ||
|
|
b04455f9c7 | ||
|
|
b4f5f2068f | ||
|
|
948d606268 | ||
|
|
f5c533e2f1 | ||
|
|
a0a122f0ef | ||
|
|
a2a29fc02c | ||
|
|
c9c84784c9 | ||
|
|
8fa2524e54 | ||
|
|
41257c07e6 | ||
|
|
e7b90055ed | ||
|
|
3f08b5f711 | ||
|
|
f0e7f38519 | ||
|
|
12b428f02d | ||
|
|
7708c4a15c | ||
|
|
9410cd9e77 | ||
|
|
0c42bc6382 | ||
|
|
b4364f661c | ||
|
|
63c71760dd | ||
|
|
25d9f008de | ||
|
|
41fbb1399c | ||
|
|
2cc2505cfd | ||
|
|
7fac63a37b | ||
|
|
c18c1725c4 | ||
|
|
5e6a742bd3 | ||
|
|
790baea191 | ||
|
|
e8e2a0d353 | ||
|
|
da3ed0756e | ||
|
|
05f7f5c73c | ||
|
|
b19e3adadd | ||
|
|
824f2f1d0f | ||
|
|
f515583ec4 | ||
|
|
c1a5806bc0 | ||
|
|
c66225ccf8 | ||
|
|
8f3b53197f | ||
|
|
4027733f2c | ||
|
|
3d606b95aa | ||
|
|
85845414b8 | ||
|
|
a95c49bd08 | ||
|
|
c422b5252e | ||
|
|
cb4c93bcd9 | ||
|
|
4edd13f4f4 | ||
|
|
a4de0056ab | ||
|
|
ee77bb0670 | ||
|
|
6482619fac | ||
|
|
b00c20180b | ||
|
|
001ebd02e3 | ||
|
|
15517732ae | ||
|
|
ebb6813df3 | ||
|
|
bfd822782d | ||
|
|
e66a1d3577 | ||
|
|
61d27f010e | ||
|
|
52d471d660 | ||
|
|
33c9a7a098 | ||
|
|
8201fc186d | ||
|
|
9d7dcf4af7 | ||
|
|
49b18a1fca | ||
|
|
26d2347812 | ||
|
|
29dea4901e | ||
|
|
1c2290dc8e | ||
|
|
8c664a939b | ||
|
|
12bc804622 | ||
|
|
593e8a94ad | ||
|
|
2c4e4d4526 | ||
|
|
74844f18ad | ||
|
|
93406fe07f | ||
|
|
820e3091bf | ||
|
|
c4b65fddac | ||
|
|
25dcf592a0 | ||
|
|
02bae15b02 | ||
|
|
b32b7091ad | ||
|
|
28e662a53e | ||
|
|
a440656a33 | ||
|
|
4e765b265f | ||
|
|
ec35eb2397 | ||
|
|
be4e1c99d5 | ||
|
|
031d9c28ed | ||
|
|
6d53a582cf | ||
|
|
64668d0043 | ||
|
|
92d423aefd | ||
|
|
0ec9b6545f | ||
|
|
003b3df378 | ||
|
|
f409d43dcc | ||
|
|
b2c2edfb12 | ||
|
|
e7dc841bb7 | ||
|
|
2303f88bd4 | ||
|
|
7291efac83 | ||
|
|
8c61185387 | ||
|
|
371591f96e | ||
|
|
98d3a4cb46 | ||
|
|
ab1bdb7eaf | ||
|
|
8ef3f81f0c | ||
|
|
cf56d8fa4a | ||
|
|
881efabc0c | ||
|
|
aef274577f | ||
|
|
5d6ca0aae6 | ||
|
|
9a4230dd96 | ||
|
|
3f4a4728e6 | ||
|
|
63cb2cf571 | ||
|
|
54b4b6984e | ||
|
|
98f90a4ad6 | ||
|
|
e29a91b69a | ||
|
|
fdcc5e6829 | ||
|
|
da99f3f22b | ||
|
|
992be881af | ||
|
|
a0cbd5af33 | ||
|
|
68f5588bd9 | ||
|
|
531fa63590 | ||
|
|
9048244cdc | ||
|
|
74754c9870 | ||
|
|
a7c2ad470c | ||
|
|
a132496643 | ||
|
|
f600038712 | ||
|
|
eae8f8a00e | ||
|
|
ac33663444 | ||
|
|
ba8b7ca090 | ||
|
|
1aa715c927 | ||
|
|
931925be15 | ||
|
|
e2ed9f21d2 | ||
|
|
e6cd4c9877 | ||
|
|
e2cc212eb8 | ||
|
|
74cf14df48 | ||
|
|
18c690d801 | ||
|
|
9e09d1fc38 | ||
|
|
454755169a | ||
|
|
bc9b990728 | ||
|
|
7ad46e70be | ||
|
|
4f9b5c4624 | ||
|
|
9744287370 | ||
|
|
59a5a3164f | ||
|
|
6fb3313eaf | ||
|
|
2bfbae9d4e | ||
|
|
daa0ff036d | ||
|
|
d642ba039d | ||
|
|
9350b9e127 | ||
|
|
9755e9b11f | ||
|
|
a64424f28c | ||
|
|
eee1acb24a | ||
|
|
0e76ca0767 | ||
|
|
e8afad7af8 | ||
|
|
f9642018ae | ||
|
|
41d96fec00 | ||
|
|
88f270180c | ||
|
|
76df0e3940 | ||
|
|
7128b6401e | ||
|
|
824fbcc25a | ||
|
|
fba91139e3 | ||
|
|
090840ee25 | ||
|
|
d3d8426430 | ||
|
|
40d4fb5641 | ||
|
|
d986fc27f3 | ||
|
|
9af1cae249 | ||
|
|
056ee2d1ce | ||
|
|
301e4ae0fa | ||
|
|
d359949a5c | ||
|
|
d4900fd607 | ||
|
|
f596d42ca2 | ||
|
|
04b36e2252 | ||
|
|
2add545ab6 | ||
|
|
70cf179139 | ||
|
|
b6086a7d6d | ||
|
|
62ab4c769d | ||
|
|
c920663444 | ||
|
|
1d30ae447a | ||
|
|
ff5980dfca | ||
|
|
e4b77a4a0a | ||
|
|
666b568eea | ||
|
|
998892ef0b | ||
|
|
847a79f526 | ||
|
|
07841a7803 | ||
|
|
4e55d674da | ||
|
|
3522f9af7d | ||
|
|
c27fe7af31 | ||
|
|
30c7559d3a | ||
|
|
7e2fad2397 | ||
|
|
06466db1b0 | ||
|
|
213a7ab66c | ||
|
|
a1650d7f03 | ||
|
|
5737d26ffe | ||
|
|
c85226d5a6 | ||
|
|
37b2297fb7 | ||
|
|
450ac7aa57 | ||
|
|
68e7c66e3a | ||
|
|
55f4e2939a | ||
|
|
b18679af53 | ||
|
|
d0b64ef255 | ||
|
|
e2f97ba0b4 | ||
|
|
c6e614ebec | ||
|
|
76b4b1e939 | ||
|
|
29ff39fdc9 | ||
|
|
d2e1eeb61e | ||
|
|
e86bada307 | ||
|
|
59750843d4 | ||
|
|
ab46a33913 | ||
|
|
5bf5a7d0b1 | ||
|
|
23f2a42372 | ||
|
|
14b4c06969 | ||
|
|
6d4c606abd | ||
|
|
eb039201e8 | ||
|
|
723105157f | ||
|
|
17d32ca696 | ||
|
|
0eb8e1609f | ||
|
|
85390302e2 | ||
|
|
993f2e54f9 | ||
|
|
87b7e5c742 | ||
|
|
8b9fef4ed5 | ||
|
|
d4cd8308f1 | ||
|
|
73eebdb0e3 | ||
|
|
0c30655605 | ||
|
|
e1455c3a0d | ||
|
|
205fd66a18 | ||
|
|
705b65fa3c | ||
|
|
fb2823f59e | ||
|
|
ddfaf51393 | ||
|
|
f1231bd8e5 | ||
|
|
19aa1eebd5 | ||
|
|
24e7b7797c | ||
|
|
caa8f73f7c | ||
|
|
0674b48223 | ||
|
|
5bf75ef139 | ||
|
|
6e310517e0 | ||
|
|
e2db99f7ab | ||
|
|
3151c10556 | ||
|
|
3ddf8680fb | ||
|
|
25ec8fde23 | ||
|
|
bcd026a1b6 | ||
|
|
1a6290eacc | ||
|
|
f4217d5593 | ||
|
|
8bba5c04db | ||
|
|
345301a9ee | ||
|
|
0f907a209b | ||
|
|
03c0ff8529 | ||
|
|
80dd02483e | ||
|
|
8452620f99 | ||
|
|
287e62b71b | ||
|
|
934a46a8b6 | ||
|
|
23e06ba872 | ||
|
|
1fd87b7caa | ||
|
|
f7c105fd80 | ||
|
|
9e2cda0cdc | ||
|
|
4fb5e932e6 | ||
|
|
fe4dac8236 | ||
|
|
2a45c91175 | ||
|
|
439a840700 | ||
|
|
a38e5c855e | ||
|
|
90d3a0ef9c | ||
|
|
f989814e69 | ||
|
|
9f4ccfe7c3 | ||
|
|
520ac9b50a | ||
|
|
dbb6c6f3a4 | ||
|
|
993d606bc6 | ||
|
|
ffb48ef287 | ||
|
|
62fcbc23e3 | ||
|
|
a43b726f32 | ||
|
|
3bdd33567b | ||
|
|
6bdb0280be | ||
|
|
77746cea5f | ||
|
|
25cede0dc3 | ||
|
|
acfe034990 | ||
|
|
e2021deae1 | ||
|
|
37cc257555 | ||
|
|
a22758d5c7 | ||
|
|
aa8375673b | ||
|
|
5831e253cd | ||
|
|
34f5bc93ae | ||
|
|
aa45b0e121 | ||
|
|
8e0f71ac74 | ||
|
|
55445b1bb7 | ||
|
|
3cc8a4b168 | ||
|
|
8859c1b0c1 | ||
|
|
867275461c | ||
|
|
2190f4ac85 | ||
|
|
b51d3a28e3 | ||
|
|
7720724ec0 | ||
|
|
dbd71c93e7 | ||
|
|
d137528f67 | ||
|
|
e292abceee | ||
|
|
8de6bd7c7a | ||
|
|
bb9b221cf6 | ||
|
|
5a1bab2eaa | ||
|
|
4f87e87d3e | ||
|
|
c7106dfc97 | ||
|
|
11b1f8c7af | ||
|
|
f5e6ff0c32 | ||
|
|
71123c0de1 | ||
|
|
35b2e56bd4 | ||
|
|
41abd0a171 | ||
|
|
cea4eca775 | ||
|
|
6b22debb79 | ||
|
|
630d0bf5be | ||
|
|
19fe101d73 | ||
|
|
75cc0dcce9 | ||
|
|
0e8e489ad6 | ||
|
|
b03e7e14f3 | ||
|
|
e2050b4fc1 | ||
|
|
d1aa69bfa3 | ||
|
|
7e2456fa69 | ||
|
|
ecac2ae2ca | ||
|
|
bfc02dd3e5 | ||
|
|
c527c32e5f | ||
|
|
777419b4a9 | ||
|
|
832a32474f | ||
|
|
f2ddc9bd98 | ||
|
|
d053db8dfd | ||
|
|
a58a81a841 | ||
|
|
df40c7fb76 | ||
|
|
ef65d8dd1c | ||
|
|
a8780f03a8 | ||
|
|
61e78c1b69 | ||
|
|
a167beaad1 | ||
|
|
4c76e1b98f | ||
|
|
f614a13db4 | ||
|
|
6fae07d29e | ||
|
|
2126f69c93 | ||
|
|
6f0581104a | ||
|
|
47c92f04d2 | ||
|
|
06be60e3a3 | ||
|
|
45a6228b58 | ||
|
|
773cbe3ace | ||
|
|
c6648ff6bc | ||
|
|
f737b3aaf7 | ||
|
|
41d937d1b4 | ||
|
|
8e349f17c1 | ||
|
|
4ab5ace668 | ||
|
|
f46ae75f73 | ||
|
|
61c5a5b2a4 | ||
|
|
2c137a99d1 | ||
|
|
d1ed8a769a | ||
|
|
4bb3e2a154 | ||
|
|
a5e8c561e4 | ||
|
|
7b6e233d22 | ||
|
|
ac8a63bc33 | ||
|
|
15ebc39f95 | ||
|
|
46cb6b204a | ||
|
|
b219806959 | ||
|
|
8fc6feb316 | ||
|
|
7377158631 | ||
|
|
86ba7ffd5c | ||
|
|
ed227a239e | ||
|
|
7360015fa4 | ||
|
|
04ee2f786d | ||
|
|
43451c485f | ||
|
|
565f26bdad | ||
|
|
b7478ba817 | ||
|
|
1c92b5d2ea | ||
|
|
b744a54340 | ||
|
|
8805d1d0cf | ||
|
|
b674e5c57a | ||
|
|
139af85b52 | ||
|
|
ee0f7375d6 | ||
|
|
893fe8823e | ||
|
|
f54d42b8fa | ||
|
|
3dc1a523cf | ||
|
|
90438ddf9f | ||
|
|
08e336e476 | ||
|
|
eebf32362a | ||
|
|
0e864f5df1 | ||
|
|
854dea797e | ||
|
|
c88631d910 | ||
|
|
897fba35df | ||
|
|
74b4e4508a | ||
|
|
74a872d763 | ||
|
|
3b65f5962c | ||
|
|
5fa2dfeca7 | ||
|
|
7cac8dad7a | ||
|
|
b80a1f66da | ||
|
|
20da97de6a | ||
|
|
bb9d3bff74 | ||
|
|
ec873129d5 | ||
|
|
a90f413f94 | ||
|
|
db1abd8cca | ||
|
|
18a0b07181 | ||
|
|
8fd7489d99 | ||
|
|
ee84253b90 | ||
|
|
06e45ee32a | ||
|
|
7696f30009 | ||
|
|
c211d87164 | ||
|
|
dc57b62157 | ||
|
|
c71ac5b535 | ||
|
|
5a467f55ba | ||
|
|
9a24bddead | ||
|
|
d3a63c38fc | ||
|
|
08aa024499 | ||
|
|
5db1a4d328 | ||
|
|
e95adecd41 | ||
|
|
8b68a752ed | ||
|
|
51a5510b38 | ||
|
|
2c9c0da501 | ||
|
|
60195ed6d4 | ||
|
|
e37fbc4fdb | ||
|
|
3e774fc158 | ||
|
|
c62ee6bb0c | ||
|
|
3aea476014 | ||
|
|
8c5f7a9849 | ||
|
|
6bc3e8a3f4 | ||
|
|
6c571b5227 | ||
|
|
8628b127f7 | ||
|
|
91d673a539 | ||
|
|
851ab7ba28 | ||
|
|
7bc1bc487f | ||
|
|
05d50c1821 | ||
|
|
40b6476cf7 | ||
|
|
13bfcb3561 | ||
|
|
1a850b0256 | ||
|
|
78b0f923f4 | ||
|
|
1aa7981ab1 | ||
|
|
17d9c0ab97 | ||
|
|
8d1404bd1e | ||
|
|
4634179a7e | ||
|
|
23872c77e9 | ||
|
|
cac8267e85 | ||
|
|
267c6cbaca | ||
|
|
03b214a9d2 | ||
|
|
8a026071eb | ||
|
|
9fb1cd0944 | ||
|
|
a338d1f0e1 | ||
|
|
8a338b37b2 | ||
|
|
2b2fa29db5 | ||
|
|
ddcfa7f97c | ||
|
|
ddf24de16d | ||
|
|
8277df1034 | ||
|
|
3bed768d10 | ||
|
|
e03bba0b62 | ||
|
|
e61260a263 | ||
|
|
a9261b2032 | ||
|
|
9366a6e561 | ||
|
|
488c04543d | ||
|
|
cb2bb6ee6b | ||
|
|
0bb62515a8 | ||
|
|
f535d96929 | ||
|
|
15c5075d38 | ||
|
|
2449604844 | ||
|
|
6035a84675 | ||
|
|
1151019186 | ||
|
|
8075ced1af | ||
|
|
1b2c46e224 | ||
|
|
7f6e9fff30 | ||
|
|
bb1b7b66f7 | ||
|
|
deae84a563 | ||
|
|
20f113d3da | ||
|
|
61702ba1e4 | ||
|
|
d100a24a29 | ||
|
|
d06211015d | ||
|
|
93ff4f63b5 | ||
|
|
86aa8e1de0 | ||
|
|
495a48add1 | ||
|
|
a6457d2c0a | ||
|
|
5c429e15bd | ||
|
|
05725e0dc9 | ||
|
|
53acb2d9cd | ||
|
|
f050337dbc | ||
|
|
6d7b09a3aa | ||
|
|
dd87aa3ce6 | ||
|
|
b602b09ca3 | ||
|
|
628defa891 | ||
|
|
962edb8e4c | ||
|
|
5dbf9034ec | ||
|
|
a67aab0797 | ||
|
|
52500bfbf5 | ||
|
|
f4790a46ee | ||
|
|
fd5b4f403a | ||
|
|
c393bd97e8 | ||
|
|
d32bb9270a | ||
|
|
9e7b69f4a0 | ||
|
|
3c60b35e2e | ||
|
|
69c0da695b | ||
|
|
68887d145f | ||
|
|
f37808859f | ||
|
|
14c835b41e | ||
|
|
abc122f77b | ||
|
|
0ababb7f3b | ||
|
|
30af8a0e20 | ||
|
|
389fc67d51 | ||
|
|
c2fc15160f | ||
|
|
40d4f5a292 | ||
|
|
38e6b4e6c6 | ||
|
|
3541625825 | ||
|
|
f8f83fccf1 | ||
|
|
b3175af16f | ||
|
|
5f8a06443c | ||
|
|
4e0194d576 | ||
|
|
16fc416d88 | ||
|
|
6faee910de | ||
|
|
f7308beee9 | ||
|
|
5233dc303e | ||
|
|
3d0913ca94 | ||
|
|
c37bb0d1ed | ||
|
|
a4a9a2b0b0 | ||
|
|
9e320921a9 | ||
|
|
21602dc5a1 | ||
|
|
2cf08fda2d | ||
|
|
c2c7d2b327 | ||
|
|
585f802a91 | ||
|
|
26cf04e0c1 | ||
|
|
3bb8f4ac21 | ||
|
|
2afc4c25a2 | ||
|
|
a5c25ff887 | ||
|
|
e823bb2012 | ||
|
|
eec25791a5 | ||
|
|
18a47a048a | ||
|
|
6936dbd588 | ||
|
|
621530b3b8 | ||
|
|
123fda1844 | ||
|
|
1299c23bcc | ||
|
|
1747f53276 | ||
|
|
669a567545 | ||
|
|
6bdc9069c9 | ||
|
|
ad1e2650e3 | ||
|
|
1bf1b7624c | ||
|
|
0b53fde352 | ||
|
|
4eb10b56c1 | ||
|
|
227323b7a4 | ||
|
|
1256beea96 | ||
|
|
01cc4f28e0 | ||
|
|
689ac431d3 | ||
|
|
ccfb1f3921 | ||
|
|
cc008766ec | ||
|
|
4b1530b554 | ||
|
|
5eecae8889 | ||
|
|
5aa10031bc | ||
|
|
52010ade41 | ||
|
|
101985ca11 | ||
|
|
8f61fb3fc4 | ||
|
|
39b2e44492 | ||
|
|
26bc6d05a0 | ||
|
|
47f71fd363 | ||
|
|
4574e3317c | ||
|
|
b59c098d58 | ||
|
|
22ff87b885 | ||
|
|
42edf49468 | ||
|
|
da0b4e9a1a | ||
|
|
7d5dd1ff51 | ||
|
|
d1ccac1b04 | ||
|
|
a289886365 | ||
|
|
2c7fc3fb46 | ||
|
|
fc24d9b5aa | ||
|
|
da366495b0 | ||
|
|
552ed77882 | ||
|
|
c53e0baca6 | ||
|
|
aa3353ae49 | ||
|
|
521ad57b61 | ||
|
|
f62456bb68 | ||
|
|
f7864e8cb7 | ||
|
|
1690e71ad1 | ||
|
|
7d02e7d370 | ||
|
|
7404b127bf | ||
|
|
10810226ff | ||
|
|
a2497e0854 | ||
|
|
c0f22decd7 | ||
|
|
3da82ce895 | ||
|
|
ba90fb3349 | ||
|
|
618d0ebd5b | ||
|
|
f301741807 | ||
|
|
0044d6cf75 | ||
|
|
5a76a03cb6 | ||
|
|
09846413bd | ||
|
|
a8cb255d24 | ||
|
|
6172924021 | ||
|
|
7bb570c051 | ||
|
|
d5d0d0e31c | ||
|
|
83e797d7f6 | ||
|
|
6c2b3d9ed6 | ||
|
|
c6f4c17597 | ||
|
|
d19afd856b | ||
|
|
29e59e783a | ||
|
|
107860ea19 | ||
|
|
a9bfd8d6ae | ||
|
|
9ee95d57fc | ||
|
|
4a8a8b8d33 | ||
|
|
c825d88dbf | ||
|
|
15ade508aa | ||
|
|
53df361d23 | ||
|
|
8fe15f507b | ||
|
|
3dde7af576 | ||
|
|
7af6e9e0dd | ||
|
|
56d394fb5f | ||
|
|
3585d861ff | ||
|
|
f181af8547 | ||
|
|
84baa5e065 | ||
|
|
781b915717 | ||
|
|
fa29c92760 | ||
|
|
fec917b9c1 | ||
|
|
4f3b5cd802 | ||
|
|
250831a86a | ||
|
|
323bccb0ae | ||
|
|
0084d4684f | ||
|
|
740fad5cbe | ||
|
|
fa8190356a | ||
|
|
4ca33505b5 | ||
|
|
20b6b22334 | ||
|
|
b8c91c61bb | ||
|
|
c30ea04959 | ||
|
|
27a09d4a4f | ||
|
|
23bf43f40c | ||
|
|
6f60b8b827 | ||
|
|
965f8f1238 | ||
|
|
ab78b7f5fa | ||
|
|
083092ad5e | ||
|
|
7bce37a720 | ||
|
|
14ac2bac39 | ||
|
|
32fa2460ff | ||
|
|
f796f701f2 | ||
|
|
fe4ea9b33a | ||
|
|
c278972f26 | ||
|
|
bce41ddab4 | ||
|
|
666a62ae3d | ||
|
|
5bf20096b2 | ||
|
|
52099c7da7 | ||
|
|
9619b8dfca | ||
|
|
e8faadc324 | ||
|
|
667d5ac622 | ||
|
|
afba045c29 | ||
|
|
b64fa51567 | ||
|
|
fe3e25982c | ||
|
|
9ad55e93b9 | ||
|
|
9e7e168175 | ||
|
|
f84df50d7f | ||
|
|
f0ed4e5513 | ||
|
|
0af445cbbc | ||
|
|
99fe28366f | ||
|
|
39005ffc14 | ||
|
|
239e5c2126 | ||
|
|
28e88d89a5 | ||
|
|
c85b795fc6 | ||
|
|
979f8efa79 | ||
|
|
4c4bccfe73 | ||
|
|
8cb0f99e30 | ||
|
|
44554b63f4 | ||
|
|
6076861cd0 | ||
|
|
8b427a92ac | ||
|
|
6c4736d1bd | ||
|
|
9c50cfdcdb | ||
|
|
8f80e8a3da | ||
|
|
cd44ae02a3 | ||
|
|
848f1f025c | ||
|
|
d2cc8355dc | ||
|
|
ee00d1984a | ||
|
|
2fb71610a4 | ||
|
|
948be0f040 | ||
|
|
b9bb337a07 | ||
|
|
d93f86030c | ||
|
|
d9e1e88c05 | ||
|
|
f579c8dbb8 | ||
|
|
0005ba0715 | ||
|
|
13d74d1f76 | ||
|
|
ddf678af8a | ||
|
|
2c6d6d1602 | ||
|
|
7d0e9f6ab9 | ||
|
|
200a22c93b | ||
|
|
31a90e7f60 | ||
|
|
2bf098b5a3 | ||
|
|
458561fe6f | ||
|
|
157d6dca93 | ||
|
|
7b1b7fbf13 | ||
|
|
876cdce51f | ||
|
|
205a10bc8e | ||
|
|
ef8d1cfdc2 | ||
|
|
879c01ba00 | ||
|
|
8bb7517f32 | ||
|
|
70d9b3ed74 | ||
|
|
f39a21cf7b | ||
|
|
67b169c5a1 | ||
|
|
ff10546152 | ||
|
|
bd6fe4f986 | ||
|
|
360f4e50a0 | ||
|
|
f11f486f5a | ||
|
|
fe53cc97d9 | ||
|
|
baf6ff3f50 | ||
|
|
051e63d76b | ||
|
|
8cdda7d845 | ||
|
|
6a8789050c | ||
|
|
9140c95a0c | ||
|
|
4b52c88d45 | ||
|
|
b7dd9c176a | ||
|
|
6f2e2e727e | ||
|
|
48b137fe4c | ||
|
|
a6326ee7a1 | ||
|
|
c91034ee4c | ||
|
|
7c505f2e23 | ||
|
|
bfcd84a568 | ||
|
|
a314c530bc | ||
|
|
4cb2a8f2aa | ||
|
|
9d272befcd | ||
|
|
1695b26416 | ||
|
|
91fb78b077 | ||
|
|
0bbc8f5b7e | ||
|
|
464bd11c3a | ||
|
|
b88e771eb2 | ||
|
|
7a76763610 | ||
|
|
02d2bad03f | ||
|
|
f65a50a6a3 | ||
|
|
0a9ef07787 | ||
|
|
0e9ad16a2e | ||
|
|
cd0a9ba9e8 | ||
|
|
e08d0eb8f9 | ||
|
|
82f9cc57fd | ||
|
|
ae53129b25 | ||
|
|
543f65fc24 | ||
|
|
896a16c15d | ||
|
|
3d7dee1208 | ||
|
|
e03db79f79 | ||
|
|
0b0150ad73 | ||
|
|
0490250da4 | ||
|
|
dd952ddbe3 | ||
|
|
776fe41e03 | ||
|
|
8b1ea0a6a7 | ||
|
|
183617c85b | ||
|
|
386c67fa70 | ||
|
|
ba7f326e72 | ||
|
|
b9caf2c4a9 | ||
|
|
3188165514 | ||
|
|
1c036cce78 | ||
|
|
a2e648fcfc | ||
|
|
b75a89e876 | ||
|
|
82a3b5e9c5 | ||
|
|
7ecc8103d8 | ||
|
|
6b72b4de11 | ||
|
|
8e1ad0aa8b | ||
|
|
5b7f80487f | ||
|
|
ebc400804f | ||
|
|
9940327b9a | ||
|
|
b300afb504 | ||
|
|
0941618048 | ||
|
|
abc7b11e26 | ||
|
|
85dee27c92 | ||
|
|
5ae3485411 | ||
|
|
8f98469969 | ||
|
|
e1cdbdc91d | ||
|
|
908c9a3bfa | ||
|
|
f45a2cba32 | ||
|
|
763fefb33f | ||
|
|
ce7463f684 | ||
|
|
cd65b6fd13 | ||
|
|
472bb6af1b | ||
|
|
7f96e35b0e | ||
|
|
f9fef3d586 | ||
|
|
ec4f9011b9 | ||
|
|
d634c009d3 | ||
|
|
1872256d7a | ||
|
|
81d107d99c | ||
|
|
a7589914d0 | ||
|
|
0736fa506b | ||
|
|
dfec3c1a15 | ||
|
|
edc89be794 | ||
|
|
d41881596d | ||
|
|
3b34bddf0a | ||
|
|
23c012f417 | ||
|
|
2c8d643f4b | ||
|
|
0abc081edc | ||
|
|
11b36ff4c8 | ||
|
|
c407d32f5d | ||
|
|
87cc487051 | ||
|
|
5ea945ca0b | ||
|
|
1f06339d5a | ||
|
|
09d248b3cc | ||
|
|
28bf5215e4 | ||
|
|
f3a71245e8 | ||
|
|
37fee71898 | ||
|
|
61a354fa60 | ||
|
|
1d4b2c3527 | ||
|
|
694b5b9ee5 | ||
|
|
838508f342 | ||
|
|
694db88ac1 | ||
|
|
cb7c849273 | ||
|
|
9d82be6664 | ||
|
|
66745b0820 | ||
|
|
f845729adb | ||
|
|
6a587d03cf | ||
|
|
affb7d837a | ||
|
|
4ad6405f35 | ||
|
|
906e6b238e | ||
|
|
297865902b | ||
|
|
ab15074ae4 | ||
|
|
b68a1838f2 | ||
|
|
43b2cd1433 | ||
|
|
0036685ece | ||
|
|
d06c670a12 | ||
|
|
6d6cfbc7cc | ||
|
|
3c977f961f | ||
|
|
59c4593bf4 | ||
|
|
5bccfdb244 | ||
|
|
10cb20d08f | ||
|
|
693fef1964 | ||
|
|
fe7700d726 | ||
|
|
fee6de1c1f | ||
|
|
7d04ae2237 | ||
|
|
c748572bf6 | ||
|
|
660bf74d42 | ||
|
|
4db3be345b | ||
|
|
dcea8a8c78 | ||
|
|
815696485f | ||
|
|
b316b56885 | ||
|
|
269dc47b69 | ||
|
|
a7e3246a82 | ||
|
|
96ef5089fc | ||
|
|
c0d96a5733 | ||
|
|
7de4340c38 | ||
|
|
7dd5e7ceeb | ||
|
|
9de7354ea0 | ||
|
|
cfbdda411a | ||
|
|
96c4837cb8 | ||
|
|
2294e6bf36 | ||
|
|
4cffc32328 | ||
|
|
d1b23e25d9 | ||
|
|
e5ba70c3ba | ||
|
|
c88e4aecf5 | ||
|
|
469df84990 | ||
|
|
a4ec706b06 | ||
|
|
30533a7986 | ||
|
|
39b47bc569 | ||
|
|
96104e65ed | ||
|
|
e9bab33e2a | ||
|
|
3b6006d759 | ||
|
|
9535a24706 | ||
|
|
8f2be46536 | ||
|
|
9ca80a7cf7 | ||
|
|
72078f4af3 | ||
|
|
ff9ea37e0c | ||
|
|
9baf261cb3 | ||
|
|
f03a15d503 | ||
|
|
1a70a20c8b | ||
|
|
359cb83fab | ||
|
|
0893858385 | ||
|
|
e1997a6852 | ||
|
|
0f4fee475f | ||
|
|
864e963dea | ||
|
|
3d0debd775 | ||
|
|
6cc2b48a06 | ||
|
|
dee0917642 | ||
|
|
395a68b261 | ||
|
|
0d066b2d11 | ||
|
|
353ede26b8 | ||
|
|
46cfa20f65 | ||
|
|
30b1dd3ce6 | ||
|
|
da601818a3 | ||
|
|
6fa624ec8a | ||
|
|
779ba8e214 | ||
|
|
80f548fdfd | ||
|
|
6f9ec2fa66 | ||
|
|
40437e4a82 | ||
|
|
2ee6fac014 | ||
|
|
8fe5eb514a | ||
|
|
c4312cc387 | ||
|
|
110a9b1e0f | ||
|
|
035a1cf758 | ||
|
|
d4785849e1 | ||
|
|
b41756e38a | ||
|
|
60ee278519 | ||
|
|
b49e44926a | ||
|
|
81187da5b7 | ||
|
|
e70ecc8ac9 | ||
|
|
862055a6de | ||
|
|
7076c0c870 | ||
|
|
2cd6c644a0 | ||
|
|
0edacbe4c8 | ||
|
|
4b330d0e9f | ||
|
|
8ce08f7da1 | ||
|
|
127987f3ba | ||
|
|
3c7eec9456 | ||
|
|
5d45345398 | ||
|
|
b360eb6181 | ||
|
|
369cd516c0 | ||
|
|
24320d6444 | ||
|
|
1951ea09cd | ||
|
|
4d391e7429 | ||
|
|
fb88cdb445 | ||
|
|
572f1d1f6e | ||
|
|
27a8114b6f | ||
|
|
6f3873d99b | ||
|
|
e62e2cc440 | ||
|
|
a5797cdeb3 | ||
|
|
3dd99fdc92 | ||
|
|
0c2265a8da | ||
|
|
01811621a6 | ||
|
|
229ed0d7dc | ||
|
|
7f800ce5bb | ||
|
|
aa291ca79a | ||
|
|
813267ae18 | ||
|
|
fbf7928412 | ||
|
|
7167d47631 | ||
|
|
8b1bb673c1 | ||
|
|
417a8f719f | ||
|
|
ddb88eb1c9 | ||
|
|
d5f0001950 | ||
|
|
4a07b56e8a | ||
|
|
9da1b3c5b7 | ||
|
|
f9f07c47e2 | ||
|
|
f5213fb77a | ||
|
|
48a61605de | ||
|
|
76fb7697ff | ||
|
|
b71a3fee43 | ||
|
|
3a91896d8a | ||
|
|
36ae0d5f20 | ||
|
|
3f727d892c | ||
|
|
c104ed3f8b | ||
|
|
9416376a34 | ||
|
|
66e139c566 | ||
|
|
d0a82c68df | ||
|
|
93e2b3e20a | ||
|
|
2e59e5e3b8 | ||
|
|
727ea12dbd | ||
|
|
ea3225c897 | ||
|
|
bfcfa9dc3c | ||
|
|
ea21821558 | ||
|
|
a9ae1a1016 | ||
|
|
3f5efb1fa3 | ||
|
|
9ba38c84cc | ||
|
|
a5d3c0bbe1 | ||
|
|
d4aa109b73 | ||
|
|
bdeccb49b8 | ||
|
|
35ce7bbd61 | ||
|
|
293933163f | ||
|
|
22fc26797d | ||
|
|
8a95c8b7af | ||
|
|
33674f8cc3 | ||
|
|
6799de645b | ||
|
|
06a88599df | ||
|
|
03877f7ebd | ||
|
|
9f2dbd4291 | ||
|
|
6b7d575e95 | ||
|
|
640db41555 | ||
|
|
d860c2a2b4 | ||
|
|
733bb3bccf | ||
|
|
307751277d | ||
|
|
62c2539475 | ||
|
|
5b15da2c55 | ||
|
|
830f5ff6e3 | ||
|
|
ea1f260ed9 | ||
|
|
3e86254324 | ||
|
|
b5311dd1c3 | ||
|
|
a8a581d35c | ||
|
|
5f129faf32 | ||
|
|
018d78be6f | ||
|
|
b5a3929bb5 | ||
|
|
b67b4b2ef6 | ||
|
|
bed3f551fd | ||
|
|
5b0b8a19ff | ||
|
|
988c9595c7 | ||
|
|
92700808eb | ||
|
|
0dd9aa0d67 | ||
|
|
9cf1d6b710 | ||
|
|
d8d462f43c | ||
|
|
3f22f39ce1 | ||
|
|
e647663c34 | ||
|
|
b5987c75da | ||
|
|
32a973943f | ||
|
|
99c060719a | ||
|
|
d9cd215a60 | ||
|
|
2d3b1fd4eb | ||
|
|
19c2264429 | ||
|
|
ae4b984f0b | ||
|
|
897c113db1 | ||
|
|
50e5a59420 | ||
|
|
8735112f70 | ||
|
|
5a0b5dc680 | ||
|
|
d91e9aa910 | ||
|
|
be1803d558 | ||
|
|
84bbf2f76e | ||
|
|
fe7ea7c679 | ||
|
|
782e197e66 | ||
|
|
88be425ab2 | ||
|
|
7025c5102a | ||
|
|
1ee9b3c7a6 | ||
|
|
4f60a98eb2 | ||
|
|
a71e87a398 | ||
|
|
0ecfad0ae9 | ||
|
|
e909ae566b | ||
|
|
dc2f0adb3e | ||
|
|
180d981ed8 | ||
|
|
fd9bc6214a | ||
|
|
b3f3ed8026 | ||
|
|
967700dc28 | ||
|
|
3ad4ad50ec | ||
|
|
7f980446db | ||
|
|
04ced2e975 | ||
|
|
2253943eba | ||
|
|
a6fcd5d1f2 | ||
|
|
f245822814 | ||
|
|
aa3aff91d0 | ||
|
|
5537dd878f | ||
|
|
5e4e619708 | ||
|
|
d741c705cd | ||
|
|
d83abd81f8 | ||
|
|
640e000b9f | ||
|
|
8d961d6ce6 | ||
|
|
0a612ba36c | ||
|
|
eb7cc8463b | ||
|
|
50f0430d04 | ||
|
|
6e5ea935f6 | ||
|
|
c4eb3de5b2 | ||
|
|
f79b651b05 | ||
|
|
c130879ef7 | ||
|
|
5c391f4bb4 | ||
|
|
b853656aa6 | ||
|
|
8222f940b4 | ||
|
|
71b989b29b | ||
|
|
7e2f7e6f68 | ||
|
|
aefb6c5dc8 | ||
|
|
b892b81dad | ||
|
|
4d63629919 | ||
|
|
4727ee0403 | ||
|
|
fc47e8074f | ||
|
|
ff7829a078 | ||
|
|
d05bbea131 | ||
|
|
e08a681c8f | ||
|
|
6a72b3683e | ||
|
|
b201fc331c | ||
|
|
27337a9243 | ||
|
|
b671a3c2ba | ||
|
|
1cc6b7658c | ||
|
|
12e2827286 | ||
|
|
548d37929e | ||
|
|
eb783b6b6f | ||
|
|
d1b06c3688 | ||
|
|
d16b3429cd | ||
|
|
1e4b5cb450 | ||
|
|
a9da6e4bdf | ||
|
|
a099e2db6a | ||
|
|
7dbeb1f2cf | ||
|
|
550fd185a3 | ||
|
|
1720cce3b0 | ||
|
|
078b254d1d | ||
|
|
6057dc0804 | ||
|
|
5ea7c4f016 | ||
|
|
4334cdace1 | ||
|
|
6803bfae99 | ||
|
|
c49b0fdd49 | ||
|
|
bf3c56f0ec | ||
|
|
bbcecea28d | ||
|
|
a936782ddc | ||
|
|
7ac0fc5a24 | ||
|
|
e115158f86 | ||
|
|
bf34fa39b4 | ||
|
|
5dd29da84f | ||
|
|
2107ffdd18 | ||
|
|
492fc510fa | ||
|
|
c75d240a65 | ||
|
|
dbeb6bc3bc | ||
|
|
4d79a7828c | ||
|
|
aecafbbebd | ||
|
|
130f9cfc4d | ||
|
|
98d02b3fce | ||
|
|
4d52a33ee5 | ||
|
|
7ad916f116 | ||
|
|
05b47a9180 | ||
|
|
9a412bbc76 | ||
|
|
b3a16f95f7 | ||
|
|
871b3be09a | ||
|
|
7085b1e3c3 | ||
|
|
a9e4343775 | ||
|
|
3d4d895d9b | ||
|
|
3d89e3fd93 | ||
|
|
a8e2dd3f2a | ||
|
|
3ae0664fa9 | ||
|
|
98b9f2cf73 | ||
|
|
6fa65ebb9f | ||
|
|
add163a092 | ||
|
|
fcfed3f441 | ||
|
|
7b84d9b10a | ||
|
|
e9a1dbca4b | ||
|
|
145a2654d2 | ||
|
|
9f14cdb1d7 | ||
|
|
5cefa2e995 | ||
|
|
13cbe0fd42 | ||
|
|
c8bb0c0bc8 | ||
|
|
4cc941fbec | ||
|
|
b010192f8e | ||
|
|
2b1af8eea2 | ||
|
|
6ea1f6f78c | ||
|
|
1215470cc1 | ||
|
|
94195b7018 | ||
|
|
3c9c178bdc | ||
|
|
cc64547920 | ||
|
|
a23f2ca956 | ||
|
|
4dacc52ecb | ||
|
|
ccaeecfd46 | ||
|
|
3e45449e8c | ||
|
|
f352a3c597 | ||
|
|
d9a08f0c03 | ||
|
|
c77f24dde3 | ||
|
|
48c31a2119 | ||
|
|
5a578d6d79 | ||
|
|
8cdbc847d1 | ||
|
|
54dbf66cb7 | ||
|
|
177d384126 | ||
|
|
561fe757f3 | ||
|
|
c670ab638f | ||
|
|
eea5cbbafc | ||
|
|
e4c61a4a13 | ||
|
|
0f0afadc94 | ||
|
|
659563f27a | ||
|
|
cbd8a68d0f | ||
|
|
54229282a4 | ||
|
|
514575d0f1 | ||
|
|
756427560c | ||
|
|
900eeb177a | ||
|
|
e67fa3cadc | ||
|
|
c7c32fd4dc | ||
|
|
9471748f4c | ||
|
|
920b1868bb | ||
|
|
eed7c54aaf | ||
|
|
36e58a33ad | ||
|
|
93e8ff749c | ||
|
|
6cb22444ac | ||
|
|
78df101473 | ||
|
|
840508b8ba | ||
|
|
2761254790 | ||
|
|
f47d174f0b | ||
|
|
c50fbc7935 | ||
|
|
75f23a1723 | ||
|
|
f0aa31f9e8 | ||
|
|
aaf59218da | ||
|
|
0203ef66ea | ||
|
|
9c66108dc5 | ||
|
|
558da0a027 | ||
|
|
1d39ec0317 | ||
|
|
281b6468a0 | ||
|
|
9ba6da1f4d | ||
|
|
bde7a7fed6 | ||
|
|
7e021746e4 | ||
|
|
a61bb5f528 | ||
|
|
6a5289b2a2 | ||
|
|
081a5df82c | ||
|
|
3e77ec8008 | ||
|
|
b0b222afa8 | ||
|
|
d85d6f627d | ||
|
|
9505f93da6 | ||
|
|
1671034330 | ||
|
|
3f0ca563e4 | ||
|
|
ffa2a5e8fd | ||
|
|
c3633895d0 | ||
|
|
c15af0a63a | ||
|
|
56e2d77039 | ||
|
|
2cf6fa14a1 | ||
|
|
c374725255 | ||
|
|
babfb6b8d6 | ||
|
|
2ffa8ec7d0 | ||
|
|
b516119224 | ||
|
|
c0e6bb97d8 | ||
|
|
cc30223ff1 | ||
|
|
53c8748942 | ||
|
|
b3a8966e71 | ||
|
|
74800a0767 | ||
|
|
cadd1f993d | ||
|
|
d3807cb86c | ||
|
|
735e9195b5 | ||
|
|
a6619a628e | ||
|
|
456572f87c | ||
|
|
f024b96934 | ||
|
|
25c71bcd98 | ||
|
|
319e7ecc61 | ||
|
|
d9ece5e96a | ||
|
|
4bfe4450b1 | ||
|
|
f9f12ffb61 | ||
|
|
63f81d1033 | ||
|
|
8895d21abf | ||
|
|
2ac1ebeaa0 | ||
|
|
25437602d7 | ||
|
|
8159c71950 | ||
|
|
aa66af7ba6 | ||
|
|
be3633622c | ||
|
|
31655a641c | ||
|
|
b235f6fc6b | ||
|
|
76b297b173 | ||
|
|
1e5cff1d8b | ||
|
|
4e2296d5f6 | ||
|
|
18310632eb | ||
|
|
80ed8d9869 | ||
|
|
2a2658d440 | ||
|
|
710e549c03 | ||
|
|
47fae977dd | ||
|
|
0111919975 | ||
|
|
80837cb040 | ||
|
|
46153a8991 | ||
|
|
a9fccfd4ce | ||
|
|
dfb57e9438 | ||
|
|
eda2731702 | ||
|
|
f47eac5935 | ||
|
|
3bb4dece5a | ||
|
|
0504e231a1 | ||
|
|
6097e14617 | ||
|
|
0b149ce5f1 | ||
|
|
79384e7bf8 | ||
|
|
01eb79cac2 | ||
|
|
128b32c62c | ||
|
|
c1233bd26b | ||
|
|
e23f67def6 | ||
|
|
95f4b29be5 | ||
|
|
5727c708cf | ||
|
|
e49f0f99a1 | ||
|
|
84d340382b | ||
|
|
5bfb067d0f | ||
|
|
d8cb62a92c | ||
|
|
aab8cc40f9 | ||
|
|
9c82689f65 | ||
|
|
ce2755467d | ||
|
|
02b19402ae | ||
|
|
80af66e852 | ||
|
|
15cfa45646 | ||
|
|
ac63849b49 | ||
|
|
02142e352b | ||
|
|
3d92413372 | ||
|
|
5022ed2acc | ||
|
|
31b9d22649 | ||
|
|
7a54894015 | ||
|
|
7100345a09 | ||
|
|
929db15214 | ||
|
|
d9455926c5 | ||
|
|
ba5cd741e1 | ||
|
|
8c94b6243e | ||
|
|
f6473e3383 | ||
|
|
3cd9d6a604 | ||
|
|
142281777e | ||
|
|
8e7b8af180 | ||
|
|
c046cd02d5 | ||
|
|
f5508fcdf1 | ||
|
|
adc2eab6da | ||
|
|
702698d8aa | ||
|
|
73dbe1de37 | ||
|
|
c5706fa28b | ||
|
|
c906a9571e | ||
|
|
038712f431 | ||
|
|
ed2d2c6f86 | ||
|
|
e7158e5a4b | ||
|
|
c66df259a6 | ||
|
|
0b8437fa12 | ||
|
|
a32b6ebf82 | ||
|
|
edbda1696e | ||
|
|
ece218785c | ||
|
|
cc1b3609a1 | ||
|
|
0841aad07a | ||
|
|
30b04ed0c2 | ||
|
|
90b28b3dbd | ||
|
|
0f7ca8f24e | ||
|
|
726d85ebd0 | ||
|
|
c524de9198 | ||
|
|
2fbf03d8ca | ||
|
|
68b8112af0 | ||
|
|
975e83a015 | ||
|
|
1ddb02dbb7 | ||
|
|
1159204ed8 | ||
|
|
a5212e7741 | ||
|
|
1932fd72e3 | ||
|
|
c061fe091b | ||
|
|
3621983a4d | ||
|
|
b6656690ca | ||
|
|
0c9e82ee6b | ||
|
|
b8ebb1faf1 | ||
|
|
690575193f | ||
|
|
9e9cf63c90 | ||
|
|
f5dc213e53 | ||
|
|
138e9c1486 | ||
|
|
7053c0dc93 | ||
|
|
ee8a653b13 | ||
|
|
71fad7e97f | ||
|
|
04740eb928 | ||
|
|
8c63d58c96 | ||
|
|
991672d423 | ||
|
|
55ce38cc87 | ||
|
|
96f3390407 | ||
|
|
aa2426d4e4 | ||
|
|
d0e3e13852 | ||
|
|
a2a1e30d68 | ||
|
|
a6f08b9e0c | ||
|
|
e70842a38c | ||
|
|
2af98d7d42 | ||
|
|
f28eee34e8 | ||
|
|
208610b645 | ||
|
|
e38f8076bc | ||
|
|
6180e1a48b | ||
|
|
e8fe1cd14d | ||
|
|
75d1fa585c | ||
|
|
072ff4b02e | ||
|
|
d8c4e2a72e | ||
|
|
66396de6a3 | ||
|
|
1ca9f575ef | ||
|
|
272e074a15 | ||
|
|
9a738c3783 | ||
|
|
6b4519e959 | ||
|
|
9b1d5213a6 | ||
|
|
f9ed9a4505 | ||
|
|
3ed7ce5630 | ||
|
|
63cd8343c4 | ||
|
|
2d5f8aed24 | ||
|
|
b8d23d0965 | ||
|
|
f4ef92e653 | ||
|
|
c7430f4ae9 | ||
|
|
623fda3087 | ||
|
|
80f27f2b29 | ||
|
|
defbd73b4f | ||
|
|
5285982515 | ||
|
|
f83daa5a22 | ||
|
|
0a07c8f131 | ||
|
|
35481a92f5 | ||
|
|
0995d864e8 | ||
|
|
1a42e43573 | ||
|
|
132aa569fe | ||
|
|
e93a787bd6 | ||
|
|
d6f902653f | ||
|
|
d172d43ddf | ||
|
|
67909724e7 | ||
|
|
1c88cbc5f6 | ||
|
|
3fb4831804 | ||
|
|
b65570fb16 | ||
|
|
871e6e6f64 | ||
|
|
0cd92a96c0 | ||
|
|
ff31ddd2d5 | ||
|
|
e9f5b43785 | ||
|
|
36b41081ab | ||
|
|
1092313892 | ||
|
|
ce605cb8f3 | ||
|
|
0733d885fb | ||
|
|
ef6fe3d1d9 | ||
|
|
82a4fea81a | ||
|
|
14a0252819 | ||
|
|
849aeb65f0 | ||
|
|
a99dc6e93d | ||
|
|
1a5fcc1348 | ||
|
|
c9d88f5628 | ||
|
|
0910711aca | ||
|
|
6a1a9ef6bc | ||
|
|
ef1c059079 | ||
|
|
a05ab37373 | ||
|
|
7ea88cfd22 | ||
|
|
8d14fe6a90 | ||
|
|
49a05b42f9 | ||
|
|
b6da1d7769 | ||
|
|
7e2ad58edd | ||
|
|
7ad0d98404 | ||
|
|
b7052bba95 | ||
|
|
4a5618bcc4 | ||
|
|
c1850fe604 | ||
|
|
fccdf32c9f | ||
|
|
c474411e1b | ||
|
|
9033eac5b1 | ||
|
|
0ba0426591 | ||
|
|
9a9b540701 | ||
|
|
b8ddbb9a26 | ||
|
|
bc3c2b96ed | ||
|
|
622dc87d84 | ||
|
|
18edbdf404 | ||
|
|
02a3eab693 | ||
|
|
dded868a80 | ||
|
|
6fe878bdb9 | ||
|
|
505ece9e36 | ||
|
|
72acfa1e0c | ||
|
|
02cd8b9a44 | ||
|
|
f32250f431 | ||
|
|
1523cad044 | ||
|
|
ed55353700 | ||
|
|
8d3bd45102 | ||
|
|
f69753f6a0 | ||
|
|
b7e2e87ac1 | ||
|
|
3b98508986 | ||
|
|
36ebfc42d9 | ||
|
|
ae5f55ff65 | ||
|
|
01131da5b7 | ||
|
|
66ae865de0 | ||
|
|
8c50f08fec | ||
|
|
e69de47daf | ||
|
|
acef10e848 | ||
|
|
edb9d50d9e | ||
|
|
b989d4f6be | ||
|
|
84dd5c0eca | ||
|
|
db8daf20b7 | ||
|
|
cb985b5a03 | ||
|
|
28c4e38272 | ||
|
|
b29377dc2b | ||
|
|
ca09592a72 | ||
|
|
b0e14fae43 | ||
|
|
dfa68f126b | ||
|
|
4570f68727 | ||
|
|
4705acba55 | ||
|
|
1dea20a358 | ||
|
|
191f661b35 | ||
|
|
f3ab38ba8f | ||
|
|
2d0a892e70 | ||
|
|
00eefe87d7 | ||
|
|
f68fd96555 | ||
|
|
fc50a3242f | ||
|
|
64e9ce2bfa | ||
|
|
e8d04b1921 | ||
|
|
9fedf14e55 | ||
|
|
7d535ef7d1 | ||
|
|
85d610bcdf | ||
|
|
3af8b7bd0c | ||
|
|
6215d85581 | ||
|
|
308d9f5abd | ||
|
|
bcfabba038 | ||
|
|
69139b1987 | ||
|
|
3f8f9d9d73 | ||
|
|
2283579109 | ||
|
|
1e0af94ba2 | ||
|
|
dd631328e0 | ||
|
|
cca2a57c69 | ||
|
|
bbfbcacc6a | ||
|
|
3f95c1a9d5 | ||
|
|
7bd05e9b16 | ||
|
|
10742d7c51 | ||
|
|
bbf7940e45 | ||
|
|
f5af71f4e6 | ||
|
|
1ffcd67c9d | ||
|
|
cf8592ba12 | ||
|
|
1e1d036590 | ||
|
|
64afb6d305 | ||
|
|
20a9e018f7 | ||
|
|
a574c33ac6 | ||
|
|
d21e458fe1 | ||
|
|
bb3ba53c6e | ||
|
|
ba2e4df76b | ||
|
|
33e7e4bee3 | ||
|
|
b83dce2cb3 | ||
|
|
6821676853 | ||
|
|
52024986ee | ||
|
|
2e3342d20d | ||
|
|
419aa1d78f | ||
|
|
de4ddd578f | ||
|
|
5dd68a7ed4 | ||
|
|
881168dc7a | ||
|
|
295e427abe | ||
|
|
67c5d15f51 | ||
|
|
39a7f37f35 | ||
|
|
15d276b0bc | ||
|
|
4387025c48 | ||
|
|
52c20f02c7 | ||
|
|
413dab8094 | ||
|
|
6b1d11a04d | ||
|
|
5441379f38 | ||
|
|
52d954c64e | ||
|
|
415495fc16 | ||
|
|
4223298634 | ||
|
|
e058c71019 | ||
|
|
12b2bbfa26 | ||
|
|
7d22bfffa5 | ||
|
|
4ade80c8a9 | ||
|
|
9674381fc2 | ||
|
|
f6d91280d7 | ||
|
|
e4ebbd8f69 | ||
|
|
7f5560de6d | ||
|
|
0925171e07 | ||
|
|
900d3ad01d | ||
|
|
6a1efedc1f | ||
|
|
0f602e0e62 | ||
|
|
3a425fc47d | ||
|
|
36722cbbd8 | ||
|
|
9572336e96 | ||
|
|
4895cde474 | ||
|
|
21ca1a73cf | ||
|
|
457c1c6ae6 | ||
|
|
c1ff127101 | ||
|
|
5e815a5832 | ||
|
|
4604e9cd54 | ||
|
|
ef580a7680 | ||
|
|
2102776be8 | ||
|
|
f3af7a6b77 | ||
|
|
5abeceac1f | ||
|
|
22f73a0817 | ||
|
|
29b5aef73a | ||
|
|
3e602bce52 | ||
|
|
fe39901885 | ||
|
|
ee9e8b365c | ||
|
|
bd6fa35d4a | ||
|
|
5e1fdf66b0 | ||
|
|
d86e709b90 | ||
|
|
f82cffe882 | ||
|
|
3d88eb92c1 | ||
|
|
0811ebb52d | ||
|
|
491d431ea5 | ||
|
|
ceeebebcad | ||
|
|
04d766aba7 | ||
|
|
3b431e180e | ||
|
|
a2eab1577d | ||
|
|
f8dd3d05dc | ||
|
|
93993e62d6 | ||
|
|
483c88b958 | ||
|
|
4a1e846958 | ||
|
|
927b9afe7c | ||
|
|
9e8827f015 | ||
|
|
ca45f461ae | ||
|
|
7e5af11c2d | ||
|
|
102436274f | ||
|
|
49c26a76b4 | ||
|
|
d469ef7bf2 | ||
|
|
a5374cb876 | ||
|
|
a53e911a0c | ||
|
|
c4a78a149b | ||
|
|
3814912adf | ||
|
|
75ef652cb3 | ||
|
|
1eae3fe5dc | ||
|
|
d0bc383a79 | ||
|
|
ea5d31b530 | ||
|
|
e56ad19e81 | ||
|
|
ed393eb75c | ||
|
|
2e2d309d38 | ||
|
|
6bfb33adb2 | ||
|
|
05138e1db1 | ||
|
|
b65402db41 | ||
|
|
fbef187400 | ||
|
|
25d1b8b9e5 | ||
|
|
ec91594083 | ||
|
|
e67f55bd74 | ||
|
|
276a7f9d2a | ||
|
|
4888d7a567 | ||
|
|
767f78315b | ||
|
|
4311b218fb | ||
|
|
89892042dd | ||
|
|
a22dfb3864 | ||
|
|
f40c42dcf0 | ||
|
|
1d31aa9da3 | ||
|
|
f999073fb4 | ||
|
|
5eb7c64ec3 | ||
|
|
7e79818555 | ||
|
|
a51576b250 | ||
|
|
0bee2689d1 | ||
|
|
c232743963 | ||
|
|
c2d5f092b3 | ||
|
|
8e8e186de9 | ||
|
|
55f47a2b1b | ||
|
|
5c27b9eb90 | ||
|
|
1d018533d1 | ||
|
|
34b8841f5c | ||
|
|
a54166a27c | ||
|
|
4adec39dce | ||
|
|
e63ae08638 | ||
|
|
f1b4c6a1ec | ||
|
|
78b4b3ffa8 | ||
|
|
f51fd48cdb | ||
|
|
9127a816e3 | ||
|
|
1a5cd9b062 | ||
|
|
a86b4c4e58 | ||
|
|
a384960a7b | ||
|
|
5817b80c94 | ||
|
|
97efd63c99 | ||
|
|
5ebcc01bcc | ||
|
|
ddcc008ea6 | ||
|
|
8ec10e2304 | ||
|
|
172f275bc7 | ||
|
|
0dac5943bb | ||
|
|
2dcd2b13be | ||
|
|
8743decd17 | ||
|
|
a15d7ca543 | ||
|
|
a90029ec50 | ||
|
|
563896b126 | ||
|
|
5aecd02668 | ||
|
|
eef6312a17 | ||
|
|
1e48f5c8f4 | ||
|
|
7908fa2814 | ||
|
|
7ffb3ea042 | ||
|
|
842dd90780 | ||
|
|
ee71a03f96 | ||
|
|
5221631b40 | ||
|
|
ade2b4ac0d | ||
|
|
643b95c0e4 | ||
|
|
ed0f9d4fb5 | ||
|
|
6f353c31cb | ||
|
|
404412a950 | ||
|
|
0e1778d8da | ||
|
|
dfc0ca9525 | ||
|
|
53bf1fe4fd | ||
|
|
5c05daa007 | ||
|
|
f74f075b4e | ||
|
|
1b53e8e98c | ||
|
|
4caee284bc | ||
|
|
5368c91399 | ||
|
|
00b1881f20 | ||
|
|
b727bd20c8 | ||
|
|
5a703c116f | ||
|
|
57f728f40e | ||
|
|
74bbde0c34 | ||
|
|
c58fc0781c | ||
|
|
7a266d49c1 | ||
|
|
344eeeb8c0 | ||
|
|
08d50ddfc5 | ||
|
|
f83c892b13 | ||
|
|
7dbec9d395 | ||
|
|
5171c801d9 | ||
|
|
72b4a778a6 | ||
|
|
0254df63f9 | ||
|
|
03c02ff74f | ||
|
|
9277c3c674 | ||
|
|
34b1259dc6 | ||
|
|
257e38de12 | ||
|
|
1984132eae | ||
|
|
8b1d75ea98 | ||
|
|
a79af11371 | ||
|
|
087a9b70b4 | ||
|
|
8b37952128 | ||
|
|
8336930091 | ||
|
|
d875325cee | ||
|
|
f1c5d1cb23 | ||
|
|
b55752e715 | ||
|
|
36a99e6ec8 | ||
|
|
df3c5b9a77 | ||
|
|
4c2ab23c1f | ||
|
|
0a3f81e364 | ||
|
|
69c5631b25 | ||
|
|
cca4331ac4 | ||
|
|
5972dbb6a7 | ||
|
|
794f2d8576 | ||
|
|
e35dce530a | ||
|
|
6034bd3cb2 | ||
|
|
42c77bbb67 | ||
|
|
d504c8f4a3 | ||
|
|
976c8b3ce0 | ||
|
|
905e7e41a6 | ||
|
|
16f48d0400 | ||
|
|
6d7608d851 | ||
|
|
7cd30efbe9 | ||
|
|
8c3652e1ac | ||
|
|
7b9e3aace9 | ||
|
|
568737e123 | ||
|
|
808e3ea6e3 | ||
|
|
e4f14d871f | ||
|
|
1cedd8ee5b | ||
|
|
b80fe24179 | ||
|
|
3cea41a854 | ||
|
|
62b80c13cc | ||
|
|
08b612575c | ||
|
|
b3b804bf2d | ||
|
|
09d8a86e30 | ||
|
|
cc5d917c72 | ||
|
|
7cc3e9d5ef | ||
|
|
87aa9fc6c4 | ||
|
|
8d5c315817 | ||
|
|
f89486f5fe | ||
|
|
cf4d78a24c | ||
|
|
ee88970075 | ||
|
|
f5fa40feb7 | ||
|
|
da13192cfe | ||
|
|
8ff65a65db | ||
|
|
2d882a11a5 | ||
|
|
79adc677a1 | ||
|
|
a8e2dad6d0 | ||
|
|
16954c5626 | ||
|
|
56305c4ede | ||
|
|
30713cc21d | ||
|
|
2bac0a25e0 | ||
|
|
c3aa5b0515 | ||
|
|
30afa43ed4 | ||
|
|
f2e66abe67 | ||
|
|
cdb311a501 | ||
|
|
4c2b3e157c | ||
|
|
52625b710b | ||
|
|
b411bb4e85 | ||
|
|
f16111369f | ||
|
|
628d052749 | ||
|
|
2e26bb0e57 | ||
|
|
652c7d72f6 | ||
|
|
0b046ea8cf | ||
|
|
7133f05fea | ||
|
|
da1c23d7dd | ||
|
|
a329dcd844 | ||
|
|
b57212d5f7 | ||
|
|
12c9e16f37 | ||
|
|
422381ac01 | ||
|
|
4b817cef65 | ||
|
|
4255a9916d | ||
|
|
b1566e904b | ||
|
|
8629076814 | ||
|
|
761e4f2718 | ||
|
|
3441dd4d2d | ||
|
|
9a8139a8b2 | ||
|
|
a0f823910c | ||
|
|
755338ded0 | ||
|
|
ecd85a53a1 | ||
|
|
8faa0d9e01 | ||
|
|
20ca2eac37 | ||
|
|
85b56247a3 | ||
|
|
febc66fb02 | ||
|
|
3f929aee3c | ||
|
|
8c46dd194d | ||
|
|
a9f99d05b9 | ||
|
|
e224c33817 | ||
|
|
4e6afbc8ff | ||
|
|
80b912ea14 | ||
|
|
85a571521b | ||
|
|
98efa9f466 | ||
|
|
a30a9ad763 | ||
|
|
36787eeeeb | ||
|
|
104cafedb0 | ||
|
|
284d307c0f | ||
|
|
f50c98e10d | ||
|
|
a4a5d63da7 | ||
|
|
9f0f9e0928 | ||
|
|
ef62fd7e34 | ||
|
|
0151dd8875 | ||
|
|
2026024c8a | ||
|
|
3d56baddd7 | ||
|
|
c10e7b2b41 | ||
|
|
ba605e985a | ||
|
|
4cd1cdf495 | ||
|
|
f9c644b964 | ||
|
|
da68bcf952 | ||
|
|
2c3b625b6b | ||
|
|
16ef259ae1 | ||
|
|
c56b3e75ea | ||
|
|
772866389f | ||
|
|
41f8b5aee2 | ||
|
|
75052fc773 | ||
|
|
e7a580fd3a | ||
|
|
c967230969 | ||
|
|
f3e6b2ab57 | ||
|
|
7385cb0c89 | ||
|
|
d1be9e7dc8 | ||
|
|
a22052e85b | ||
|
|
ab401541f2 | ||
|
|
dab4427c18 | ||
|
|
ad7d508a3c | ||
|
|
cb30514cbc | ||
|
|
53af9db285 | ||
|
|
b557a14134 | ||
|
|
8922fd6dde | ||
|
|
6b39e549f7 | ||
|
|
eb0dd827b1 | ||
|
|
a48902d7d3 | ||
|
|
7fca1f865d | ||
|
|
c5dc06962d | ||
|
|
45ecdafd52 | ||
|
|
f9e8bf92bd | ||
|
|
de614b293c | ||
|
|
8382075147 | ||
|
|
b605630a9c | ||
|
|
0a89fb1928 | ||
|
|
4c81906b50 | ||
|
|
b6220e985a | ||
|
|
7a97a28e00 | ||
|
|
dda8f281a1 | ||
|
|
28b561a810 | ||
|
|
44aba93d37 | ||
|
|
efb5157027 | ||
|
|
f48c1562f3 | ||
|
|
098b5b28a8 | ||
|
|
998e8e748c | ||
|
|
be96c77467 | ||
|
|
b5e8aec548 | ||
|
|
278c7dc836 | ||
|
|
1425ad87ab | ||
|
|
f985a2a8ad | ||
|
|
20f423a2fc | ||
|
|
210c1215d6 | ||
|
|
1c507393a9 | ||
|
|
b04cfdc091 | ||
|
|
42a3006f6e | ||
|
|
7d5c618246 | ||
|
|
74250982d1 | ||
|
|
8bd6fb5b4e | ||
|
|
bb2fb92b0e | ||
|
|
10b016312d | ||
|
|
b4500a2914 | ||
|
|
da9b1b3f58 | ||
|
|
da31520465 | ||
|
|
f3d3a0e57f | ||
|
|
9a655988dd | ||
|
|
43b541bb7d | ||
|
|
e32853744a | ||
|
|
b017b63745 | ||
|
|
29d6d824fc | ||
|
|
d6983a25b0 | ||
|
|
4cab6726b0 | ||
|
|
509b8f3a29 | ||
|
|
461c2f9055 | ||
|
|
c1d0f9cf1b | ||
|
|
019de08c64 | ||
|
|
30f88f99b7 | ||
|
|
04ee9894c9 | ||
|
|
46368f52f5 | ||
|
|
226e9084c9 | ||
|
|
caf5d2f788 | ||
|
|
591634f175 | ||
|
|
dfaf51a432 | ||
|
|
c2ec066c85 | ||
|
|
3e7c09bb44 | ||
|
|
28f66e30dd | ||
|
|
bac7f097a9 | ||
|
|
fecd29a967 | ||
|
|
c22eed9109 | ||
|
|
2c1ba623a4 | ||
|
|
561cdba7c9 | ||
|
|
e5e25012ff | ||
|
|
c5d653a51a | ||
|
|
c4620a0b0c | ||
|
|
dc969f80f3 | ||
|
|
a82f85f058 | ||
|
|
3ef85e711d | ||
|
|
29f7307f9d | ||
|
|
24eed71959 | ||
|
|
9341ccb643 | ||
|
|
4385b90cca | ||
|
|
9c10ee12fd | ||
|
|
7a622986d6 | ||
|
|
99b3cdd45d | ||
|
|
7b9d84f528 | ||
|
|
5f49827bd2 | ||
|
|
e1b4fe02d6 | ||
|
|
cdab127b98 | ||
|
|
bac7ea5dc0 | ||
|
|
b3b5581f2d | ||
|
|
9a26d869fe | ||
|
|
57138d9629 | ||
|
|
47d897b48d | ||
|
|
6b9f1d519e | ||
|
|
8b966c61d8 | ||
|
|
87e09efff5 | ||
|
|
65a0583c85 | ||
|
|
2000a1b334 | ||
|
|
cdd44af043 | ||
|
|
df99d06d11 | ||
|
|
b32d096be2 | ||
|
|
b9d5d1041c | ||
|
|
994dc9ff33 | ||
|
|
2ac97736ce | ||
|
|
925e106c43 | ||
|
|
59002413f8 | ||
|
|
32293ee3bb | ||
|
|
e04f35af4f | ||
|
|
c50f83400a | ||
|
|
31fa974379 | ||
|
|
aa8c362cb8 | ||
|
|
e48cc40610 | ||
|
|
a931f59dda | ||
|
|
4140fc4d28 | ||
|
|
f0d39e6081 | ||
|
|
c11872a229 | ||
|
|
f4fe730dc9 | ||
|
|
a4c83a08c7 | ||
|
|
c3277d24c3 | ||
|
|
28a23590ef | ||
|
|
162275992b | ||
|
|
08e25f4622 | ||
|
|
9a300d773f | ||
|
|
a5362f3170 | ||
|
|
7b653b90a8 | ||
|
|
f464b3ae7c | ||
|
|
d4b7064d2f | ||
|
|
2540b9969f | ||
|
|
4355b799cb | ||
|
|
a7fb434f9f | ||
|
|
b3465f3bbf | ||
|
|
4dfc3ff94c | ||
|
|
2670279b0d | ||
|
|
f4fd4245be | ||
|
|
200d239e0b | ||
|
|
41a0590182 | ||
|
|
63b2c49354 | ||
|
|
c99d7ce89d | ||
|
|
79e382e064 | ||
|
|
fa1d4dae0f | ||
|
|
35f84b4414 | ||
|
|
a882bdb08c | ||
|
|
b5c2e1c95b | ||
|
|
cfd6ea5a34 | ||
|
|
b3cf5e8698 | ||
|
|
2d07223995 | ||
|
|
32bb9fd4ec | ||
|
|
37b5f22c1a | ||
|
|
e0b00c09ce | ||
|
|
71c4dba34d | ||
|
|
9a3187e36c | ||
|
|
c714a3f91c | ||
|
|
5d4846cf60 | ||
|
|
543b327d09 | ||
|
|
1ccd2a3cc9 | ||
|
|
94398071eb | ||
|
|
143476c76c | ||
|
|
e85683e40f | ||
|
|
141162e5ae | ||
|
|
13acbfb64f | ||
|
|
a55e3d8578 | ||
|
|
2486ae6d3f | ||
|
|
ba991b47ea | ||
|
|
a096d0db37 | ||
|
|
e4a30c052e | ||
|
|
ccfe80ecfa | ||
|
|
df5b889f7d | ||
|
|
55f25c2613 | ||
|
|
34eaa822a0 | ||
|
|
5072f503c4 | ||
|
|
8e6b5a5e55 | ||
|
|
246647641a | ||
|
|
06a1bdbcba | ||
|
|
a0b75d7ada | ||
|
|
d36036a9fe | ||
|
|
a68d891770 | ||
|
|
f272a9ead2 | ||
|
|
8db2e10696 | ||
|
|
bea4f7da98 | ||
|
|
3cd74c06d6 | ||
|
|
053681e4a5 | ||
|
|
e3225f7123 | ||
|
|
ef62b0e746 | ||
|
|
7756d12993 | ||
|
|
3b92d12a44 | ||
|
|
4449bf3c75 | ||
|
|
0200ed1341 | ||
|
|
02bde900d6 | ||
|
|
36e299b4cf | ||
|
|
4e34f861c9 | ||
|
|
42df2fe242 | ||
|
|
7f11da8650 | ||
|
|
dff0fcb1fd | ||
|
|
c31d22f996 | ||
|
|
14ab2b52e9 | ||
|
|
e91f34f09b | ||
|
|
6c733f84e9 | ||
|
|
71c2cd5065 | ||
|
|
45f9a10bfb | ||
|
|
6ad203f767 | ||
|
|
5bf895eefe | ||
|
|
5a8f5b14c1 | ||
|
|
27ccb37646 | ||
|
|
b4c528f6b5 | ||
|
|
517a255d5a | ||
|
|
d76feeb320 | ||
|
|
e4f23b3fce | ||
|
|
02d8dc22e9 | ||
|
|
2fc3ec7224 | ||
|
|
a243ac7a01 | ||
|
|
6f1ee78623 | ||
|
|
6633370a34 | ||
|
|
a22e29532c | ||
|
|
96c8d1f68b | ||
|
|
4aa3bd56c4 | ||
|
|
7d1cf7975a | ||
|
|
59d08a92d0 | ||
|
|
33c1a002da | ||
|
|
8f04d3418a | ||
|
|
f5b556c6cb | ||
|
|
a18f3bf08a | ||
|
|
41a0a0c64c | ||
|
|
b4c8a317d0 | ||
|
|
30ac075555 | ||
|
|
fcb317ca40 | ||
|
|
9991f79326 | ||
|
|
7cf3f8c7cd | ||
|
|
eeef183cc6 | ||
|
|
96bc02c8cb | ||
|
|
af276a24ed | ||
|
|
a92dbae459 | ||
|
|
b0bfb5e443 | ||
|
|
671e6da104 | ||
|
|
485be533c9 | ||
|
|
53ff4cd651 | ||
|
|
a98c760d93 | ||
|
|
9b0c95e37d | ||
|
|
b419b8f642 | ||
|
|
c32f8d0998 | ||
|
|
c6d253cc25 | ||
|
|
c05cb62cdb | ||
|
|
ddd213dda3 | ||
|
|
8aab844a48 | ||
|
|
8fc1cbfc54 | ||
|
|
82ea330dda | ||
|
|
c63aa3033a | ||
|
|
7837738c17 | ||
|
|
bdb0ad4836 | ||
|
|
0dbf810cfe | ||
|
|
58aa4887ad | ||
|
|
3ff0916f49 | ||
|
|
cc9fb3ff41 | ||
|
|
2c51928b9d | ||
|
|
4451978af6 | ||
|
|
d4193c55ae | ||
|
|
ec3e365b3c | ||
|
|
c245c69389 | ||
|
|
9a9a7c9d7a | ||
|
|
20b516b5fa | ||
|
|
4dcfeca092 | ||
|
|
8ca8c73868 | ||
|
|
1a1c101b9a | ||
|
|
a0a0e2d324 | ||
|
|
d544199eff | ||
|
|
407ebeb699 | ||
|
|
9ba12b3e3f | ||
|
|
97976cbee9 | ||
|
|
cb5c772233 | ||
|
|
2bfbb824bf | ||
|
|
05189651ee | ||
|
|
0c80134523 | ||
|
|
6c686520c4 | ||
|
|
49ae029572 | ||
|
|
10e43203ed | ||
|
|
e25be5a522 | ||
|
|
47ff64a9e9 | ||
|
|
35f3887d79 | ||
|
|
dacfc8b00a | ||
|
|
0840ac7d06 | ||
|
|
f34b96f565 | ||
|
|
808fbed4f3 | ||
|
|
b48800049b | ||
|
|
7e5d4166d8 | ||
|
|
6b724ba8db | ||
|
|
25e1427a84 | ||
|
|
82944bc103 | ||
|
|
1e73b2437d | ||
|
|
574cd304e3 | ||
|
|
2edcd1f5b2 | ||
|
|
e144b26619 | ||
|
|
9ef9220689 | ||
|
|
23b6a8c495 | ||
|
|
4b9552fdf5 | ||
|
|
a249097170 | ||
|
|
6f4b2d7967 | ||
|
|
888d77c96c | ||
|
|
e2ae1ca866 | ||
|
|
7ef9a0f0bf | ||
|
|
885a0882f2 | ||
|
|
2ddeca2976 | ||
|
|
2085012874 | ||
|
|
ebb495ab8b | ||
|
|
905274d6d4 | ||
|
|
120d7edf12 | ||
|
|
12cd06cae5 | ||
|
|
42c26102d8 | ||
|
|
e906564f70 | ||
|
|
b8b2dc3b6d | ||
|
|
d69f76e68a | ||
|
|
ec1ba02cca | ||
|
|
49869d071a | ||
|
|
39275682d9 | ||
|
|
1dbc3a049b | ||
|
|
0a7b2683ff | ||
|
|
9714985855 | ||
|
|
d809b8eee0 | ||
|
|
52e2ed2aa7 | ||
|
|
ab3cc3d2b6 | ||
|
|
0a33324321 | ||
|
|
a948335cb9 | ||
|
|
6d2e5af6bc | ||
|
|
9a7c71394e | ||
|
|
1ebf184e9f | ||
|
|
48b013fa85 | ||
|
|
e96dee478d | ||
|
|
25d2a12a92 | ||
|
|
b0af31a71c | ||
|
|
9fdd766429 | ||
|
|
28ce23350d | ||
|
|
1abe890067 | ||
|
|
75fbedeeac | ||
|
|
a1861a2f84 | ||
|
|
0eecfb52ed | ||
|
|
ce1c28ffb9 | ||
|
|
9667b81c8d | ||
|
|
afac596a7b | ||
|
|
636692ba6a | ||
|
|
203f7e7ea6 | ||
|
|
8aee19b0ef | ||
|
|
49ed31c895 | ||
|
|
526075cd5e | ||
|
|
71340805ec | ||
|
|
654585aa68 | ||
|
|
441d3c1220 | ||
|
|
fc2d7258fb | ||
|
|
8195cedcf4 | ||
|
|
3e98477707 | ||
|
|
7b9479eca9 | ||
|
|
f15b67d386 | ||
|
|
f3141727d7 | ||
|
|
c7e4638736 | ||
|
|
d04981969f | ||
|
|
a148d4f446 | ||
|
|
a9e105f9eb | ||
|
|
c8afce2a3a | ||
|
|
ebf9d11f4d | ||
|
|
7a3bdf93e9 | ||
|
|
35d5c585ec | ||
|
|
45eb4f7201 | ||
|
|
e4c2585e23 | ||
|
|
a05f2fe3fb | ||
|
|
94cdd6829c | ||
|
|
34aa51e98a | ||
|
|
6e93c68a77 | ||
|
|
f16eec2ad2 | ||
|
|
657c1a4626 | ||
|
|
6bf6e943cc | ||
|
|
6252f903ee | ||
|
|
a1e827fb3b | ||
|
|
40218e02e1 | ||
|
|
0891b0d294 | ||
|
|
756c43c087 | ||
|
|
6488e66088 | ||
|
|
31ee3fb12b | ||
|
|
b80d45ad45 | ||
|
|
ea2a3c3b70 | ||
|
|
c0312159eb | ||
|
|
e124f582aa | ||
|
|
54150a8f2a | ||
|
|
385c221ef8 | ||
|
|
7df6e99e28 | ||
|
|
76a5f2f516 | ||
|
|
936c1f2ada | ||
|
|
5787802077 | ||
|
|
c902d25524 | ||
|
|
d152336f42 | ||
|
|
7ab0ae18be | ||
|
|
6189eddecf | ||
|
|
cc7471a9af | ||
|
|
eabaca361c | ||
|
|
141ca2498c | ||
|
|
964db2a779 | ||
|
|
f2867542b1 | ||
|
|
8f6e886821 | ||
|
|
dfca7bc2c4 | ||
|
|
f94b04bb19 | ||
|
|
be1e97c616 | ||
|
|
9790e9f843 | ||
|
|
e339d3857f | ||
|
|
0e4f21842c | ||
|
|
d4ea9a1ba3 | ||
|
|
5300fa929b | ||
|
|
8a5a66a06a | ||
|
|
43ed4427b4 | ||
|
|
22b0d92e6b | ||
|
|
7eb528232e | ||
|
|
33b6b85e72 | ||
|
|
8177119671 | ||
|
|
4b407a9dd2 | ||
|
|
9b904a7b4a | ||
|
|
e0424a1e38 | ||
|
|
00c08e05f5 | ||
|
|
0836d9c5ee | ||
|
|
ff5b6c4eab | ||
|
|
bb5cd49327 | ||
|
|
6f40ef6015 | ||
|
|
c7758506ac | ||
|
|
192ac6959a | ||
|
|
465a4643a8 | ||
|
|
1bf0065d9d | ||
|
|
c14efc843c | ||
|
|
0506dfba10 | ||
|
|
7e4f36e8b4 | ||
|
|
55dc482a61 | ||
|
|
05b9f9cae6 | ||
|
|
c79868de3e | ||
|
|
7eeac060ab | ||
|
|
c8b2b7b279 | ||
|
|
f4b3187494 |
No files matched your search
@@ -0,0 +1,13 @@
|
||||
version: 1.0.{build}
|
||||
|
||||
|
||||
notifications:
|
||||
- provider: Slack
|
||||
incoming_webhook:
|
||||
secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
|
||||
|
||||
|
||||
clone_folder: c:\projects\lbry
|
||||
|
||||
build_script:
|
||||
- echo "Not currently building on appveyor"
|
||||
@@ -0,0 +1,20 @@
|
||||
[bumpversion]
|
||||
current_version = 0.9.2rc24
|
||||
commit = True
|
||||
tag = True
|
||||
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+)(?P<candidate>\d+))?
|
||||
serialize =
|
||||
{major}.{minor}.{patch}{release}{candidate}
|
||||
{major}.{minor}.{patch}
|
||||
|
||||
[bumpversion:part:candidate]
|
||||
first_value = 1
|
||||
|
||||
[bumpversion:part:release]
|
||||
optional_value = production
|
||||
values =
|
||||
rc
|
||||
production
|
||||
|
||||
[bumpversion:file:lbrynet/__init__.py]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
/CHANGELOG.md merge=union
|
||||
@@ -0,0 +1,48 @@
|
||||
<!--
|
||||
Thanks for reporting an issue to LBRY and helping us improve!
|
||||
|
||||
To make it possible for us to help you, please fill out below information carefully.
|
||||
|
||||
Before reporting any issues, please make sure that you're using the latest version.
|
||||
- App releases: https://github.com/lbryio/lbry-app/releases
|
||||
- Standalone daemon: https://github.com/lbryio/lbry/releases
|
||||
|
||||
We are also available on Slack at https://slack.lbry.io
|
||||
-->
|
||||
|
||||
|
||||
## The Issue
|
||||
|
||||
### Steps to reproduce
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
### Expected behaviour
|
||||
Tell us what should happen
|
||||
|
||||
### Actual behaviour
|
||||
Tell us what happens instead
|
||||
|
||||
|
||||
## System Configuration
|
||||
|
||||
<!-- For the app, this info is in the About section at the bottom of the Help page.
|
||||
You can include a screenshot instead of typing it out -->
|
||||
|
||||
<!-- For the daemon, run:
|
||||
curl 'http://localhost:5279/lbryapi' --data '{"method":"version"}'
|
||||
and include the full output -->
|
||||
|
||||
- LBRY Daemon version:
|
||||
- LBRY App version:
|
||||
- LBRY Installation ID:
|
||||
- Operating system:
|
||||
|
||||
|
||||
## Anything Else
|
||||
<!-- Include anything else that does not fit into the above sections -->
|
||||
|
||||
|
||||
## Screenshots
|
||||
<!-- If a screenshot would help explain the bug, please include one or two here -->
|
||||
+22
@@ -1 +1,23 @@
|
||||
*.pyc
|
||||
*.egg
|
||||
*.so
|
||||
*.xml
|
||||
*.iml
|
||||
*.log
|
||||
*.pem
|
||||
*.decTest
|
||||
*.prof
|
||||
.#*
|
||||
|
||||
/build
|
||||
/dist
|
||||
/lbrynet.egg-info
|
||||
/docs_build
|
||||
/lbry-venv
|
||||
|
||||
.idea/.name
|
||||
.coverage
|
||||
.DS_Store
|
||||
|
||||
# temporary files from the twisted.trial test runner
|
||||
_trial_temp/
|
||||
File renamed without changes.
@@ -0,0 +1,386 @@
|
||||
[MASTER]
|
||||
|
||||
# Specify a configuration file.
|
||||
#rcfile=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
# `\.#.*` - add emacs tmp files to the blacklist
|
||||
ignore-patterns=\.#.*
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=yes
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=1
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
# be used to obtain the result of joining multiple strings with the addition
|
||||
# operator. Joining a lot of strings can lead to a maximum recursion error in
|
||||
# Pylint and this flag can prevent that. It has one side effect, the resulting
|
||||
# AST will be different than the one from reality.
|
||||
optimize-ast=no
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||
confidence=
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
# --disable=W"
|
||||
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html. You can also give a reporter class, eg
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
output-format=text
|
||||
|
||||
# Put messages in a separate file for each module / package specified on the
|
||||
# command line instead of printing them on stdout. Reports (if any) will be
|
||||
# written in a file name "pylint_global.[txt|html]".
|
||||
files-output=no
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=yes
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details
|
||||
#msg-template=
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||
# not used).
|
||||
dummy-variables-rgx=_$|dummy
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,_cb
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format
|
||||
logging-modules=logging
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# List of builtins function names that should not be used, separated by a comma
|
||||
bad-functions=map,filter,input
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
# allow `d` as its used frequently for deferred callback chains
|
||||
good-names=i,j,k,ex,Run,_,d
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=foo,bar,baz,toto,tutu,tata
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name
|
||||
include-naming-hint=no
|
||||
|
||||
# Regular expression matching correct function names
|
||||
function-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for function names
|
||||
function-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct variable names
|
||||
variable-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for variable names
|
||||
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct constant names
|
||||
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Naming hint for constant names
|
||||
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Regular expression matching correct attribute names
|
||||
attr-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for attribute names
|
||||
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct argument names
|
||||
argument-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for argument names
|
||||
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct class attribute names
|
||||
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Naming hint for class attribute names
|
||||
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Regular expression matching correct inline iteration names
|
||||
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Naming hint for inline iteration names
|
||||
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class names
|
||||
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Naming hint for class names
|
||||
class-name-hint=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Regular expression matching correct module names
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Naming hint for module names
|
||||
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Regular expression matching correct method names
|
||||
method-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for method names
|
||||
method-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=^_
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=-1
|
||||
|
||||
|
||||
[ELIF]
|
||||
|
||||
# Maximum number of nested blocks for function / method body
|
||||
max-nested-blocks=5
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||
# install python-enchant package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to indicated private dictionary in
|
||||
# --spelling-private-dict-file option instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=100
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=no
|
||||
|
||||
# List of optional constructs for which whitespace checking is disabled. `dict-
|
||||
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
|
||||
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
|
||||
# `empty-line` allows space-only lines.
|
||||
no-space-check=trailing-comma,dict-separator
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=1000
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,XXX,TODO
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Ignore imports when computing similarities.
|
||||
ignore-imports=no
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||
ignore-mixin-members=yes
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet.reactor,RequestMessage
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,TERMIOS,Bastion,rexec
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
ignored-argument-names=_.*
|
||||
|
||||
# Maximum number of locals for function / method body
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of return / yield for function / method body
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of branch for function / method body
|
||||
max-branches=12
|
||||
|
||||
# Maximum number of statements in function / method body
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
# Maximum number of boolean expressions in a if statement
|
||||
max-bool-expr=5
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,__new__,setUp
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,_fields,_replace,_source,_make
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when being caught. Defaults to
|
||||
# "Exception"
|
||||
overgeneral-exceptions=Exception
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
os: linux
|
||||
dist: trusty
|
||||
language: python
|
||||
python: 2.7
|
||||
|
||||
addons:
|
||||
srcclr:
|
||||
debug: true
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- libgmp3-dev
|
||||
- build-essential
|
||||
- git
|
||||
- libssl-dev
|
||||
- libffi-dev
|
||||
|
||||
before_install:
|
||||
- virtualenv venv
|
||||
- source venv/bin/activate
|
||||
|
||||
install:
|
||||
- pip install -U pip
|
||||
- pip install -r requirements.txt
|
||||
- pip install git+https://github.com/lbryio/lbryum.git
|
||||
- pip install .
|
||||
|
||||
script:
|
||||
- pip install cython
|
||||
- pip install mock pylint unqlite
|
||||
- ./run_pylint.sh
|
||||
- ./run_tests.sh
|
||||
- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
# Change Log
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/) with
|
||||
regard to the json-rpc api. As we're currently pre-1.0 release, we
|
||||
can and probably will change functionality and break backwards compatability
|
||||
at anytime.
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
*
|
||||
*
|
||||
*
|
||||
|
||||
### Changed
|
||||
*
|
||||
*
|
||||
*
|
||||
|
||||
### Fixed
|
||||
*
|
||||
*
|
||||
*
|
||||
|
||||
## [0.9.2rc22] - 2017-04-12
|
||||
### Added
|
||||
* Add `claim_id` parameter to `claim_show`
|
||||
* Add `hex` field to claim responses for the raw claim value
|
||||
* Add an `error` field to to file responses if an error occurs
|
||||
### Changed
|
||||
* Use `uri` instead of `name` in `get_availability`
|
||||
* Add `channel_name` to claim and file responses where applicable
|
||||
* Return None (instead of errors) if a uri cannot be resolved
|
||||
* Use `uri` instead of `name` for `stream_cost_estimate`, update cost estimate for lbryschema
|
||||
### Fixed
|
||||
* `file_list` for files with bad signatures
|
||||
* return None from resolve commands when nothing is found
|
||||
* return lbry files with claims that are abandoned
|
||||
* unhelpful error messages in `publish` and `channel_new`
|
||||
|
||||
## [0.9.2rc9] - 2017-04-08
|
||||
### Added
|
||||
* Use `claim_id` instead of outpoint for `claim_abandon`
|
||||
* Add `channel_name` parameter to `publish`
|
||||
* Add `delete_all` parameter to `file_delete` to allow deleting multiple files
|
||||
* Add `channel_list_mine`
|
||||
* Add `channel_new`
|
||||
* Add `resolve` to resolve lbry uris
|
||||
### Changed
|
||||
* Use `uri` instead of `name` for `get`, remove explicit `claim_id` parameter
|
||||
* Increase default download timeout
|
||||
* Use lbry.io api for exchange rate data
|
||||
|
||||
## [0.9.2rc4] - 2017-04-06
|
||||
### Changed
|
||||
* Use lbryschema library for metadata
|
||||
### Fixed
|
||||
* Removed update_metadata function that could cause update problems
|
||||
* Fix DHT contact bug
|
||||
|
||||
## [0.9.2rc3] - 2017-03-29
|
||||
### Added
|
||||
* Create wallet_unused_address API command
|
||||
* Add `claim_id` parameter to `get`, used to specify non-default claim for `name`
|
||||
### Changed
|
||||
* wallet_new_address API command always returns new address
|
||||
* Improved ConnectionManager speed
|
||||
* Remove unused `stream_info` parameter in `get`
|
||||
|
||||
## [0.9.2rc2] - 2017-03-25
|
||||
### Added
|
||||
* Add `wallet_list` command
|
||||
* Add checks for missing/extraneous params when calling jsonrpc commands
|
||||
* Added colors to cli error messages
|
||||
### Changed
|
||||
* Removed check_pending logic from Daemon
|
||||
* Switched to txrequests so requests can use twisted event loop
|
||||
* Renamed API command file_seed to file_set_status
|
||||
### Fixed
|
||||
* Fix restart procedure in DaemonControl
|
||||
* Create download directory if it doesn't exist
|
||||
* Fixed descriptor_get
|
||||
* Fixed jsonrpc_reflect()
|
||||
* Fixed api help return
|
||||
* Fixed API command descriptor_get
|
||||
* Fixed API command transaction_show
|
||||
* Fixed error handling for jsonrpc commands
|
||||
|
||||
## [0.9.2rc1] - 2017-03-21
|
||||
### Added
|
||||
* Add `wallet_list` command
|
||||
### Changed
|
||||
* Dont add expected payment to wallet when payment rate is 0
|
||||
### Fixed
|
||||
* Fixed descriptor_get
|
||||
* Fixed jsonrpc_reflect()
|
||||
* Fixed api help return
|
||||
* Fixed API command descriptor_get
|
||||
* Fixed API command transaction_show
|
||||
* Handle failure to decode claim cache file
|
||||
|
||||
## [0.9.1] - 2017-03-17
|
||||
### Fixed
|
||||
* Fix wallet_public_key API command
|
||||
|
||||
## [0.9.1rc5] - 2017-03-16
|
||||
### Added
|
||||
* publish API command can take metadata fields as arguments
|
||||
* Added `reflect_uploads` config to disable reflecting on upload
|
||||
### Fixed
|
||||
* Fixed jsonrpc_reflect()
|
||||
* Fixed api help return
|
||||
|
||||
## [0.9.1rc2] - 2017-03-15
|
||||
### Added
|
||||
* Added `--version` flag
|
||||
### Changed
|
||||
* Removed `simplejson` dependency in favor of bulitin `json`
|
||||
|
||||
## [0.9.0rc17] - 2017-03-10
|
||||
### Fixed
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
* Made dictionary key names in API commmand outputs to be more consistent
|
||||
|
||||
## [0.9.0rc15] - 2017-03-09
|
||||
### Added
|
||||
* Add file filters: `claim_id`, `outpoint`, and `rowid`
|
||||
* Make loggly logs less verbose
|
||||
### Changed
|
||||
* Change file filter `uri` to `name` and return field `lbry_uri` to `name`
|
||||
* Refactor file_list, add `full_status` argument to populate resource intensive fields
|
||||
* Remove deprecated file commands: `get_lbry_files`, `get_lbry_file`, and `file_get`
|
||||
* Remove deprecated `delete_lbry_file` command
|
||||
* Return standard file json from `get`
|
||||
### Fixed
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
|
||||
## [0.9.0rc12] - 2017-03-06
|
||||
### Fixed
|
||||
* Fixed ExchangeRateManager freezing the app
|
||||
* Fixed download not timing out properly when downloading sd blob
|
||||
* Fixed ExchangeRateManager freezing the app
|
||||
* Fixed download not timing out properly when downloading sd blob
|
||||
* Fixed get not reassembling an already downloaded file that was deleted from download directory
|
||||
|
||||
## [0.9.0rc11] - 2017-02-27
|
||||
### Fixed
|
||||
* Added timeout to ClientProtocol
|
||||
* Add check for when local height of wallet is less than zero
|
||||
|
||||
## [0.9.0rc9] - 2017-02-22
|
||||
### Changed
|
||||
* Add blockchain status to jsonrpc_status
|
||||
|
||||
## [0.8.7] - 2017-02-21
|
||||
|
||||
## [0.8.6] - 2017-02-19
|
||||
|
||||
## [0.8.6rc0] - 2017-02-19
|
||||
### Changed
|
||||
* Add `file_get` by stream hash
|
||||
* Add utils.call_later to replace reactor.callLater
|
||||
|
||||
### Fixed
|
||||
* Fix unhandled error in `get`
|
||||
* Fix sd blob timeout handling in `get_availability`, return 0.0
|
||||
|
||||
## [0.8.5] - 2017-02-18
|
||||
|
||||
## [0.8.5rc0] - 2017-02-18
|
||||
### Fixed
|
||||
* Fix result expected by ui from file_get for missing files
|
||||
|
||||
## [0.8.4] - 2017-02-17
|
||||
|
||||
## [0.8.4rc0] - 2017-02-17
|
||||
### Changed
|
||||
* Remove unused upload_allowed option
|
||||
* Remove code related to packaging as that step is now done in the electron client
|
||||
* Remove lbryum version check; use lbry-electron as version source
|
||||
* Include download url in version check
|
||||
|
||||
### Fixed
|
||||
* add misssing traceback to logging
|
||||
|
||||
## [0.8.3] - 2017-02-15
|
||||
### Fixed
|
||||
* Get lbry files with pending claims
|
||||
* Add better logging to help track down [#478](https://github.com/lbryio/lbry/issues/478)
|
||||
* Catch UnknownNameErrors when resolving a name. [#479](https://github.com/lbryio/lbry/issues/479)
|
||||
|
||||
### Changed
|
||||
* Add blob_get, descriptor_get, and blob_delete
|
||||
* Add filter keyword args to blob_list
|
||||
* Refactor get_availability
|
||||
* Add optional peer search timeout, add peer_search_timeout setting
|
||||
|
||||
## [0.8.3rc3] - 2017-02-14
|
||||
|
||||
## [0.8.3rc2] - 2017-02-13
|
||||
|
||||
## [0.8.3rc1] - 2017-02-13
|
||||
### Changed
|
||||
* make connection manager unit testeable
|
||||
|
||||
### Fixed
|
||||
* Change EWOULDBLOCK error in DHT to warning. #481
|
||||
* mark peers as down if it fails download protocol
|
||||
* Made hash reannounce time to be adjustable to fix [#432](https://github.com/lbryio/lbry/issues/432)
|
||||
|
||||
|
||||
## [0.8.3rc0] - 2017-02-10
|
||||
### Changed
|
||||
* Convert EncryptedFileDownloader to inlineCallbacks
|
||||
* Convert EncryptedFileManager to use inlineCallbacks
|
||||
* Convert Daemon._delete_lbry_file to inlineCallbacks
|
||||
* Add uri to stream reflector to de-obfuscate reflector logs
|
||||
* Simplify lbrynet.lbrynet_daemon.Publisher
|
||||
* Reflect streams in file manager looping call rather than in each file
|
||||
* Convert GetStream to inclineCallbacks
|
||||
* Change callback condition in GetStream to the first data blob completing
|
||||
* Add local and remote heights to blockchain status
|
||||
|
||||
### Fixed
|
||||
* Fix recursion depth error upon failed blob
|
||||
* Call stopProducing in reflector client file_sender when uploading is done
|
||||
* Ensure streams in stream_info_manager are saved in lbry_file_manager
|
||||
* Fixed file_delete not deleting data from stream_info_manager [#470](https://github.com/lbryio/lbry/issues/470)
|
||||
* Fixed upload of bug reports to Slack ([#472](https://github.com/lbryio/lbry/issues/472))
|
||||
* Fixed claim updates [#473](https://github.com/lbryio/lbry/issues/473)
|
||||
* Handle ConnectionLost error in reflector client
|
||||
* Fix updating a claim where the stream doesn't change
|
||||
* Fix claim_abandon
|
||||
|
||||
## [0.8.1] - 2017-02-01
|
||||
### Changed
|
||||
* reflect all the blobs in a stream
|
||||
* change command line flags so that the more common usage is the default
|
||||
* change daemon function signatures to include names arguments
|
||||
|
||||
### Fixed
|
||||
* disable verbose twisted logs
|
||||
* improved wallet balance calculations
|
||||
* fix block too deep error
|
||||
|
||||
## [0.8.0] - 2017-01-24
|
||||
### Changed
|
||||
* renamed api endpoints
|
||||
* improved command line user experience
|
||||
* integrate twisted logging with python logging
|
||||
* Updated READMEs
|
||||
|
||||
### Fixed
|
||||
* Fixed bug where ConnectionManager wasn't being stopped
|
||||
* Fixed: #343
|
||||
* Stop hanging if github is down
|
||||
* paths for debian package have been updated to be correct
|
||||
* improved output of the publish command
|
||||
@@ -0,0 +1,6 @@
|
||||
# Add a CHANGELOG entry for app changes
|
||||
has_app_changes = !git.modified_files.grep(/lbrynet/).empty?
|
||||
if !git.modified_files.include?("CHANGELOG.md") && has_app_changes
|
||||
fail("Please include a CHANGELOG entry.")
|
||||
message "See http://keepachangelog.com/en/0.3.0/ for details on good changelog guidelines"
|
||||
end
|
||||
@@ -0,0 +1,86 @@
|
||||
#### Getting LBRY for development
|
||||
|
||||
Q: How do I get lbry for command line?
|
||||
|
||||
A: In order to run lbry from command line, you need more than the packaged app/deb.
|
||||
|
||||
###### On OS X
|
||||
|
||||
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app.
|
||||
|
||||
###### On Linux
|
||||
|
||||
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
sudo python setup.py install
|
||||
|
||||
#### Using LBRY
|
||||
|
||||
Q: How do I run lbry from command line?
|
||||
|
||||
A: The command is `lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I stop lbry from the command line?
|
||||
|
||||
A: You can ctrl-c or run `stop-lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I run lbry with lbrycrdd (the blockchain node application)?
|
||||
|
||||
A: Start lbry with the --wallet flag set: `lbrynet-daemon --wallet=lbrycrd`
|
||||
|
||||
Note: when you change the wallet it is persistant until you specify you want to use another wallet - lbryum - with the --wallet flag again.
|
||||
|
||||
***********
|
||||
|
||||
Q: Where are all the behind the scenes files?
|
||||
|
||||
A: On linux, the relevant directories are `~/.lbrynet`, `~/.lbrycrd`, and `~/.lbryum`, depending on which wallets you've used. On OS X, the folders of interest are `~/Library/Application Support/LBRY`, `~/.lbrycrd` and `~/.lbryum`, also depending on which wallets you've used.
|
||||
|
||||
***********
|
||||
|
||||
Q: How can I see the log in the console?
|
||||
|
||||
A: Run lbry with the --log-to-console flag set: `lbrynet-daemon --log-to-console`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I specify a web-UI to use?
|
||||
|
||||
A: If the files for the UI you'd like to use are storred locally on your computer, start lbry with the --ui flag: `lbrynet-daemon --ui=/full/path/to/ui/files/root/folder`
|
||||
|
||||
Note, once set with the UI flag the given UI will be cached by lbry and used as the default going forward. Also, it will only successfully load a UI if it contains a conforming requirements.txt file to specify required lbrynet and lbryum versions. [Here](https://github.com/lbryio/lbry-web-ui/blob/master/dist/requirements.txt) is an example requirements.txt file.
|
||||
|
||||
To reset your ui to pull from lbryio, or to try a UI still in development, run lbry with the --branch flag: `lbrynet=daemon --branch=master`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I see the list of API functions I can call, and how do I call them?
|
||||
|
||||
A: Here is an example script to get the documentation for the various API calls. To use any of the functions displayed, just provide any specified arguments in a dictionary.
|
||||
|
||||
Note: the lbry api can only be used while either the app or lbrynet-daemon command line are running
|
||||
|
||||
import sys
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
sys.exit(0)
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
status = api.status()
|
||||
if not status['is_running']:
|
||||
print status
|
||||
else:
|
||||
for cmd in api.commands():
|
||||
print "%s:\n%s" % (cmd, api.help({'command': cmd}))
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
To use the LBRYWallet, which enables spending and accepting LBRYcrds in exchange for data, the
|
||||
LBRYcrd application (insert link to LBRYcrd website here) must be installed and running. If
|
||||
this is not desired, the testing client can be used to simulate trading points, which is
|
||||
built into LBRYnet.
|
||||
|
||||
on Ubuntu:
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
|
||||
Getting the source
|
||||
------------------
|
||||
|
||||
Don't you already have it?
|
||||
|
||||
Setting up the environment
|
||||
--------------------------
|
||||
|
||||
It's recommended that you use a virtualenv
|
||||
|
||||
sudo apt-get install python-virtualenv
|
||||
cd <source base directory>
|
||||
virtualenv .
|
||||
source bin/activate
|
||||
|
||||
(to deactivate the virtualenv, enter 'deactivate')
|
||||
|
||||
python setup.py install
|
||||
|
||||
this will install all of the libraries and a few applications
|
||||
|
||||
For running the file sharing application, see RUNNING
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
# Installing LBRY
|
||||
|
||||
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### OS X Prerequisites
|
||||
|
||||
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install mpfr libmpc protobuf
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
```
|
||||
|
||||
### Linux Prerequisites
|
||||
|
||||
On Ubuntu (we recommend 16.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
|
||||
```
|
||||
|
||||
If you're running another Linux flavor, install the equivalent of the above packages for your system.
|
||||
|
||||
### Windows Prerequisites
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
|
||||
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- Yapsy
|
||||
- cx_Freeze
|
||||
- requests
|
||||
- gmpy
|
||||
|
||||
Install the above binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
|
||||
|
||||
## Installation
|
||||
|
||||
To install:
|
||||
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
pip install -U -r requirements.txt
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
|
||||
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
## Making a Change
|
||||
|
||||
If you are not already inside the virtual environment, run:
|
||||
|
||||
```
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
This will update `lbrynet-daemon` and other executables.
|
||||
@@ -1,22 +1,15 @@
|
||||
Copyright (c) 2015, LBRY, Inc.
|
||||
All rights reserved.
|
||||
The MIT License (MIT)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
Copyright (c) 2015-2016 LBRY Inc
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -1,57 +0,0 @@
|
||||
LBRYnet
|
||||
=======
|
||||
|
||||
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
|
||||
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
|
||||
table, used by peers to discover other peers.
|
||||
|
||||
Overview
|
||||
--------
|
||||
|
||||
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
|
||||
guarantees that peers can verify the correctness of each chunk without having to know anything
|
||||
about its contents, and can confidently re-transmit the chunk to other peers. Peers wishing to
|
||||
transmit chunks to other peers announce to the distributed hash table that they are associated
|
||||
with the sha384 hash sum in question. When a peer wants to download that chunk from the network,
|
||||
it asks the distributed hash table which peers are associated with that sha384 hash sum. The
|
||||
distributed hash table can also be used more generally. It simply stores IP addresses and
|
||||
ports which are associated with 384-bit numbers, and can be used by any type of application to
|
||||
help peers find each other. For example, an application for which clients don't know all of the
|
||||
necessary chunks may use some identifier, chosen by the application, to find clients which do
|
||||
know all of the necessary chunks.
|
||||
|
||||
Running
|
||||
-------
|
||||
|
||||
LBRYnet comes with an file sharing application, called 'lbrynet-console', which breaks
|
||||
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
|
||||
a special file called a 'stream descriptor' containing the hash sums and some other file metadata,
|
||||
and makes the chunks available for download by other peers. A peer wishing to download the file
|
||||
must first obtain the 'stream descriptor' and then may open it with his 'lbrynet-console' client,
|
||||
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
|
||||
chunks into the original file, according to the metadata included in the 'stream descriptor'.
|
||||
|
||||
To install and use this client, see INSTALL and RUNNING
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
See INSTALL
|
||||
|
||||
Developers
|
||||
----------
|
||||
|
||||
Documentation: doc.lbry.io
|
||||
Source code: trac.lbry.io/browser
|
||||
|
||||
To contribute to the development of LBRYnet or lbrynet-console, contact jimmy@lbry.io
|
||||
|
||||
Support
|
||||
-------
|
||||
|
||||
Send all support requests to jimmy@lbry.io
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
See LICENSE
|
||||
@@ -0,0 +1,52 @@
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
[](https://coveralls.io/github/lbryio/lbry)
|
||||
|
||||
# LBRY
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network.
|
||||
|
||||
This repo is a reference implementation of the [LBRY API](https://lbry.io/api).
|
||||
|
||||
It provides a daemon that can interact with the network via a json-rpc interface over HTTP.
|
||||
|
||||
## Installing
|
||||
|
||||
**Note**: This project no longer directly bundles a graphic interface (browser). If you want to use LBRY via a browser, [use the LBRY App](https://github.com/lbryio/lbry-app).
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry-app/releases/latest) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows.
|
||||
|
||||
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
## Running
|
||||
|
||||
Run `lbrynet-daemon` to launch the daemon.
|
||||
|
||||
## Using
|
||||
|
||||
By default, `lbrynet-daemon` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
Our [quickstart guide](http://lbry.io/quickstart) provides clear sample usages and free credits for learning.
|
||||
|
||||
The full API is documented [here](https://lbry.io/api).
|
||||
|
||||
## What is LBRY?
|
||||
|
||||
LBRY is a protocol providing fully decentralized network for the discovery, distribution, and payment of data.
|
||||
|
||||
It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules.
|
||||
|
||||
LBRY also provides a data network consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
|
||||
|
||||
## Contributions
|
||||
|
||||
To contribute, [join us on Slack](https://slack.lbry.io/) or contact jeremy@lbry.io. Pull requests are also welcome.
|
||||
|
||||
## Support
|
||||
|
||||
Please open an issue and describe your situation in detail. We will respond as soon as we can.
|
||||
|
||||
For private issues, contact jeremy@lbry.io.
|
||||
|
||||
## License
|
||||
|
||||
See [LICENSE](LICENSE)
|
||||
@@ -1,52 +0,0 @@
|
||||
To install LBRYnet and lbrynet-console, see INSTALL
|
||||
|
||||
lbrynet-console is a console application which makes use of the LBRYnet to share files.
|
||||
|
||||
In particular, lbrynet-console splits files into encrypted chunks of data compatible with
|
||||
LBRYnet, groups all metadata into a 'stream descriptor file' which can be sent directly to
|
||||
others wishing to obtain the file, or can be itself turned into a chunk compatible with
|
||||
LBRYnet and downloaded via LBRYnet by anyone knowing its sha384 hashsum. lbrynet-console
|
||||
also acts as a client whichreads a stream descriptor file, downloads the chunks of data
|
||||
specified by the hash sums found in the stream descriptor file, decrypts them according to
|
||||
metadata found in the stream, and reconstructs the original file. lbrynet-console features
|
||||
a server so that clients can connect to it and download the chunks and other data gotten
|
||||
from files created locally and files that have been downloaded from LBRYnet.
|
||||
|
||||
lbrynet-console also has a plugin system. There are two plugins: a live stream proof of
|
||||
concept which is currently far behind the development of the rest of the application and
|
||||
therefore will not run, and a plugin which attempts to determine which chunks on the
|
||||
network should be downloaded in order for the application to turn a profit. It will run,
|
||||
but its usefulness is extremely limited.
|
||||
|
||||
Passing '--help' to lbrynet-console will cause it to print out a quick help message
|
||||
describing other command line options to the application.
|
||||
|
||||
Once the application has been started, the user is presented with a numbered list of
|
||||
actions which looks something like this:
|
||||
|
||||
...
|
||||
[2] Toggle whether an LBRY File is running
|
||||
[3] Create an LBRY File from file
|
||||
[4] Publish a stream descriptor file to the DHT for an LBRY File
|
||||
...
|
||||
|
||||
To perform an action, type the desired number and then hit enter. For example, if you wish
|
||||
to create an LBRY file from a file as described in the beginning of this document, type 3 and
|
||||
hit enter.
|
||||
|
||||
If the application needs more input in order to for the action to be taken, the application
|
||||
will continue to print prompts for input until it has received what it needs.
|
||||
|
||||
For example, when creating an LBRY file from a file, the application needs to know which file
|
||||
it's supposed to use to create the LBRY file, so the user will be prompted for it:
|
||||
|
||||
File name:
|
||||
|
||||
The user should input the desired file name and hit enter, at which point the application
|
||||
will go about splitting the file and making it available on the network.
|
||||
|
||||
Some actions will produce sub-menus of actions, which work the same way.
|
||||
|
||||
A more detailed user guide is available at doc.lbry.io
|
||||
|
||||
Any issues may be reported to jimmy@lbry.io
|
||||
@@ -0,0 +1,79 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
width="80.207558mm"
|
||||
height="58.081333mm"
|
||||
viewBox="0 0 284.20001 205.8"
|
||||
id="svg3479"
|
||||
version="1.1"
|
||||
inkscape:version="0.91 r13725"
|
||||
sodipodi:docname="lbry-white-logo-only.svg">
|
||||
<defs
|
||||
id="defs3481" />
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="0.35"
|
||||
inkscape:cx="-470.0429"
|
||||
inkscape:cy="-5.6714247"
|
||||
inkscape:document-units="px"
|
||||
inkscape:current-layer="layer1"
|
||||
showgrid="false"
|
||||
inkscape:window-width="1920"
|
||||
inkscape:window-height="1056"
|
||||
inkscape:window-x="0"
|
||||
inkscape:window-y="24"
|
||||
inkscape:window-maximized="1"
|
||||
fit-margin-top="0"
|
||||
fit-margin-left="0"
|
||||
fit-margin-right="0"
|
||||
fit-margin-bottom="0" />
|
||||
<metadata
|
||||
id="metadata3484">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title></dc:title>
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Layer 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(-363.61433,-320.89078)">
|
||||
<g
|
||||
id="g3396"
|
||||
transform="translate(363.61433,320.89078)">
|
||||
<g
|
||||
id="g3398">
|
||||
<polygon
|
||||
style="fill:#ffffff"
|
||||
points="138.8,155.2 271,74 271,68.2 146.2,8 7,94.1 7,132.6 138.8,197.8 276.4,113.4 280.3,119.4 139.2,205.8 0,137 0,90.2 145.8,0 278,63.8 278,77.9 139.2,163.2 34.6,111.9 34.8,104 "
|
||||
id="polygon3400" />
|
||||
</g>
|
||||
<g
|
||||
id="g3402">
|
||||
<polygon
|
||||
style="fill:#ffffff"
|
||||
points="276.5,128.5 278.5,115.9 266.3,113.8 267.1,108.9 284.2,111.8 281.4,129.3 "
|
||||
id="polygon3404" />
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.3 KiB |
+651
@@ -0,0 +1,651 @@
|
||||
# LBRY JSON-RPC API Documentation
|
||||
|
||||
## blob_announce_all
|
||||
|
||||
```text
|
||||
Announce all blobs to the DHT
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(str) Success/fail message
|
||||
```
|
||||
|
||||
## blob_delete
|
||||
|
||||
```text
|
||||
Delete a blob
|
||||
|
||||
Args:
|
||||
'blob_hash': (str) hash of blob to get
|
||||
Returns:
|
||||
(str) Success/fail message
|
||||
```
|
||||
|
||||
## blob_get
|
||||
|
||||
```text
|
||||
Download and return a blob
|
||||
|
||||
Args:
|
||||
'blob_hash': (str) blob hash of blob to get
|
||||
'timeout'(optional): (int) timeout in number of seconds
|
||||
'encoding'(optional): (str) by default no attempt at decoding is made,
|
||||
can be set to one of the following decoders:
|
||||
'json'
|
||||
'payment_rate_manager'(optional): if not given the default payment rate manager
|
||||
will be used. supported alternative rate managers:
|
||||
'only-free'
|
||||
|
||||
Returns
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
```
|
||||
|
||||
## blob_list
|
||||
|
||||
```text
|
||||
Returns blob hashes. If not given filters, returns all blobs known by the blob manager
|
||||
|
||||
Args:
|
||||
'uri' (optional): (str) filter by blobs in stream for winning claim
|
||||
'stream_hash' (optional): (str) filter by blobs in given stream hash
|
||||
'sd_hash' (optional): (str) filter by blobs in given sd hash
|
||||
'needed' (optional): (bool) only return needed blobs
|
||||
'finished' (optional): (bool) only return finished blobs
|
||||
'page_size' (optional): (int) limit number of results returned
|
||||
'page' (optional): (int) filter to page x of [page_size] results
|
||||
Returns:
|
||||
(list) List of blob hashes
|
||||
```
|
||||
|
||||
## blob_reflect_all
|
||||
|
||||
```text
|
||||
Reflects all saved blobs
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## block_show
|
||||
|
||||
```text
|
||||
Get contents of a block
|
||||
|
||||
Args:
|
||||
'blockhash': (str) hash of the block to look up
|
||||
Returns:
|
||||
(dict) Requested block
|
||||
```
|
||||
|
||||
## claim_abandon
|
||||
|
||||
```text
|
||||
Abandon a name and reclaim credits from the claim
|
||||
|
||||
Args:
|
||||
'txid': (str) txid of claim
|
||||
'nout': (int) nout of claim
|
||||
Return:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
txid : (str) txid of resulting transaction
|
||||
fee : (float) fee paid for the transaction
|
||||
}
|
||||
```
|
||||
|
||||
## claim_list
|
||||
|
||||
```text
|
||||
Get claims for a name
|
||||
|
||||
Args:
|
||||
'name': (str) search for claims on this name
|
||||
Returns
|
||||
(dict) State of claims assigned for the name
|
||||
{
|
||||
'claims': (list) list of claims for the name
|
||||
[
|
||||
{
|
||||
'amount': (float) amount assigned to the claim
|
||||
'effective_amount': (float) total amount assigned to the claim,
|
||||
including supports
|
||||
'claim_id': (str) claim ID of the claim
|
||||
'height': (int) height of block containing the claim
|
||||
'txid': (str) txid of the claim
|
||||
'nout': (int) nout of the claim
|
||||
'supports': (list) a list of supports attached to the claim
|
||||
'value': (str) the value of the claim
|
||||
},
|
||||
]
|
||||
'supports_without_claims': (list) supports without any claims attached to them
|
||||
'last_takeover_height': (int) the height of last takeover for the name
|
||||
}
|
||||
```
|
||||
|
||||
## claim_list_mine
|
||||
|
||||
```text
|
||||
List my name claims
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns
|
||||
(list) List of name claims owned by user
|
||||
[
|
||||
{
|
||||
'address': (str) address that owns the claim
|
||||
'amount': (float) amount assigned to the claim
|
||||
'blocks_to_expiration': (int) number of blocks until it expires
|
||||
'category': (str) "claim", "update" , or "support"
|
||||
'claim_id': (str) claim ID of the claim
|
||||
'confirmations': (int) number of blocks of confirmations for the claim
|
||||
'expiration_height': (int) the block height which the claim will expire
|
||||
'expired': (bool) true if expired, false otherwise
|
||||
'height': (int) height of the block containing the claim
|
||||
'is_spent': (bool) true if claim is abandoned, false otherwise
|
||||
'name': (str) name of the claim
|
||||
'txid': (str) txid of the cliam
|
||||
'nout': (int) nout of the claim
|
||||
'value': (str) value of the claim
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
## claim_new_support
|
||||
|
||||
```text
|
||||
Support a name claim
|
||||
|
||||
Args:
|
||||
'name': (str) name
|
||||
'claim_id': (str) claim ID of claim to support
|
||||
'amount': (float) amount to support by
|
||||
Return:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
txid : (str) txid of resulting support claim
|
||||
nout : (int) nout of the resulting support claim
|
||||
fee : (float) fee paid for the transaction
|
||||
}
|
||||
```
|
||||
|
||||
## claim_show
|
||||
|
||||
```text
|
||||
Resolve claim info from a LBRY name
|
||||
|
||||
Args:
|
||||
'name': (str) name to look up, do not include lbry:// prefix
|
||||
'txid'(optional): (str) if specified, look for claim with this txid
|
||||
'nout'(optional): (int) if specified, look for claim with this nout
|
||||
Returns:
|
||||
(dict) Dictionary contaning claim info, (bool) false if claim is not
|
||||
resolvable
|
||||
|
||||
{
|
||||
'txid': (str) txid of claim
|
||||
'nout': (int) nout of claim
|
||||
'amount': (float) amount of claim
|
||||
'value': (str) value of claim
|
||||
'height' : (int) height of claim takeover
|
||||
'claim_id': (str) claim ID of claim
|
||||
'supports': (list) list of supports associated with claim
|
||||
}
|
||||
```
|
||||
|
||||
## commands
|
||||
|
||||
```text
|
||||
Return a list of available commands
|
||||
|
||||
Returns:
|
||||
(list) list of available commands
|
||||
```
|
||||
|
||||
## daemon_stop
|
||||
|
||||
```text
|
||||
Stop lbrynet-daemon
|
||||
|
||||
Returns:
|
||||
(string) Shutdown message
|
||||
```
|
||||
|
||||
## descriptor_get
|
||||
|
||||
```text
|
||||
Download and return a sd blob
|
||||
|
||||
Args:
|
||||
'sd_hash': (str) hash of sd blob
|
||||
'timeout'(optional): (int) timeout in number of seconds
|
||||
'payment_rate_manager'(optional): (str) if not given the default payment rate manager
|
||||
will be used. supported alternative rate managers:
|
||||
only-free
|
||||
|
||||
Returns
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
```
|
||||
|
||||
## file_delete
|
||||
|
||||
```text
|
||||
Delete a lbry file
|
||||
|
||||
Args:
|
||||
'name' (optional): (str) delete file by lbry name,
|
||||
'sd_hash' (optional): (str) delete file by sd hash,
|
||||
'file_name' (optional): (str) delete file by the name in the downloads folder,
|
||||
'stream_hash' (optional): (str) delete file by stream hash,
|
||||
'claim_id' (optional): (str) delete file by claim ID,
|
||||
'outpoint' (optional): (str) delete file by claim outpoint,
|
||||
'rowid': (optional): (int) delete file by rowid in the file manager
|
||||
'delete_target_file' (optional): (bool) delete file from downloads folder,
|
||||
defaults to true if false only the blobs and
|
||||
db entries will be deleted
|
||||
Returns:
|
||||
(bool) true if deletion was successful
|
||||
```
|
||||
|
||||
## file_list
|
||||
|
||||
```text
|
||||
List files limited by optional filters
|
||||
|
||||
Args:
|
||||
'name' (optional): (str) filter files by lbry name,
|
||||
'sd_hash' (optional): (str) filter files by sd hash,
|
||||
'file_name' (optional): (str) filter files by the name in the downloads folder,
|
||||
'stream_hash' (optional): (str) filter files by stream hash,
|
||||
'claim_id' (optional): (str) filter files by claim id,
|
||||
'outpoint' (optional): (str) filter files by claim outpoint,
|
||||
'rowid' (optional): (int) filter files by internal row id,
|
||||
'full_status': (optional): (bool) if true populate the 'message' and 'size' fields
|
||||
|
||||
Returns:
|
||||
(list) List of files
|
||||
|
||||
[
|
||||
{
|
||||
'completed': (bool) true if download is completed,
|
||||
'file_name': (str) name of file,
|
||||
'download_directory': (str) download directory,
|
||||
'points_paid': (float) credit paid to download file,
|
||||
'stopped': (bool) true if download is stopped,
|
||||
'stream_hash': (str) stream hash of file,
|
||||
'stream_name': (str) stream name ,
|
||||
'suggested_file_name': (str) suggested file name,
|
||||
'sd_hash': (str) sd hash of file,
|
||||
'name': (str) name claim attached to file
|
||||
'outpoint': (str) claim outpoint attached to file
|
||||
'claim_id': (str) claim ID attached to file,
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes, None if full_status is false
|
||||
'written_bytes': (int) written size in bytes
|
||||
'message': (str), None if full_status is false
|
||||
'metadata': (dict) Metadata dictionary
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
## file_set_status
|
||||
|
||||
```text
|
||||
Start or stop downloading a file
|
||||
|
||||
Args:
|
||||
'status': (str) "start" or "stop"
|
||||
'name' (optional): (str) start file by lbry name,
|
||||
'sd_hash' (optional): (str) start file by the hash in the name claim,
|
||||
'file_name' (optional): (str) start file by its name in the downloads folder,
|
||||
Returns:
|
||||
(str) Confirmation message
|
||||
```
|
||||
|
||||
## get
|
||||
|
||||
```text
|
||||
Download stream from a LBRY name.
|
||||
|
||||
Args:
|
||||
'name': (str) name to download
|
||||
'file_name'(optional): (str) a user specified name for the downloaded file
|
||||
'stream_info'(optional): (str) specified stream info overrides name
|
||||
'timeout'(optional): (int) download timeout in number of seconds
|
||||
'download_directory'(optional): (str) path to directory where file will be saved
|
||||
'wait_for_write'(optional): (bool) defaults to True. When set, waits for the file to
|
||||
only start to be written before returning any results.
|
||||
Returns:
|
||||
(dict) Dictionary contaning information about the stream
|
||||
|
||||
{
|
||||
'completed': (bool) true if download is completed,
|
||||
'file_name': (str) name of file,
|
||||
'download_directory': (str) download directory,
|
||||
'points_paid': (float) credit paid to download file,
|
||||
'stopped': (bool) true if download is stopped,
|
||||
'stream_hash': (str) stream hash of file,
|
||||
'stream_name': (str) stream name ,
|
||||
'suggested_file_name': (str) suggested file name,
|
||||
'sd_hash': (str) sd hash of file,
|
||||
'name': (str) name claim attached to file
|
||||
'outpoint': (str) claim outpoint attached to file
|
||||
'claim_id': (str) claim ID attached to file,
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes, None if full_status is false
|
||||
'written_bytes': (int) written size in bytes
|
||||
'message': (str), None if full_status is false
|
||||
'metadata': (dict) Metadata dictionary
|
||||
}
|
||||
```
|
||||
|
||||
## get_availability
|
||||
|
||||
```text
|
||||
Get stream availability for a winning claim
|
||||
|
||||
Args:
|
||||
'name' : (str) lbry name
|
||||
'sd_timeout' (optional): (int) sd blob download timeout
|
||||
'peer_timeout' (optional): (int) how long to look for peers
|
||||
|
||||
Returns:
|
||||
(float) Peers per blob / total blobs
|
||||
```
|
||||
|
||||
## help
|
||||
|
||||
```text
|
||||
Return a useful message for an API command
|
||||
|
||||
Args:
|
||||
'command'(optional): (str) command to retrieve documentation for
|
||||
Returns:
|
||||
(str) if given a command, returns documentation about that command
|
||||
otherwise returns general help message
|
||||
```
|
||||
|
||||
## peer_list
|
||||
|
||||
```text
|
||||
Get peers for blob hash
|
||||
|
||||
Args:
|
||||
'blob_hash': (str) blob hash
|
||||
'timeout'(optional): (int) peer search timeout in seconds
|
||||
Returns:
|
||||
(list) List of contacts
|
||||
```
|
||||
|
||||
## publish
|
||||
|
||||
```text
|
||||
Make a new name claim and publish associated data to lbrynet,
|
||||
update over existing claim if user already has a claim for name.
|
||||
|
||||
Fields required in the final Metadata are:
|
||||
'title'
|
||||
'description'
|
||||
'author'
|
||||
'language'
|
||||
'license',
|
||||
'nsfw'
|
||||
|
||||
Metadata can be set by either using the metadata argument or by setting individual arguments
|
||||
fee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,
|
||||
or sources. Individual arguments will overwrite the fields specified in metadata argument.
|
||||
|
||||
Args:
|
||||
'name': (str) name to be claimed
|
||||
'bid': (float) amount of credits to commit in this claim,
|
||||
'metadata'(optional): (dict) Metadata to associate with the claim.
|
||||
'file_path'(optional): (str) path to file to be associated with name. If provided,
|
||||
a lbry stream of this file will be used in 'sources'.
|
||||
If no path is given but a metadata dict is provided, the source
|
||||
from the given metadata will be used.
|
||||
'fee'(optional): (dict) Dictionary representing key fee to download content:
|
||||
{currency_symbol: {'amount': float, 'address': str, optional}}
|
||||
supported currencies: LBC, USD, BTC
|
||||
If an address is not provided a new one will be automatically
|
||||
generated. Default fee is zero.
|
||||
'title'(optional): (str) title of the file
|
||||
'description'(optional): (str) description of the file
|
||||
'author'(optional): (str) author of the file
|
||||
'language'(optional): (str), language code
|
||||
'license'(optional): (str) license for the file
|
||||
'license_url'(optional): (str) URL to license
|
||||
'thumbnail'(optional): (str) thumbnail URL for the file
|
||||
'preview'(optional): (str) preview URL for the file
|
||||
'nsfw'(optional): (bool) True if not safe for work
|
||||
'sources'(optional): (dict){'lbry_sd_hash':sd_hash} specifies sd hash of file
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
{
|
||||
'tx' : (str) hex encoded transaction
|
||||
'txid' : (str) txid of resulting claim
|
||||
'nout' : (int) nout of the resulting claim
|
||||
'fee' : (float) fee paid for the claim transaction
|
||||
'claim_id' : (str) claim ID of the resulting claim
|
||||
}
|
||||
```
|
||||
|
||||
## reflect
|
||||
|
||||
```text
|
||||
Reflect a stream
|
||||
|
||||
Args:
|
||||
'sd_hash': (str) sd_hash of lbry file
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## report_bug
|
||||
|
||||
```text
|
||||
Report a bug to slack
|
||||
|
||||
Args:
|
||||
'message': (str) message to send
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
```
|
||||
|
||||
## resolve_name
|
||||
|
||||
```text
|
||||
Resolve stream info from a LBRY name
|
||||
|
||||
Args:
|
||||
'name': (str) name to look up, do not include lbry:// prefix
|
||||
Returns:
|
||||
(dict) Metadata dictionary from name claim, None if the name is not
|
||||
resolvable
|
||||
```
|
||||
|
||||
## send_amount_to_address
|
||||
|
||||
```text
|
||||
Send credits to an address
|
||||
|
||||
Args:
|
||||
'amount': (float) the amount to send
|
||||
'address': (str) the address of the recipient in base58
|
||||
Returns:
|
||||
(bool) true if payment successfully scheduled
|
||||
```
|
||||
|
||||
## settings_get
|
||||
|
||||
```text
|
||||
Get daemon settings
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary of daemon settings
|
||||
See ADJUSTABLE_SETTINGS in lbrynet/conf.py for full list of settings
|
||||
```
|
||||
|
||||
## settings_set
|
||||
|
||||
```text
|
||||
Set daemon settings
|
||||
|
||||
Args:
|
||||
'run_on_startup': (bool) currently not supported
|
||||
'data_rate': (float) data rate,
|
||||
'max_key_fee': (float) maximum key fee,
|
||||
'download_directory': (str) path of where files are downloaded,
|
||||
'max_upload': (float), currently not supported
|
||||
'max_download': (float), currently not supported
|
||||
'download_timeout': (int) download timeout in seconds
|
||||
'search_timeout': (float) search timeout in seconds
|
||||
'cache_time': (int) cache timeout in seconds
|
||||
Returns:
|
||||
(dict) Updated dictionary of daemon settings
|
||||
```
|
||||
|
||||
## status
|
||||
|
||||
```text
|
||||
Return daemon status
|
||||
|
||||
Args:
|
||||
'session_status' (optional): (bool) true to return session status,
|
||||
default is false
|
||||
Returns:
|
||||
(dict) Daemon status dictionary
|
||||
```
|
||||
|
||||
## stream_cost_estimate
|
||||
|
||||
```text
|
||||
Get estimated cost for a lbry stream
|
||||
|
||||
Args:
|
||||
'name': (str) lbry name
|
||||
'size' (optional): (int) stream size, in bytes. if provided an sd blob
|
||||
won't be downloaded.
|
||||
Returns:
|
||||
(float) Estimated cost in lbry credits
|
||||
```
|
||||
|
||||
## transaction_list
|
||||
|
||||
```text
|
||||
List transactions belonging to wallet
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(list) List of transactions
|
||||
```
|
||||
|
||||
## transaction_show
|
||||
|
||||
```text
|
||||
Get a decoded transaction from a txid
|
||||
|
||||
Args:
|
||||
'txid': (str) txid of transaction
|
||||
Returns:
|
||||
(dict) JSON formatted transaction
|
||||
```
|
||||
|
||||
## version
|
||||
|
||||
```text
|
||||
Get lbry version information
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(dict) Dictionary of lbry version information
|
||||
{
|
||||
'build': (str) build type (e.g. "dev", "rc", "release"),
|
||||
'ip': (str) remote ip, if available,
|
||||
'lbrynet_update_available': (bool) whether there's an update available,
|
||||
'lbrynet_version': (str) lbrynet_version,
|
||||
'lbryum_version': (str) lbryum_version,
|
||||
'os_release': (str) os release string
|
||||
'os_system': (str) os name
|
||||
'platform': (str) platform string
|
||||
'processor': (str) processor type,
|
||||
'python_version': (str) python version,
|
||||
'remote_lbrynet': (str) most recent lbrynet version available from github
|
||||
}
|
||||
```
|
||||
|
||||
## wallet_balance
|
||||
|
||||
```text
|
||||
Return the balance of the wallet
|
||||
|
||||
Returns:
|
||||
(float) amount of lbry credits in wallet
|
||||
```
|
||||
|
||||
## wallet_is_address_mine
|
||||
|
||||
```text
|
||||
Checks if an address is associated with the current wallet.
|
||||
|
||||
Args:
|
||||
'address': (str) address to check in base58
|
||||
Returns:
|
||||
(bool) true, if address is associated with current wallet
|
||||
```
|
||||
|
||||
## wallet_list
|
||||
|
||||
```text
|
||||
List wallet addresses
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
List of wallet addresses
|
||||
```
|
||||
|
||||
## wallet_new_address
|
||||
|
||||
```text
|
||||
Generate a new wallet address
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(str) New wallet address in base58
|
||||
```
|
||||
|
||||
## wallet_public_key
|
||||
|
||||
```text
|
||||
Get public key from wallet address
|
||||
|
||||
Args:
|
||||
'address': (str) wallet address in base58
|
||||
Returns:
|
||||
(list) list of public keys associated with address.
|
||||
Could contain more than one public key if multisig.
|
||||
```
|
||||
|
||||
## wallet_unused_address
|
||||
|
||||
```text
|
||||
Return an address containing no balance, will create
|
||||
a new address if there is none.
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
(str) Unused wallet address in base58
|
||||
```
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
import logging
|
||||
|
||||
__version__ = "0.9.2rc24"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -0,0 +1,6 @@
|
||||
from constants import *
|
||||
from events import *
|
||||
from api import Api
|
||||
from track import Track
|
||||
from manager import Manager
|
||||
from logging_handler import Handler
|
||||
@@ -0,0 +1,81 @@
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
|
||||
from requests import auth
|
||||
from txrequests import Session
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.analytics import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def log_response(fn):
|
||||
def _log_error(failure):
|
||||
log.warning('Failed to send an analytics event. %s', failure.getTraceback())
|
||||
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
d = fn(*args, **kwargs)
|
||||
d.addErrback(_log_error)
|
||||
return d
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class Api(object):
|
||||
def __init__(self, session, url, write_key):
|
||||
self.session = session
|
||||
self.url = url
|
||||
self.write_key = write_key
|
||||
|
||||
def post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
assert endpoint[0] == '/'
|
||||
headers = {"Connection": "close"}
|
||||
return self.session.post(
|
||||
self.url + endpoint, json=data, auth=self.auth, headers=headers)
|
||||
|
||||
@property
|
||||
def auth(self):
|
||||
return auth.HTTPBasicAuth(self.write_key, '')
|
||||
|
||||
@log_response
|
||||
def batch(self, events):
|
||||
"""Send multiple events in one request.
|
||||
|
||||
Each event needs to have its type specified.
|
||||
"""
|
||||
data = json.dumps({
|
||||
'batch': events,
|
||||
'sentAt': utils.now(),
|
||||
})
|
||||
log.debug('sending %s events', len(events))
|
||||
log.debug('Data: %s', data)
|
||||
return self.post('/batch', data)
|
||||
|
||||
@log_response
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
log.debug('Sending track event: %s', event)
|
||||
return self.post('/track', event)
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, session=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if not session:
|
||||
session = Session()
|
||||
return cls(
|
||||
session,
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN'])
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Constants for metrics"""
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
BLOB_BYTES_AVAILABLE = 'Blob Bytes Available'
|
||||
@@ -0,0 +1,128 @@
|
||||
import logging
|
||||
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.conf import LBRYUM_WALLET
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_sd_hash(stream_info):
|
||||
if not stream_info:
|
||||
return None
|
||||
try:
|
||||
return stream_info['stream']['source']['source']
|
||||
except (KeyError, TypeError, ValueError):
|
||||
log.debug('Failed to get sd_hash from %s', stream_info, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
class Events(object):
|
||||
def __init__(self, context, installation_id, session_id):
|
||||
"""Contains all of the analytics events that can be sent
|
||||
|
||||
Attributes:
|
||||
context: usually the output of `make_context`
|
||||
installation_id: id unique to this installation. Can be anything, but
|
||||
generally should be base58 encoded.
|
||||
session_id: id for tracking events during this session. Can be
|
||||
anything, but generally should be base58 encoded.
|
||||
"""
|
||||
self.context = context
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
|
||||
def update_context(self, context):
|
||||
self.context = context
|
||||
|
||||
def server_startup(self):
|
||||
return self._event('Server Startup')
|
||||
|
||||
def server_startup_success(self):
|
||||
return self._event('Server Startup Success')
|
||||
|
||||
def server_startup_error(self, message):
|
||||
return self._event('Server Startup Error', {
|
||||
'message': message,
|
||||
})
|
||||
|
||||
def heartbeat(self):
|
||||
return self._event('Heartbeat')
|
||||
|
||||
def download_started(self, *args, **kwargs):
|
||||
properties = download_properties(*args, **kwargs)
|
||||
return self._event('Download Started', properties)
|
||||
|
||||
def download_errored(self, *args, **kwargs):
|
||||
properties = download_properties(*args, **kwargs)
|
||||
return self._event('Download Errored', properties)
|
||||
|
||||
def download_finished(self, *args, **kwargs):
|
||||
properties = download_properties(*args, **kwargs)
|
||||
return self._event('Download Finished', properties)
|
||||
|
||||
def error(self, log_record):
|
||||
"""Record when a log message of ERROR or higher was emitted"""
|
||||
properties = {
|
||||
'message': log_record.message,
|
||||
'module': log_record.module,
|
||||
'lineno': log_record.lineno,
|
||||
'name': log_record.name,
|
||||
'traceback': log_record.exc_text,
|
||||
}
|
||||
return self._event('Error', properties)
|
||||
|
||||
def metric_observed(self, metric_name, value):
|
||||
properties = {
|
||||
'value': value,
|
||||
}
|
||||
return self._event(metric_name, properties)
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _properties(self, event_properties=None):
|
||||
event_properties = event_properties or {}
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties)
|
||||
return properties
|
||||
|
||||
|
||||
def make_context(platform, wallet):
|
||||
return {
|
||||
'app': {
|
||||
'name': 'lbrynet',
|
||||
'version': platform['lbrynet_version'],
|
||||
'python_version': platform['python_version'],
|
||||
'build': platform['build'],
|
||||
'wallet': {
|
||||
'name': wallet,
|
||||
'version': platform['lbryum_version'] if wallet == LBRYUM_WALLET else None
|
||||
},
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
'library': {
|
||||
'name': 'lbrynet-analytics',
|
||||
'version': '1.0.0'
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def download_properties(id_, name, stream_info=None):
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': get_sd_hash(stream_info)
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import logging
|
||||
|
||||
|
||||
class Handler(logging.Handler):
|
||||
"""A logging handler that reports errors to the analytics manager"""
|
||||
def __init__(self, manager, level=logging.ERROR):
|
||||
self.manager = manager
|
||||
logging.Handler.__init__(self, level)
|
||||
|
||||
def emit(self, record):
|
||||
# We need to call format to ensure that record.message and
|
||||
# record.exc_text attributes are populated
|
||||
self.format(record)
|
||||
self.manager.send_error(record)
|
||||
@@ -0,0 +1,116 @@
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import task
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import looping_call_manager
|
||||
from lbrynet.core.system_info import get_platform
|
||||
|
||||
import constants
|
||||
from api import Api
|
||||
from events import Events, make_context
|
||||
from track import Track
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, analytics_api, events_generator, track):
|
||||
self.analytics_api = analytics_api
|
||||
self.events_generator = events_generator
|
||||
self.track = track
|
||||
self.looping_call_manager = self.setup_looping_calls()
|
||||
self.is_started = False
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, api=None, events=None):
|
||||
if api is None:
|
||||
api = Api.new_instance()
|
||||
if events is None:
|
||||
events = Events(
|
||||
make_context(get_platform(), conf.settings['wallet']),
|
||||
conf.settings.installation_id,
|
||||
conf.settings.get_session_id(),
|
||||
)
|
||||
return cls(api, events, Track())
|
||||
|
||||
def update_events_generator(self, events_generator):
|
||||
self.events_generator = events_generator
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 60),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 300),
|
||||
]
|
||||
|
||||
def setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def send_server_startup(self):
|
||||
event = self.events_generator.server_startup()
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_server_startup_success(self):
|
||||
event = self.events_generator.server_startup_success()
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
event = self.events_generator.server_startup_error(message)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_download_started(self, id_, name, stream_info=None):
|
||||
event = self.events_generator.download_started(id_, name, stream_info)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_download_errored(self, id_, name, stream_info=None):
|
||||
event = self.events_generator.download_errored(id_, name, stream_info)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_download_finished(self, id_, name, stream_info=None):
|
||||
event = self.events_generator.download_finished(id_, name, stream_info)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_error(self, message):
|
||||
event = self.events_generator.error(message)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _send_heartbeat(self):
|
||||
heartbeat = self.events_generator.heartbeat()
|
||||
self.analytics_api.track(heartbeat)
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
should_send, value = self.track.summarize_and_reset(constants.BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
event = self.events_generator.metric_observed(constants.BLOB_BYTES_UPLOADED, value)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
event = self.events_generator.metric_observed(event_name, value)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
|
||||
def if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
@@ -0,0 +1,24 @@
|
||||
import collections
|
||||
|
||||
|
||||
class Track(object):
|
||||
"""Track and summarize observations of metrics."""
|
||||
def __init__(self):
|
||||
self.data = collections.defaultdict(list)
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self.data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self.data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
@@ -0,0 +1,2 @@
|
||||
# dont touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
+468
-20
@@ -1,25 +1,473 @@
|
||||
"""
|
||||
Some network wide and also application specific parameters
|
||||
"""
|
||||
|
||||
|
||||
import base58
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
import envparse
|
||||
from appdirs import user_data_dir
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ENV_NAMESPACE = 'LBRY_'
|
||||
|
||||
LBRYCRD_WALLET = 'lbrycrd'
|
||||
LBRYUM_WALLET = 'lbryum'
|
||||
PTC_WALLET = 'ptc'
|
||||
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
APP_NAME = 'LBRY'
|
||||
|
||||
LINUX = 1
|
||||
DARWIN = 2
|
||||
WINDOWS = 3
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
DEFAULT_DHT_NODES = [
|
||||
('lbrynet1.lbry.io', 4444),
|
||||
('lbrynet2.lbry.io', 4444),
|
||||
('lbrynet3.lbry.io', 4444)
|
||||
]
|
||||
|
||||
settings_decoders = {
|
||||
'.json': json.loads,
|
||||
'.yml': yaml.load
|
||||
}
|
||||
|
||||
settings_encoders = {
|
||||
'.json': json.dumps,
|
||||
'.yml': yaml.safe_dump
|
||||
}
|
||||
|
||||
if sys.platform.startswith('darwin'):
|
||||
platform = DARWIN
|
||||
default_download_directory = os.path.join(os.path.expanduser('~'), 'Downloads')
|
||||
default_data_dir = user_data_dir('LBRY')
|
||||
default_lbryum_dir = os.path.join(os.path.expanduser('~'), '.lbryum')
|
||||
elif sys.platform.startswith('win'):
|
||||
platform = WINDOWS
|
||||
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
|
||||
|
||||
default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
default_data_dir = os.path.join(
|
||||
get_path(FOLDERID.RoamingAppData, UserHandle.current), 'lbrynet')
|
||||
default_lbryum_dir = os.path.join(
|
||||
get_path(FOLDERID.RoamingAppData, UserHandle.current), 'lbryum')
|
||||
else:
|
||||
platform = LINUX
|
||||
default_download_directory = os.path.join(os.path.expanduser('~'), 'Downloads')
|
||||
default_data_dir = os.path.join(os.path.expanduser('~'), '.lbrynet')
|
||||
default_lbryum_dir = os.path.join(os.path.expanduser('~'), '.lbryum')
|
||||
|
||||
ICON_PATH = 'icons' if platform is WINDOWS else 'app.icns'
|
||||
|
||||
|
||||
MAX_HANDSHAKE_SIZE = 2**16
|
||||
MAX_REQUEST_SIZE = 2**16
|
||||
MAX_BLOB_REQUEST_SIZE = 2**16
|
||||
MAX_RESPONSE_INFO_SIZE = 2**16
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
BLOBFILES_DIR = ".blobfiles"
|
||||
BLOB_SIZE = 2**21
|
||||
MIN_BLOB_DATA_PAYMENT_RATE = .5 # points/megabyte
|
||||
MIN_BLOB_INFO_PAYMENT_RATE = 2.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MAX_CONNECTIONS_PER_STREAM = 5
|
||||
def server_port(server_and_port):
|
||||
server, port = server_and_port.split(':')
|
||||
return server, int(port)
|
||||
|
||||
POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
|
||||
#POINTTRADER_SERVER = 'http://127.0.0.1:2424'
|
||||
|
||||
CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
class Env(envparse.Env):
|
||||
"""An Env parser that automatically namespaces the variables with LBRY"""
|
||||
|
||||
def __init__(self, **schema):
|
||||
self.original_schema = schema
|
||||
my_schema = {
|
||||
self._convert_key(key): self._convert_value(value)
|
||||
for key, value in schema.items()
|
||||
}
|
||||
envparse.Env.__init__(self, **my_schema)
|
||||
|
||||
def __call__(self, key, *args, **kwargs):
|
||||
my_key = self._convert_key(key)
|
||||
return super(Env, self).__call__(my_key, *args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _convert_key(key):
|
||||
return ENV_NAMESPACE + key.upper()
|
||||
|
||||
@staticmethod
|
||||
def _convert_value(value):
|
||||
""" Allow value to be specified as a tuple or list.
|
||||
|
||||
If you do this, the tuple/list must be of the
|
||||
form (cast, default) or (cast, default, subcast)
|
||||
"""
|
||||
if isinstance(value, (tuple, list)):
|
||||
new_value = {'cast': value[0], 'default': value[1]}
|
||||
if len(value) == 3:
|
||||
new_value['subcast'] = value[2]
|
||||
return new_value
|
||||
return value
|
||||
|
||||
TYPE_DEFAULT = 'default'
|
||||
TYPE_PERSISTED = 'persisted'
|
||||
TYPE_ENV = 'env'
|
||||
TYPE_CLI = 'cli'
|
||||
TYPE_RUNTIME = 'runtime'
|
||||
|
||||
FIXED_SETTINGS = {
|
||||
'ANALYTICS_ENDPOINT': 'https://api.segment.io/v1',
|
||||
'ANALYTICS_TOKEN': 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H=',
|
||||
'API_ADDRESS': 'lbryapi',
|
||||
'APP_NAME': APP_NAME,
|
||||
'BLOBFILES_DIR': 'blobfiles',
|
||||
'BLOB_SIZE': 2 * MB,
|
||||
'CRYPTSD_FILE_EXTENSION': '.cryptsd',
|
||||
'CURRENCIES': {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
},
|
||||
'DB_REVISION_FILE_NAME': 'db_revision',
|
||||
'ICON_PATH': ICON_PATH,
|
||||
'LOGGLY_TOKEN': 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv',
|
||||
'LOG_FILE_NAME': 'lbrynet.log',
|
||||
'LOG_POST_URL': 'https://lbry.io/log-upload',
|
||||
'MAX_BLOB_REQUEST_SIZE': 64 * KB,
|
||||
'MAX_HANDSHAKE_SIZE': 64 * KB,
|
||||
'MAX_REQUEST_SIZE': 64 * KB,
|
||||
'MAX_RESPONSE_INFO_SIZE': 64 * KB,
|
||||
'MAX_BLOB_INFOS_TO_REQUEST': 20,
|
||||
'PROTOCOL_PREFIX': PROTOCOL_PREFIX,
|
||||
'SLACK_WEBHOOK': ('nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='),
|
||||
'SOURCE_TYPES': ['lbry_sd_hash', 'url', 'btih'],
|
||||
'WALLET_TYPES': [LBRYUM_WALLET, LBRYCRD_WALLET],
|
||||
}
|
||||
|
||||
ADJUSTABLE_SETTINGS = {
|
||||
# By default, daemon will block all cross origin requests
|
||||
# but if this is set, this value will be used for the
|
||||
# Access-Control-Allow-Origin. For example
|
||||
# set to '*' to allow all requests, or set to 'http://localhost:8080'
|
||||
# if you're running a test UI on that port
|
||||
'allowed_origin': (str, ''),
|
||||
|
||||
# Changing this value is not-advised as it could potentially
|
||||
# expose the lbrynet daemon to the outside world which would
|
||||
# give an attacker access to your wallet and you could lose
|
||||
# all of your credits.
|
||||
'api_host': (str, 'localhost'),
|
||||
|
||||
'api_port': (int, 5279),
|
||||
'cache_time': (int, 150),
|
||||
'check_ui_requirements': (bool, True),
|
||||
'data_dir': (str, default_data_dir),
|
||||
'data_rate': (float, .0001), # points/megabyte
|
||||
'default_ui_branch': (str, 'master'),
|
||||
'delete_blobs_on_remove': (bool, True),
|
||||
'dht_node_port': (int, 4444),
|
||||
'download_directory': (str, default_download_directory),
|
||||
'download_timeout': (int, 180),
|
||||
'host_ui': (bool, True),
|
||||
'is_generous_host': (bool, True),
|
||||
'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_port),
|
||||
|
||||
# TODO: this should not be configured; move it elsewhere
|
||||
'last_version': (dict, {'lbrynet': '0.0.1', 'lbryum': '0.0.1'}),
|
||||
|
||||
'lbryum_wallet_dir': (str, default_lbryum_dir),
|
||||
'local_ui_path': (str, ''),
|
||||
'max_connections_per_stream': (int, 5),
|
||||
'max_download': (float, 0.0),
|
||||
|
||||
# TODO: this field is more complicated than it needs to be because
|
||||
# it goes through a Fee validator when loaded by the exchange rate
|
||||
# manager. Look into refactoring the exchange rate conversion to
|
||||
# take in a simpler form.
|
||||
#
|
||||
# TODO: writing json on the cmd line is a pain, come up with a nicer
|
||||
# parser for this data structure. (maybe MAX_KEY_FEE': USD:25
|
||||
'max_key_fee': (json.loads, {'currency':'USD', 'amount': 25.0, 'address':''}),
|
||||
|
||||
'max_search_results': (int, 25),
|
||||
'max_upload': (float, 0.0),
|
||||
'min_info_rate': (float, .02), # points/1000 infos
|
||||
'min_valuable_hash_rate': (float, .05), # points/1000 infos
|
||||
'min_valuable_info_rate': (float, .05), # points/1000 infos
|
||||
'peer_port': (int, 3333),
|
||||
'pointtrader_server': (str, 'http://127.0.0.1:2424'),
|
||||
'reflector_port': (int, 5566),
|
||||
'reflect_uploads': (bool, True),
|
||||
'reflector_servers': (list, [('reflector.lbry.io', 5566)], server_port),
|
||||
'run_on_startup': (bool, False),
|
||||
'run_reflector_server': (bool, False),
|
||||
'sd_download_timeout': (int, 3),
|
||||
'share_debug_info': (bool, True), # whether to share diagnostic info with LBRY
|
||||
'peer_search_timeout': (int, 3),
|
||||
'search_servers': (list, ['lighthouse1.lbry.io:50005']),
|
||||
'search_timeout': (float, 5.0),
|
||||
'startup_scripts': (list, []),
|
||||
'ui_branch': (str, 'master'),
|
||||
'use_auth_http': (bool, False),
|
||||
'use_upnp': (bool, True),
|
||||
'wallet': (str, LBRYUM_WALLET),
|
||||
}
|
||||
|
||||
|
||||
class Config(object):
|
||||
def __init__(self, fixed_defaults, adjustable_defaults, persisted_settings=None,
|
||||
environment=None, cli_settings=None):
|
||||
|
||||
self._installation_id = None
|
||||
self._session_id = base58.b58encode(utils.generate_id())
|
||||
|
||||
self._fixed_defaults = fixed_defaults
|
||||
self._adjustable_defaults = adjustable_defaults
|
||||
|
||||
self._data = {
|
||||
TYPE_DEFAULT: {}, # defaults
|
||||
TYPE_PERSISTED: {}, # stored settings from daemon_settings.yml (or from a db, etc)
|
||||
TYPE_ENV: {}, # settings from environment variables
|
||||
TYPE_CLI: {}, # command-line arguments
|
||||
TYPE_RUNTIME: {}, # set during runtime (using self.set(), etc)
|
||||
}
|
||||
|
||||
# the order in which a piece of data is searched for. earlier types override later types
|
||||
self._search_order = (
|
||||
TYPE_RUNTIME, TYPE_CLI, TYPE_ENV, TYPE_PERSISTED, TYPE_DEFAULT
|
||||
)
|
||||
|
||||
self._data[TYPE_DEFAULT].update(self._fixed_defaults)
|
||||
self._data[TYPE_DEFAULT].update(
|
||||
{k: v[1] for (k, v) in self._adjustable_defaults.iteritems()})
|
||||
|
||||
if persisted_settings is None:
|
||||
persisted_settings = {}
|
||||
self._validate_settings(persisted_settings)
|
||||
self._data[TYPE_PERSISTED].update(persisted_settings)
|
||||
|
||||
env_settings = self._parse_environment(environment)
|
||||
self._validate_settings(env_settings)
|
||||
self._data[TYPE_ENV].update(env_settings)
|
||||
|
||||
if cli_settings is None:
|
||||
cli_settings = {}
|
||||
self._validate_settings(cli_settings)
|
||||
self._data[TYPE_CLI].update(cli_settings)
|
||||
|
||||
def __repr__(self):
|
||||
return self.get_current_settings_dict().__repr__()
|
||||
|
||||
def __iter__(self):
|
||||
for k in self._data[TYPE_DEFAULT].iterkeys():
|
||||
yield k
|
||||
|
||||
def __getitem__(self, name):
|
||||
return self.get(name)
|
||||
|
||||
def __setitem__(self, name, value):
|
||||
return self.set(name, value)
|
||||
|
||||
def __contains__(self, name):
|
||||
return name in self._data[TYPE_DEFAULT]
|
||||
|
||||
@staticmethod
|
||||
def _parse_environment(environment):
|
||||
env_settings = {}
|
||||
if environment is not None:
|
||||
assert isinstance(environment, Env)
|
||||
for opt in environment.original_schema:
|
||||
if environment(opt) is not None:
|
||||
env_settings[opt] = environment(opt)
|
||||
return env_settings
|
||||
|
||||
def _assert_valid_data_type(self, data_type):
|
||||
assert data_type in self._data, KeyError('{} in is not a valid data type'.format(data_type))
|
||||
|
||||
def get_valid_setting_names(self):
|
||||
return self._data[TYPE_DEFAULT].keys()
|
||||
|
||||
def _is_valid_setting(self, name):
|
||||
return name in self.get_valid_setting_names()
|
||||
|
||||
def _assert_valid_setting(self, name):
|
||||
assert self._is_valid_setting(name), \
|
||||
KeyError('{} is not a valid setting'.format(name))
|
||||
|
||||
def _validate_settings(self, data):
|
||||
invalid_settings = set(data.keys()) - set(self.get_valid_setting_names())
|
||||
if len(invalid_settings) > 0:
|
||||
raise KeyError('invalid settings: {}'.format(', '.join(invalid_settings)))
|
||||
|
||||
def _assert_editable_setting(self, name):
|
||||
self._assert_valid_setting(name)
|
||||
assert name not in self._fixed_defaults, \
|
||||
ValueError('{} is not an editable setting'.format(name))
|
||||
|
||||
def get(self, name, data_type=None):
|
||||
"""Get a config value
|
||||
|
||||
Args:
|
||||
name: the name of the value to get
|
||||
data_type: if given, get the value from a specific data set (see below)
|
||||
|
||||
Returns: the config value for the given name
|
||||
|
||||
If data_type is None, get() will search for the given name in each data set, in
|
||||
order of precedence. It will return the first value it finds. This is the "effective"
|
||||
value of a config name. For example, ENV values take precedence over DEFAULT values,
|
||||
so if a value is present in ENV and in DEFAULT, the ENV value will be returned
|
||||
"""
|
||||
self._assert_valid_setting(name)
|
||||
if data_type is not None:
|
||||
self._assert_valid_data_type(data_type)
|
||||
return self._data[data_type][name]
|
||||
for data_type in self._search_order:
|
||||
if name in self._data[data_type]:
|
||||
return self._data[data_type][name]
|
||||
raise KeyError('{} is not a valid setting'.format(name))
|
||||
|
||||
def set(self, name, value, data_types=(TYPE_RUNTIME,)):
|
||||
"""Set a config value
|
||||
|
||||
Args:
|
||||
name: the name of the value to set
|
||||
value: the value
|
||||
data_types: what type(s) of data this is
|
||||
|
||||
Returns: None
|
||||
|
||||
By default, this sets the RUNTIME value of a config. If you wish to set other
|
||||
data types (e.g. PERSISTED values to save to a file, CLI values from parsed
|
||||
command-line options, etc), you can specify that with the data_types param
|
||||
"""
|
||||
self._assert_editable_setting(name)
|
||||
for data_type in data_types:
|
||||
self._assert_valid_data_type(data_type)
|
||||
self._data[data_type][name] = value
|
||||
|
||||
def update(self, updated_settings, data_types=(TYPE_RUNTIME,)):
|
||||
for k, v in updated_settings.iteritems():
|
||||
try:
|
||||
self.set(k, v, data_types=data_types)
|
||||
except (KeyError, AssertionError):
|
||||
pass
|
||||
|
||||
def get_current_settings_dict(self):
|
||||
current_settings = {}
|
||||
for key in self.get_valid_setting_names():
|
||||
current_settings[key] = self.get(key)
|
||||
return current_settings
|
||||
|
||||
def get_adjustable_settings_dict(self):
|
||||
return {
|
||||
key: val for key, val in self.get_current_settings_dict().iteritems()
|
||||
if key in self._adjustable_defaults
|
||||
}
|
||||
|
||||
def save_conf_file_settings(self):
|
||||
path = self.get_conf_filename()
|
||||
ext = os.path.splitext(path)[1]
|
||||
encoder = settings_encoders.get(ext, False)
|
||||
assert encoder is not False, 'Unknown settings format %s' % ext
|
||||
with open(path, 'w') as settings_file:
|
||||
settings_file.write(encoder(self._data[TYPE_PERSISTED]))
|
||||
|
||||
def load_conf_file_settings(self):
|
||||
path = self.get_conf_filename()
|
||||
ext = os.path.splitext(path)[1]
|
||||
decoder = settings_decoders.get(ext, False)
|
||||
assert decoder is not False, 'Unknown settings format %s' % ext
|
||||
try:
|
||||
with open(path, 'r') as settings_file:
|
||||
data = settings_file.read()
|
||||
decoded = self._fix_old_conf_file_settings(decoder(data))
|
||||
log.info('Loaded settings file: %s', path)
|
||||
self._validate_settings(decoded)
|
||||
self._data[TYPE_PERSISTED].update(decoded)
|
||||
except (IOError, OSError) as err:
|
||||
log.info('%s: Failed to update settings from %s', err, path)
|
||||
|
||||
def _fix_old_conf_file_settings(self, settings_dict):
|
||||
if 'API_INTERFACE' in settings_dict:
|
||||
settings_dict['api_host'] = settings_dict['API_INTERFACE']
|
||||
del settings_dict['API_INTERFACE']
|
||||
if 'startup_scripts' in settings_dict:
|
||||
del settings_dict['startup_scripts']
|
||||
if 'upload_log' in settings_dict:
|
||||
settings_dict['share_debug_info'] = settings_dict['upload_log']
|
||||
del settings_dict['upload_log']
|
||||
for key in settings_dict.keys():
|
||||
if not self._is_valid_setting(key):
|
||||
log.warning('Ignoring invalid conf file setting: %s', key)
|
||||
del settings_dict[key]
|
||||
return settings_dict
|
||||
|
||||
def ensure_data_dir(self):
|
||||
# although there is a risk of a race condition here we don't
|
||||
# expect there to be multiple processes accessing this
|
||||
# directory so the risk can be ignored
|
||||
if not os.path.isdir(self['data_dir']):
|
||||
os.makedirs(self['data_dir'])
|
||||
return self['data_dir']
|
||||
|
||||
def get_log_filename(self):
|
||||
"""
|
||||
Return the log file for this platform.
|
||||
Also ensure the containing directory exists.
|
||||
"""
|
||||
return os.path.join(self.ensure_data_dir(), self['LOG_FILE_NAME'])
|
||||
|
||||
def get_api_connection_string(self):
|
||||
return 'http://%s:%i/%s' % (self['api_host'], self['api_port'], self['API_ADDRESS'])
|
||||
|
||||
def get_ui_address(self):
|
||||
return 'http://%s:%i' % (self['api_host'], self['api_port'])
|
||||
|
||||
def get_db_revision_filename(self):
|
||||
return os.path.join(self.ensure_data_dir(), self['DB_REVISION_FILE_NAME'])
|
||||
|
||||
def get_conf_filename(self):
|
||||
data_dir = self.ensure_data_dir()
|
||||
yml_path = os.path.join(data_dir, 'daemon_settings.yml')
|
||||
json_path = os.path.join(data_dir, 'daemon_settings.json')
|
||||
if os.path.isfile(yml_path):
|
||||
return yml_path
|
||||
elif os.path.isfile(json_path):
|
||||
return json_path
|
||||
else:
|
||||
return yml_path
|
||||
|
||||
def get_installation_id(self):
|
||||
install_id_filename = os.path.join(self.ensure_data_dir(), "install_id")
|
||||
if not self._installation_id:
|
||||
if os.path.isfile(install_id_filename):
|
||||
with open(install_id_filename, "r") as install_id_file:
|
||||
self._installation_id = install_id_file.read()
|
||||
if not self._installation_id:
|
||||
self._installation_id = base58.b58encode(utils.generate_id())
|
||||
with open(install_id_filename, "w") as install_id_file:
|
||||
install_id_file.write(self._installation_id)
|
||||
return self._installation_id
|
||||
|
||||
def get_session_id(self):
|
||||
return self._session_id
|
||||
|
||||
|
||||
# type: Config
|
||||
settings = None
|
||||
|
||||
|
||||
def get_default_env():
|
||||
env_defaults = {}
|
||||
for k, v in ADJUSTABLE_SETTINGS.iteritems():
|
||||
if len(v) == 3:
|
||||
env_defaults[k] = (v[0], None, v[2])
|
||||
else:
|
||||
env_defaults[k] = (v[0], None)
|
||||
return Env(**env_defaults)
|
||||
|
||||
|
||||
def initialize_settings(load_conf_file=True):
|
||||
global settings
|
||||
if settings is None:
|
||||
settings = Config(FIXED_SETTINGS, ADJUSTABLE_SETTINGS,
|
||||
environment=get_default_env())
|
||||
settings.installation_id = settings.get_installation_id()
|
||||
if load_conf_file:
|
||||
settings.load_conf_file_settings()
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
from decimal import Decimal
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityTracker(object):
|
||||
"""
|
||||
Class to track peer counts for known blobs, and to discover new popular blobs
|
||||
|
||||
Attributes:
|
||||
availability (dict): dictionary of peers for known blobs
|
||||
"""
|
||||
|
||||
def __init__(self, blob_manager, peer_finder, dht_node):
|
||||
self.availability = {}
|
||||
self._last_mean_availability = Decimal(0.0)
|
||||
self._blob_manager = blob_manager
|
||||
self._peer_finder = peer_finder
|
||||
self._dht_node = dht_node
|
||||
self._check_popular = LoopingCall(self._update_most_popular)
|
||||
self._check_mine = LoopingCall(self._update_mine)
|
||||
|
||||
def start(self):
|
||||
log.info("Starting %s", self)
|
||||
self._check_popular.start(30)
|
||||
self._check_mine.start(600)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self._check_popular.running:
|
||||
self._check_popular.stop()
|
||||
if self._check_mine.running:
|
||||
self._check_mine.stop()
|
||||
|
||||
def get_blob_availability(self, blob, timeout=None):
|
||||
def _get_peer_count(peers):
|
||||
have_blob = sum(1 for peer in peers if peer.is_available())
|
||||
return {blob: have_blob}
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob, timeout)
|
||||
d.addCallback(_get_peer_count)
|
||||
return d
|
||||
|
||||
def get_availability_for_blobs(self, blobs, timeout=None):
|
||||
dl = [self.get_blob_availability(blob, timeout) for blob in blobs if blob]
|
||||
d = defer.DeferredList(dl)
|
||||
d.addCallback(lambda results: [val for success, val in results if success])
|
||||
return d
|
||||
|
||||
@property
|
||||
def last_mean_availability(self):
|
||||
return max(Decimal(0.01), self._last_mean_availability)
|
||||
|
||||
def _update_peers_for_blob(self, blob):
|
||||
def _save_peer_info(blob_hash, peers):
|
||||
v = {blob_hash: peers}
|
||||
self.availability.update(v)
|
||||
return v
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob)
|
||||
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
|
||||
d.addCallback(lambda peers: _save_peer_info(blob, peers))
|
||||
return d
|
||||
|
||||
def _get_most_popular(self):
|
||||
dl = []
|
||||
for (hash, _) in self._dht_node.get_most_popular_hashes(100):
|
||||
encoded = hash.encode('hex')
|
||||
dl.append(self._update_peers_for_blob(encoded))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
def _update_most_popular(self):
|
||||
d = self._get_most_popular()
|
||||
d.addCallback(lambda _: self._set_mean_peers())
|
||||
|
||||
def _update_mine(self):
|
||||
def _get_peers(blobs):
|
||||
dl = []
|
||||
for hash in blobs:
|
||||
dl.append(self._update_peers_for_blob(hash))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
def sample(blobs):
|
||||
return random.sample(blobs, min(len(blobs), 100))
|
||||
|
||||
start = time.time()
|
||||
log.debug('==> Updating the peers for my blobs')
|
||||
d = self._blob_manager.get_all_verified_blobs()
|
||||
# as far as I can tell, this only is used to set _last_mean_availability
|
||||
# which... seems like a very expensive operation for such little payoff.
|
||||
# so taking a sample should get about the same effect as querying the entire
|
||||
# list of blobs
|
||||
d.addCallback(sample)
|
||||
d.addCallback(_get_peers)
|
||||
d.addCallback(lambda _: self._set_mean_peers())
|
||||
d.addCallback(lambda _: log.debug('<== Done updating peers for my blobs. Took %s seconds',
|
||||
time.time() - start))
|
||||
# although unused, need to return or else the looping call
|
||||
# could overrun on a previous call
|
||||
return d
|
||||
|
||||
def _set_mean_peers(self):
|
||||
num_peers = [len(self.availability[blob]) for blob in self.availability]
|
||||
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
|
||||
self._last_mean_availability = mean
|
||||
@@ -15,4 +15,4 @@ class BlobInfo(object):
|
||||
def __init__(self, blob_hash, blob_num, length):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.length = length
|
||||
+216
-181
@@ -1,14 +1,18 @@
|
||||
import logging
|
||||
import os
|
||||
import leveldb
|
||||
import time
|
||||
import json
|
||||
from twisted.internet import threads, defer, reactor, task
|
||||
import sqlite3
|
||||
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.HashBlob import BlobFile, TempBlob, BlobFileCreator, TempBlobCreator
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.Error import NoSuchBlobError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager(DHTHashSupplier):
|
||||
@@ -20,19 +24,19 @@ class BlobManager(DHTHashSupplier):
|
||||
def setup(self):
|
||||
pass
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length):
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
pass
|
||||
|
||||
def get_blob_creator(self):
|
||||
pass
|
||||
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length):
|
||||
def _make_new_blob(self, blob_hash, length):
|
||||
pass
|
||||
|
||||
def blob_completed(self, blob, next_announce_time=None):
|
||||
pass
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
pass
|
||||
|
||||
def hashes_to_announce(self):
|
||||
@@ -44,12 +48,6 @@ class BlobManager(DHTHashSupplier):
|
||||
def delete_blob(self, blob_hash):
|
||||
pass
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
pass
|
||||
|
||||
def check_consistency(self):
|
||||
pass
|
||||
|
||||
def blob_requested(self, blob_hash):
|
||||
pass
|
||||
|
||||
@@ -62,91 +60,94 @@ class BlobManager(DHTHashSupplier):
|
||||
def blob_paid_for(self, blob_hash, amount):
|
||||
pass
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
pass
|
||||
|
||||
def add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
pass
|
||||
|
||||
def add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
pass
|
||||
|
||||
def _immediate_announce(self, blob_hashes):
|
||||
if self.hash_announcer:
|
||||
return self.hash_announcer.immediate_announce(blob_hashes)
|
||||
|
||||
|
||||
# TODO: Having different managers for different blobs breaks the
|
||||
# abstraction of a HashBlob. Why should the management of blobs
|
||||
# care what kind of Blob it has?
|
||||
class DiskBlobManager(BlobManager):
|
||||
"""This class stores blobs on the hard disk"""
|
||||
def __init__(self, hash_announcer, blob_dir, db_dir):
|
||||
BlobManager.__init__(self, hash_announcer)
|
||||
self.blob_dir = blob_dir
|
||||
self.db_dir = db_dir
|
||||
self.db = None
|
||||
self.db_file = os.path.join(db_dir, "blobs.db")
|
||||
self.db_conn = None
|
||||
self.blob_type = BlobFile
|
||||
self.blob_creator_type = BlobFileCreator
|
||||
# TODO: consider using an LRU for blobs as there could potentially
|
||||
# be thousands of blobs loaded up, many stale
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
self._next_manage_call = None
|
||||
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
log.info("Setting up the DiskBlobManager. blob_dir: %s, db_file: %s", str(self.blob_dir),
|
||||
str(self.db_file))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping the DiskBlobManager")
|
||||
if self._next_manage_call is not None and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
self.db = None
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a blob that is already on the hard disk"""
|
||||
# TODO: if blob.upload_allowed and upload_allowed is False, change upload_allowed in blob and on disk
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a
|
||||
blob that is already on the hard disk
|
||||
"""
|
||||
assert length is None or isinstance(length, int)
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self._make_new_blob(blob_hash, upload_allowed, length)
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
return self.blob_creator_type(self, self.blob_dir)
|
||||
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
|
||||
blob = self.blob_type(self.blob_dir, blob_hash, upload_allowed, length)
|
||||
def _make_new_blob(self, blob_hash, length=None):
|
||||
log.debug('Making a new blob for %s', blob_hash)
|
||||
blob = self.blob_type(self.blob_dir, blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
d = threads.deferToThread(self._completed_blobs, [blob_hash])
|
||||
|
||||
def check_completed(completed_blobs):
|
||||
|
||||
def set_length(length):
|
||||
blob.length = length
|
||||
|
||||
if len(completed_blobs) == 1 and completed_blobs[0] == blob_hash:
|
||||
blob.verified = True
|
||||
inner_d = threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
inner_d.addCallback(set_length)
|
||||
inner_d.addCallback(lambda _: blob)
|
||||
else:
|
||||
inner_d = defer.succeed(blob)
|
||||
return inner_d
|
||||
|
||||
d.addCallback(check_completed)
|
||||
return d
|
||||
return defer.succeed(blob)
|
||||
|
||||
def blob_completed(self, blob, next_announce_time=None):
|
||||
if next_announce_time is None:
|
||||
next_announce_time = time.time()
|
||||
return threads.deferToThread(self._add_completed_blob, blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
next_announce_time = self.get_next_announce_time()
|
||||
d = self._add_completed_blob(blob.blob_hash, blob.length, next_announce_time)
|
||||
d.addCallback(lambda _: self._immediate_announce([blob.blob_hash]))
|
||||
return d
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
return threads.deferToThread(self._completed_blobs, blobs_to_check)
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
return self._completed_blobs(blobhashes_to_check)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_blobs_to_announce, next_announce_time)
|
||||
return self._get_blobs_to_announce()
|
||||
|
||||
def creator_finished(self, blob_creator):
|
||||
logging.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
assert blob_creator.blob_hash is not None
|
||||
assert blob_creator.blob_hash not in self.blobs
|
||||
assert blob_creator.length is not None
|
||||
new_blob = self.blob_type(self.blob_dir, blob_creator.blob_hash, True, blob_creator.length)
|
||||
new_blob.verified = True
|
||||
new_blob = self.blob_type(self.blob_dir, blob_creator.blob_hash, blob_creator.length)
|
||||
self.blobs[blob_creator.blob_hash] = new_blob
|
||||
if self.hash_announcer is not None:
|
||||
self.hash_announcer.immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
else:
|
||||
d = self.blob_completed(new_blob)
|
||||
self._immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = self.get_next_announce_time()
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
return d
|
||||
|
||||
def delete_blobs(self, blob_hashes):
|
||||
@@ -154,19 +155,23 @@ class DiskBlobManager(BlobManager):
|
||||
if not blob_hash in self.blob_hashes_to_delete:
|
||||
self.blob_hashes_to_delete[blob_hash] = False
|
||||
|
||||
def update_all_last_verified_dates(self, timestamp):
|
||||
return threads.deferToThread(self._update_all_last_verified_dates, timestamp)
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
d = threads.deferToThread(self._get_all_verified_blob_hashes)
|
||||
d.addCallback(self.hash_announcer.immediate_announce)
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self._immediate_announce)
|
||||
return d
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
return threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
def check_consistency(self):
|
||||
return threads.deferToThread(self._check_consistency)
|
||||
def add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
d = self._add_blob_to_download_history(blob_hash, host, rate)
|
||||
return d
|
||||
|
||||
def add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
d = self._add_blob_to_upload_history(blob_hash, host, rate)
|
||||
return d
|
||||
|
||||
def _manage(self):
|
||||
from twisted.internet import reactor
|
||||
@@ -185,19 +190,20 @@ class DiskBlobManager(BlobManager):
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return err
|
||||
|
||||
def delete_from_db(result):
|
||||
b_hs = [r[1] for r in result if r[0] is True]
|
||||
if b_hs:
|
||||
d = threads.deferToThread(self._delete_blobs_from_db, b_hs)
|
||||
d = self._delete_blobs_from_db(b_hs)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
log.warning(
|
||||
"Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
@@ -211,8 +217,10 @@ class DiskBlobManager(BlobManager):
|
||||
for blob_hash, being_deleted in self.blob_hashes_to_delete.items():
|
||||
if being_deleted is False:
|
||||
self.blob_hashes_to_delete[blob_hash] = True
|
||||
d = self.get_blob(blob_hash, True)
|
||||
d.addCallbacks(delete, set_not_deleting, callbackArgs=(blob_hash,), errbackArgs=(blob_hash,))
|
||||
d = self.get_blob(blob_hash)
|
||||
d.addCallbacks(
|
||||
delete, set_not_deleting,
|
||||
callbackArgs=(blob_hash,), errbackArgs=(blob_hash,))
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
dl.addCallback(delete_from_db)
|
||||
@@ -221,96 +229,120 @@ class DiskBlobManager(BlobManager):
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "blobs.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', self.db_file, check_same_thread=False)
|
||||
|
||||
def _add_completed_blob(self, blob_hash, length, timestamp, next_announce_time=None):
|
||||
logging.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
if next_announce_time is None:
|
||||
next_announce_time = timestamp
|
||||
self.db.Put(blob_hash, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" last_verified_time real, " +
|
||||
" next_announce_time real)")
|
||||
|
||||
def _completed_blobs(self, blobs_to_check):
|
||||
blobs = []
|
||||
for b in blobs_to_check:
|
||||
if is_valid_blobhash(b):
|
||||
try:
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(b))
|
||||
except KeyError:
|
||||
continue
|
||||
file_path = os.path.join(self.blob_dir, b)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blobs.append(b)
|
||||
return blobs
|
||||
transaction.execute("create table if not exists download (" +
|
||||
" id integer primary key autoincrement, " +
|
||||
" blob text, " +
|
||||
" host text, " +
|
||||
" rate float, " +
|
||||
" ts integer)")
|
||||
|
||||
def _get_blob_length(self, blob):
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
return length
|
||||
transaction.execute("create table if not exists upload (" +
|
||||
" id integer primary key autoincrement, " +
|
||||
" blob text, " +
|
||||
" host text, " +
|
||||
" rate float, " +
|
||||
" ts integer)")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_completed_blob(self, blob_hash, length, next_announce_time):
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
d = self.db_conn.runQuery(
|
||||
"insert into blobs (blob_hash, blob_length, next_announce_time) values (?, ?, ?)",
|
||||
(blob_hash, length, next_announce_time)
|
||||
)
|
||||
d.addErrback(lambda err: err.trap(sqlite3.IntegrityError))
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _completed_blobs(self, blobhashes_to_check):
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
|
||||
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
|
||||
defer.returnValue(blob_hashes)
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_blob_verified_timestamp(self, blob, timestamp):
|
||||
length, old_verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
self.db.Put(blob, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(blob, timestamp))
|
||||
|
||||
def _get_blobs_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
blobs = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(blob_hash, json.dumps((length, verified_time, next_announce_time)))
|
||||
blobs.append(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
return blobs
|
||||
@rerun_if_locked
|
||||
def _get_blobs_to_announce(self):
|
||||
|
||||
def _update_all_last_verified_dates(self, timestamp):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
batch.Put(blob_hash, json.dumps((length, timestamp, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select blob_hash from blobs " +
|
||||
"where next_announce_time < ? and blob_hash is not null",
|
||||
(timestamp,))
|
||||
blobs = [b for b, in r.fetchall()]
|
||||
next_announce_time = self.get_next_announce_time(len(blobs))
|
||||
transaction.execute(
|
||||
"update blobs set next_announce_time = ? where next_announce_time < ?",
|
||||
(next_announce_time, timestamp))
|
||||
log.debug("Got %s blobs to announce, next announce time is in %s seconds",
|
||||
len(blobs), next_announce_time-time.time())
|
||||
return blobs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_blobs_from_db(self, blob_hashes):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash in blob_hashes:
|
||||
batch.Delete(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
|
||||
def _check_consistency(self):
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time < os.path.getctime(file_path):
|
||||
h = get_lbry_hash_obj()
|
||||
len_so_far = 0
|
||||
f = open(file_path)
|
||||
while True:
|
||||
data = f.read(2**12)
|
||||
if not data:
|
||||
break
|
||||
h.update(data)
|
||||
len_so_far += len(data)
|
||||
if len_so_far == length and h.hexdigest() == blob_hash:
|
||||
batch.Put(blob_hash, json.dumps((length, current_time, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
def delete_blobs(transaction):
|
||||
for b in blob_hashes:
|
||||
transaction.execute("delete from blobs where blob_hash = ?", (b,))
|
||||
|
||||
return self.db_conn.runInteraction(delete_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
blob_hashes = []
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blob_hashes.append(blob_hash)
|
||||
return blob_hashes
|
||||
d = self.db_conn.runQuery("select blob_hash from blobs")
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash, in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
ts = int(time.time())
|
||||
d = self.db_conn.runQuery(
|
||||
"insert into download values (null, ?, ?, ?, ?) ",
|
||||
(blob_hash, str(host), float(rate), ts))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
ts = int(time.time())
|
||||
d = self.db_conn.runQuery(
|
||||
"insert into upload values (null, ?, ?, ?, ?) ",
|
||||
(blob_hash, str(host), float(rate), ts))
|
||||
return d
|
||||
|
||||
|
||||
# TODO: Having different managers for different blobs breaks the
|
||||
# abstraction of a HashBlob. Why should the management of blobs
|
||||
# care what kind of Blob it has?
|
||||
class TempBlobManager(BlobManager):
|
||||
"""This class stores blobs in memory"""
|
||||
def __init__(self, hash_announcer):
|
||||
@@ -331,16 +363,16 @@ class TempBlobManager(BlobManager):
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length=None):
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self._make_new_blob(blob_hash, upload_allowed, length)
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
return self.blob_creator_type(self)
|
||||
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
|
||||
blob = self.blob_type(blob_hash, upload_allowed, length)
|
||||
def _make_new_blob(self, blob_hash, length=None):
|
||||
blob = self.blob_type(blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
return defer.succeed(blob)
|
||||
|
||||
@@ -350,14 +382,24 @@ class TempBlobManager(BlobManager):
|
||||
self.blob_next_announces[blob.blob_hash] = next_announce_time
|
||||
return defer.succeed(True)
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
blobs = [b.blob_hash for b in self.blobs.itervalues() if b.blob_hash in blobs_to_check and b.is_validated()]
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
blobs = [
|
||||
b.blob_hash for b in self.blobs.itervalues()
|
||||
if b.blob_hash in blobhashes_to_check and b.is_validated()
|
||||
]
|
||||
return defer.succeed(blobs)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self.completed_blobs(self.blobs)
|
||||
return d
|
||||
|
||||
def hashes_to_announce(self):
|
||||
now = time.time()
|
||||
blobs = [blob_hash for blob_hash, announce_time in self.blob_next_announces.iteritems() if announce_time < now]
|
||||
next_announce_time = now + self.hash_reannounce_time
|
||||
blobs = [
|
||||
blob_hash for blob_hash, announce_time in self.blob_next_announces.iteritems()
|
||||
if announce_time < now
|
||||
]
|
||||
next_announce_time = self.get_next_announce_time(len(blobs))
|
||||
for b in blobs:
|
||||
self.blob_next_announces[b] = next_announce_time
|
||||
return defer.succeed(blobs)
|
||||
@@ -366,17 +408,16 @@ class TempBlobManager(BlobManager):
|
||||
assert blob_creator.blob_hash is not None
|
||||
assert blob_creator.blob_hash not in self.blobs
|
||||
assert blob_creator.length is not None
|
||||
new_blob = self.blob_type(blob_creator.blob_hash, True, blob_creator.length)
|
||||
new_blob.verified = True
|
||||
new_blob = self.blob_type(blob_creator.blob_hash, blob_creator.length)
|
||||
# TODO: change this; its breaks the encapsulation of the
|
||||
# blob. Maybe better would be to have the blob_creator
|
||||
# produce a blob.
|
||||
new_blob.data_buffer = blob_creator.data_buffer
|
||||
new_blob.length = blob_creator.length
|
||||
new_blob._verified = True
|
||||
self.blobs[blob_creator.blob_hash] = new_blob
|
||||
if self.hash_announcer is not None:
|
||||
self.hash_announcer.immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
else:
|
||||
d = self.blob_completed(new_blob)
|
||||
self._immediate_announce([blob_creator.blob_hash])
|
||||
next_announce_time = self.get_next_announce_time()
|
||||
d = self.blob_completed(new_blob, next_announce_time)
|
||||
d.addCallback(lambda _: new_blob)
|
||||
return d
|
||||
|
||||
@@ -385,14 +426,9 @@ class TempBlobManager(BlobManager):
|
||||
if not blob_hash in self.blob_hashes_to_delete:
|
||||
self.blob_hashes_to_delete[blob_hash] = False
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
if blob_hash in self.blobs:
|
||||
if self.blobs[blob_hash].length is not None:
|
||||
return defer.succeed(self.blobs[blob_hash].length)
|
||||
return defer.fail(ValueError("No such blob hash is known"))
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
|
||||
if self.hash_announcer:
|
||||
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
|
||||
|
||||
def _manage(self):
|
||||
from twisted.internet import reactor
|
||||
@@ -400,20 +436,19 @@ class TempBlobManager(BlobManager):
|
||||
d = self._delete_blobs_marked_for_deletion()
|
||||
|
||||
def set_next_manage_call():
|
||||
logging.info("Setting the next manage call in %s", str(self))
|
||||
log.info("Setting the next manage call in %s", str(self))
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def _delete_blobs_marked_for_deletion(self):
|
||||
|
||||
def remove_from_list(b_h):
|
||||
del self.blob_hashes_to_delete[b_h]
|
||||
logging.info("Deleted blob %s", blob_hash)
|
||||
log.info("Deleted blob %s", blob_hash)
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return b_h
|
||||
|
||||
@@ -422,7 +457,7 @@ class TempBlobManager(BlobManager):
|
||||
if being_deleted is False:
|
||||
if blob_hash in self.blobs:
|
||||
self.blob_hashes_to_delete[blob_hash] = True
|
||||
logging.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
log.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
blob = self.blobs[blob_hash]
|
||||
d = blob.delete()
|
||||
|
||||
@@ -432,7 +467,7 @@ class TempBlobManager(BlobManager):
|
||||
ds.append(d)
|
||||
else:
|
||||
remove_from_list(blob_hash)
|
||||
d = defer.fail(Failure(ValueError("No such blob known")))
|
||||
logging.warning("Blob %s cannot be deleted because it is unknown")
|
||||
d = defer.fail(Failure(NoSuchBlobError(blob_hash)))
|
||||
log.warning("Blob %s cannot be deleted because it is unknown")
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds)
|
||||
return defer.DeferredList(ds)
|
||||
@@ -1,6 +1,21 @@
|
||||
class DownloadOptionChoice(object):
|
||||
"""A possible choice that can be picked for some option.
|
||||
|
||||
An option can have one or more choices that can be picked from.
|
||||
"""
|
||||
def __init__(self, value, short_description, long_description, bool_options_description=None):
|
||||
self.value = value
|
||||
self.short_description = short_description
|
||||
self.long_description = long_description
|
||||
self.bool_options_description = bool_options_description
|
||||
|
||||
|
||||
class DownloadOption(object):
|
||||
def __init__(self, option_types, long_description, short_description, default):
|
||||
"""An option for a user to select a value from several different choices."""
|
||||
def __init__(self, option_types, long_description, short_description, default_value,
|
||||
default_value_description):
|
||||
self.option_types = option_types
|
||||
self.long_description = long_description
|
||||
self.short_description = short_description
|
||||
self.default = default
|
||||
self.default_value = default_value
|
||||
self.default_value_description = default_value_description
|
||||
+67
-2
@@ -22,14 +22,49 @@ class ConnectionClosedBeforeResponseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowed(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidExchangeRateResponse(Exception):
|
||||
def __init__(self, source, reason):
|
||||
Exception.__init__(self, 'Failed to get exchange rate from {}:{}'.format(source, reason))
|
||||
self.source = source
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class UnknownNameError(Exception):
|
||||
def __init__(self, name):
|
||||
Exception.__init__(self, 'Name {} is unknown'.format(name))
|
||||
self.name = name
|
||||
|
||||
|
||||
class InvalidName(Exception):
|
||||
def __init__(self, name, invalid_characters):
|
||||
self.name = name
|
||||
self.invalid_characters = invalid_characters
|
||||
Exception.__init__(
|
||||
self, 'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
|
||||
|
||||
|
||||
class UnknownStreamTypeError(Exception):
|
||||
def __init__(self, stream_type):
|
||||
self.stream_type = stream_type
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.stream_type)
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidStreamInfoError(Exception):
|
||||
def __init__(self, name):
|
||||
def __init__(self, name, stream_info):
|
||||
msg = '{} has claim with invalid stream info: {}'.format(name, stream_info)
|
||||
Exception.__init__(self, msg)
|
||||
self.name = name
|
||||
self.stream_info = stream_info
|
||||
|
||||
|
||||
class MisbehavingPeerError(Exception):
|
||||
@@ -45,4 +80,34 @@ class NoResponseError(MisbehavingPeerError):
|
||||
|
||||
|
||||
class InvalidResponseError(MisbehavingPeerError):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchBlobError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchStreamHash(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchSDHash(Exception):
|
||||
"""
|
||||
Raised if sd hash is not known
|
||||
"""
|
||||
|
||||
|
||||
class InvalidBlobHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidHeaderError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidAuthenticationToken(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NegotiationError(Exception):
|
||||
pass
|
||||
@@ -11,5 +11,8 @@ class DummyHashAnnouncer(object):
|
||||
def add_supplier(self, *args):
|
||||
pass
|
||||
|
||||
def hash_queue_size(self):
|
||||
return 0
|
||||
|
||||
def immediate_announce(self, *args):
|
||||
pass
|
||||
pass
|
||||
+69
-45
@@ -8,9 +8,13 @@ from twisted.internet import interfaces, defer, threads
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
@@ -45,17 +49,26 @@ class HashBlobWriter(object):
|
||||
self.write_handle = write_handle
|
||||
self.length_getter = length_getter
|
||||
self.finished_cb = finished_cb
|
||||
self.hashsum = get_lbry_hash_obj()
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
|
||||
@property
|
||||
def blob_hash(self):
|
||||
return self._hashsum.hexdigest()
|
||||
|
||||
def write(self, data):
|
||||
self.hashsum.update(data)
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > self.length_getter():
|
||||
self.finished_cb(self, Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (str(self.len_so_far),
|
||||
str(self.length_getter())))))
|
||||
self.finished_cb(
|
||||
self,
|
||||
Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (self.len_so_far,
|
||||
self.length_getter()))))
|
||||
else:
|
||||
if self.write_handle is None:
|
||||
log.debug("Tried to write to a write_handle that was None.")
|
||||
return
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb(self)
|
||||
@@ -69,32 +82,35 @@ class HashBlobWriter(object):
|
||||
class HashBlob(object):
|
||||
"""A chunk of data available on the network which is specified by a hashsum"""
|
||||
|
||||
def __init__(self, blob_hash, upload_allowed, length=None):
|
||||
def __init__(self, blob_hash, length=None):
|
||||
assert is_valid_blobhash(blob_hash)
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.writers = {} # {Peer: writer, finished_deferred}
|
||||
self.finished_deferred = None
|
||||
self.verified = False
|
||||
self.upload_allowed = upload_allowed
|
||||
self._verified = False
|
||||
self.readers = 0
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
# protect verified from being modified by other classes
|
||||
return self._verified
|
||||
|
||||
def set_length(self, length):
|
||||
if self.length is not None and length == self.length:
|
||||
return True
|
||||
if self.length is None and 0 <= length <= BLOB_SIZE:
|
||||
if self.length is None and 0 <= length <= conf.settings['BLOB_SIZE']:
|
||||
self.length = length
|
||||
return True
|
||||
logging.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
|
||||
self.length, length)
|
||||
return False
|
||||
|
||||
def get_length(self):
|
||||
return self.length
|
||||
|
||||
def is_validated(self):
|
||||
if self.verified:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
return bool(self._verified)
|
||||
|
||||
def is_downloading(self):
|
||||
if self.writers:
|
||||
@@ -120,14 +136,15 @@ class HashBlob(object):
|
||||
def writer_finished(self, writer, err=None):
|
||||
|
||||
def fire_finished_deferred():
|
||||
self.verified = True
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
if w == writer:
|
||||
finished_deferred.callback(self)
|
||||
del self.writers[p]
|
||||
return True
|
||||
logging.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
log.warning(
|
||||
"Somehow, the writer that was accepted as being valid was already removed: %s",
|
||||
writer)
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
@@ -141,8 +158,8 @@ class HashBlob(object):
|
||||
w.cancel()
|
||||
|
||||
if err is None:
|
||||
if writer.len_so_far == self.length and writer.hashsum.hexdigest() == self.blob_hash:
|
||||
if self.verified is False:
|
||||
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
|
||||
if self._verified is False:
|
||||
d = self._save_verified_blob(writer)
|
||||
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
|
||||
d.addCallback(lambda _: cancel_other_downloads())
|
||||
@@ -152,7 +169,7 @@ class HashBlob(object):
|
||||
else:
|
||||
err_string = "length vs expected: {0}, {1}, hash vs expected: {2}, {3}"
|
||||
err_string = err_string.format(self.length, writer.len_so_far, self.blob_hash,
|
||||
writer.hashsum.hexdigest())
|
||||
writer.blob_hash)
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(True)
|
||||
else:
|
||||
@@ -184,7 +201,7 @@ class HashBlob(object):
|
||||
return self.blob_hash[:16]
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
return '<{}({})>'.format(self.__class__.__name__, str(self))
|
||||
|
||||
|
||||
class BlobFile(HashBlob):
|
||||
@@ -196,33 +213,42 @@ class BlobFile(HashBlob):
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.setting_verified_blob_lock = threading.Lock()
|
||||
self.moved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
self.set_length(os.path.getsize(self.file_path))
|
||||
# This assumes that the hash of the blob has already been
|
||||
# checked as part of the blob creation process. It might
|
||||
# be worth having a function that checks the actual hash;
|
||||
# its probably too expensive to have that check be part of
|
||||
# this call.
|
||||
self._verified = True
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
if not peer in self.writers:
|
||||
logging.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
write_file = tempfile.NamedTemporaryFile(delete=False, dir=self.blob_dir)
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(write_file, self.get_length, self.writer_finished)
|
||||
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return finished_deferred, writer.write, writer.cancel
|
||||
logging.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
if self.verified is True:
|
||||
if self._verified is True:
|
||||
file_handle = None
|
||||
try:
|
||||
file_handle = open(self.file_path, 'rb')
|
||||
self.readers += 1
|
||||
return file_handle
|
||||
except IOError:
|
||||
log.exception('Failed to open %s', self.file_path)
|
||||
self.close_read_handle(file_handle)
|
||||
return None
|
||||
|
||||
def delete(self):
|
||||
if not self.writers and not self.readers:
|
||||
self.verified = False
|
||||
self._verified = False
|
||||
self.moved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
@@ -232,13 +258,15 @@ class BlobFile(HashBlob):
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("File is currently being read or written and cannot be deleted")))
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
|
||||
def close_read_handle(self, file_handle):
|
||||
if file_handle is not None:
|
||||
@@ -247,7 +275,7 @@ class BlobFile(HashBlob):
|
||||
|
||||
def _close_writer(self, writer):
|
||||
if writer.write_handle is not None:
|
||||
logging.debug("Closing %s", str(self))
|
||||
log.debug("Closing %s", str(self))
|
||||
name = writer.write_handle.name
|
||||
writer.write_handle.close()
|
||||
threads.deferToThread(os.remove, name)
|
||||
@@ -287,17 +315,18 @@ class TempBlob(HashBlob):
|
||||
return None, None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
if self.verified is True:
|
||||
if self._verified is True:
|
||||
return StringIO(self.data_buffer)
|
||||
return None
|
||||
|
||||
def delete(self):
|
||||
if not self.writers and not self.readers:
|
||||
self.verified = False
|
||||
self._verified = False
|
||||
self.data_buffer = ''
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Blob is currently being read or written and cannot be deleted")))
|
||||
return defer.fail(Failure(
|
||||
ValueError("Blob is currently being read or written and cannot be deleted")))
|
||||
|
||||
def close_read_handle(self, file_handle):
|
||||
file_handle.close()
|
||||
@@ -320,7 +349,7 @@ class TempBlob(HashBlob):
|
||||
class HashBlobCreator(object):
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.hashsum = get_lbry_hash_obj()
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.blob_hash = None
|
||||
self.length = None
|
||||
@@ -333,9 +362,8 @@ class HashBlobCreator(object):
|
||||
if self.length == 0:
|
||||
self.blob_hash = None
|
||||
else:
|
||||
self.blob_hash = self.hashsum.hexdigest()
|
||||
self.blob_hash = self._hashsum.hexdigest()
|
||||
d = self._close()
|
||||
|
||||
if self.blob_hash is not None:
|
||||
d.addCallback(lambda _: self.blob_manager.creator_finished(self))
|
||||
d.addCallback(lambda _: self.blob_hash)
|
||||
@@ -344,7 +372,7 @@ class HashBlobCreator(object):
|
||||
return d
|
||||
|
||||
def write(self, data):
|
||||
self.hashsum.update(data)
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
self._write(data)
|
||||
|
||||
@@ -364,16 +392,11 @@ class BlobFileCreator(HashBlobCreator):
|
||||
def _close(self):
|
||||
temp_file_name = self.out_file.name
|
||||
self.out_file.close()
|
||||
|
||||
def change_file_name():
|
||||
shutil.move(temp_file_name, os.path.join(self.blob_dir, self.blob_hash))
|
||||
return True
|
||||
|
||||
if self.blob_hash is not None:
|
||||
d = threads.deferToThread(change_file_name)
|
||||
shutil.move(temp_file_name, os.path.join(self.blob_dir, self.blob_hash))
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
return d
|
||||
os.remove(temp_file_name)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _write(self, data):
|
||||
self.out_file.write(data)
|
||||
@@ -382,10 +405,11 @@ class BlobFileCreator(HashBlobCreator):
|
||||
class TempBlobCreator(HashBlobCreator):
|
||||
def __init__(self, blob_manager):
|
||||
HashBlobCreator.__init__(self, blob_manager)
|
||||
# TODO: use StringIO
|
||||
self.data_buffer = ''
|
||||
|
||||
def _close(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _write(self, data):
|
||||
self.data_buffer += data
|
||||
self.data_buffer += data
|
||||
@@ -1,468 +0,0 @@
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest
|
||||
from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
|
||||
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
|
||||
from twisted.internet import threads, reactor, defer, task
|
||||
from twisted.python.failure import Failure
|
||||
from collections import defaultdict, deque
|
||||
from zope.interface import implements
|
||||
from decimal import Decimal
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
import subprocess
|
||||
import socket
|
||||
import time
|
||||
import os
|
||||
|
||||
|
||||
class ReservedPoints(object):
|
||||
def __init__(self, identifier, amount):
|
||||
self.identifier = identifier
|
||||
self.amount = amount
|
||||
|
||||
|
||||
class LBRYcrdWallet(object):
|
||||
"""This class implements the LBRYWallet interface for the LBRYcrd payment system"""
|
||||
implements(ILBRYWallet)
|
||||
|
||||
def __init__(self, rpc_user, rpc_pass, rpc_url, rpc_port, start_lbrycrdd=False,
|
||||
wallet_dir=None, wallet_conf=None):
|
||||
self.rpc_conn_string = "http://%s:%s@%s:%s" % (rpc_user, rpc_pass, rpc_url, str(rpc_port))
|
||||
self.next_manage_call = None
|
||||
self.wallet_balance = Decimal(0.0)
|
||||
self.total_reserved_points = Decimal(0.0)
|
||||
self.peer_addresses = {} # {Peer: string}
|
||||
self.queued_payments = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.expected_balances = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.current_address_given_to_peer = {} # {Peer: address(string)}
|
||||
self.expected_balance_at_time = deque() # (Peer, address(string), amount(Decimal), time(datetime), count(int),
|
||||
# incremental_amount(float))
|
||||
self.max_expected_payment_time = datetime.timedelta(minutes=3)
|
||||
self.stopped = True
|
||||
self.start_lbrycrdd = start_lbrycrdd
|
||||
self.started_lbrycrdd = False
|
||||
self.wallet_dir = wallet_dir
|
||||
self.wallet_conf = wallet_conf
|
||||
self.lbrycrdd = None
|
||||
self.manage_running = False
|
||||
|
||||
def start(self):
|
||||
|
||||
def make_connection():
|
||||
if self.start_lbrycrdd is True:
|
||||
self._start_daemon()
|
||||
logging.info("Trying to connect to %s", self.rpc_conn_string)
|
||||
self.rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Connected!")
|
||||
|
||||
def start_manage():
|
||||
self.stopped = False
|
||||
self.manage()
|
||||
return True
|
||||
|
||||
d = threads.deferToThread(make_connection)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
self.stopped = True
|
||||
# If self.next_manage_call is None, then manage is currently running or else
|
||||
# start has not been called, so set stopped and do nothing else.
|
||||
if self.next_manage_call is not None:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
|
||||
d = self.manage()
|
||||
if self.start_lbrycrdd is True:
|
||||
d.addBoth(lambda _: self._stop_daemon())
|
||||
return d
|
||||
|
||||
def manage(self):
|
||||
logging.info("Doing manage")
|
||||
self.next_manage_call = None
|
||||
have_set_manage_running = [False]
|
||||
|
||||
def check_if_manage_running():
|
||||
|
||||
d = defer.Deferred()
|
||||
|
||||
def fire_if_not_running():
|
||||
if self.manage_running is False:
|
||||
self.manage_running = True
|
||||
have_set_manage_running[0] = True
|
||||
d.callback(True)
|
||||
else:
|
||||
task.deferLater(reactor, 1, fire_if_not_running)
|
||||
|
||||
fire_if_not_running()
|
||||
return d
|
||||
|
||||
d = check_if_manage_running()
|
||||
|
||||
d.addCallback(lambda _: self._check_expected_balances())
|
||||
|
||||
d.addCallback(lambda _: self._send_payments())
|
||||
|
||||
d.addCallback(lambda _: threads.deferToThread(self._get_wallet_balance))
|
||||
|
||||
def set_wallet_balance(balance):
|
||||
self.wallet_balance = balance
|
||||
|
||||
d.addCallback(set_wallet_balance)
|
||||
|
||||
def set_next_manage_call():
|
||||
if not self.stopped:
|
||||
self.next_manage_call = reactor.callLater(60, self.manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def log_error(err):
|
||||
logging.error("Something went wrong during manage. Error message: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
|
||||
def set_manage_not_running(arg):
|
||||
if have_set_manage_running[0] is True:
|
||||
self.manage_running = False
|
||||
return arg
|
||||
|
||||
d.addBoth(set_manage_not_running)
|
||||
return d
|
||||
|
||||
def get_info_exchanger(self):
|
||||
return LBRYcrdAddressRequester(self)
|
||||
|
||||
def get_wallet_info_query_handler_factory(self):
|
||||
return LBRYcrdAddressQueryHandlerFactory(self)
|
||||
|
||||
def get_balance(self):
|
||||
d = threads.deferToThread(self._get_wallet_balance)
|
||||
return d
|
||||
|
||||
def reserve_points(self, peer, amount):
|
||||
"""
|
||||
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
|
||||
|
||||
@param peer: The peer to which the payment will ultimately be sent
|
||||
|
||||
@param amount: The amount of points to reserve
|
||||
|
||||
@return: A ReservedPoints object which is given to send_points once the service has been rendered
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
#if peer in self.peer_addresses:
|
||||
if self.wallet_balance >= self.total_reserved_points + rounded_amount:
|
||||
self.total_reserved_points += rounded_amount
|
||||
return ReservedPoints(peer, rounded_amount)
|
||||
return None
|
||||
|
||||
def cancel_point_reservation(self, reserved_points):
|
||||
"""
|
||||
Return all of the points that were reserved previously for some ReservedPoints object
|
||||
|
||||
@param reserved_points: ReservedPoints previously returned by reserve_points
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self.total_reserved_points -= reserved_points.amount
|
||||
|
||||
def send_points(self, reserved_points, amount):
|
||||
"""
|
||||
Schedule a payment to be sent to a peer
|
||||
|
||||
@param reserved_points: ReservedPoints object previously returned by reserve_points
|
||||
|
||||
@param amount: amount of points to actually send, must be less than or equal to the
|
||||
amount reserved in reserved_points
|
||||
|
||||
@return: Deferred which fires when the payment has been scheduled
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
peer = reserved_points.identifier
|
||||
assert(rounded_amount <= reserved_points.amount)
|
||||
assert(peer in self.peer_addresses)
|
||||
self.queued_payments[self.peer_addresses[peer]] += rounded_amount
|
||||
# make any unused points available
|
||||
self.total_reserved_points -= (reserved_points.amount - rounded_amount)
|
||||
logging.info("ordering that %s points be sent to %s", str(rounded_amount),
|
||||
str(self.peer_addresses[peer]))
|
||||
peer.update_stats('points_sent', amount)
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_expected_payment(self, peer, amount):
|
||||
"""Increase the number of points expected to be paid by a peer"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
assert(peer in self.current_address_given_to_peer)
|
||||
address = self.current_address_given_to_peer[peer]
|
||||
logging.info("expecting a payment at address %s in the amount of %s", str(address), str(rounded_amount))
|
||||
self.expected_balances[address] += rounded_amount
|
||||
expected_balance = self.expected_balances[address]
|
||||
expected_time = datetime.datetime.now() + self.max_expected_payment_time
|
||||
self.expected_balance_at_time.append((peer, address, expected_balance, expected_time, 0, amount))
|
||||
peer.update_stats('expected_points', amount)
|
||||
|
||||
def update_peer_address(self, peer, address):
|
||||
self.peer_addresses[peer] = address
|
||||
|
||||
def get_new_address_for_peer(self, peer):
|
||||
def set_address_for_peer(address):
|
||||
self.current_address_given_to_peer[peer] = address
|
||||
return address
|
||||
d = threads.deferToThread(self._get_new_address)
|
||||
d.addCallback(set_address_for_peer)
|
||||
return d
|
||||
|
||||
def get_stream_info_for_name(self, name):
|
||||
|
||||
def get_stream_info_from_value(result):
|
||||
r_dict = {}
|
||||
if 'value' in result:
|
||||
value = result['value']
|
||||
try:
|
||||
value_dict = json.loads(value)
|
||||
except ValueError:
|
||||
return Failure(InvalidStreamInfoError(name))
|
||||
if 'stream_hash' in value_dict:
|
||||
r_dict['stream_hash'] = value_dict['stream_hash']
|
||||
if 'name' in value_dict:
|
||||
r_dict['name'] = value_dict['name']
|
||||
if 'description' in value_dict:
|
||||
r_dict['description'] = value_dict['description']
|
||||
return r_dict
|
||||
return Failure(UnknownNameError(name))
|
||||
|
||||
d = threads.deferToThread(self._get_value_for_name, name)
|
||||
d.addCallback(get_stream_info_from_value)
|
||||
return d
|
||||
|
||||
def claim_name(self, stream_hash, name, amount):
|
||||
value = json.dumps({"stream_hash": stream_hash})
|
||||
d = threads.deferToThread(self._claim_name, name, value, amount)
|
||||
return d
|
||||
|
||||
def get_available_balance(self):
|
||||
return float(self.wallet_balance - self.total_reserved_points)
|
||||
|
||||
def get_new_address(self):
|
||||
return threads.deferToThread(self._get_new_address)
|
||||
|
||||
def _start_daemon(self):
|
||||
|
||||
if os.name == "nt":
|
||||
si = subprocess.STARTUPINFO
|
||||
si.dwFlags = subprocess.STARTF_USESHOWWINDOW
|
||||
si.wShowWindow = subprocess.SW_HIDE
|
||||
self.lbrycrdd = subprocess.Popen(["lbrycrdd.exe", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf], startupinfo=si)
|
||||
else:
|
||||
self.lbrycrdd = subprocess.Popen(["./lbrycrdd", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf])
|
||||
self.started_lbrycrdd = True
|
||||
|
||||
tries = 0
|
||||
while tries < 5:
|
||||
try:
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.getinfo()
|
||||
break
|
||||
except (socket.error, JSONRPCException):
|
||||
tries += 1
|
||||
logging.warning("Failed to connect to lbrycrdd.")
|
||||
if tries < 5:
|
||||
time.sleep(2 ** tries)
|
||||
logging.warning("Trying again in %d seconds", 2 ** tries)
|
||||
else:
|
||||
logging.warning("Giving up.")
|
||||
else:
|
||||
self.lbrycrdd.terminate()
|
||||
raise ValueError("Couldn't open lbrycrdd")
|
||||
|
||||
def _stop_daemon(self):
|
||||
if self.lbrycrdd is not None and self.started_lbrycrdd is True:
|
||||
d = threads.deferToThread(self._rpc_stop)
|
||||
return d
|
||||
return defer.succeed(True)
|
||||
|
||||
def _check_expected_balances(self):
|
||||
now = datetime.datetime.now()
|
||||
balances_to_check = []
|
||||
try:
|
||||
while self.expected_balance_at_time[0][3] < now:
|
||||
balances_to_check.append(self.expected_balance_at_time.popleft())
|
||||
except IndexError:
|
||||
pass
|
||||
ds = []
|
||||
for balance_to_check in balances_to_check:
|
||||
d = threads.deferToThread(self._check_expected_balance, balance_to_check)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def handle_checks(results):
|
||||
from future_builtins import zip
|
||||
for balance, (success, result) in zip(balances_to_check, results):
|
||||
peer = balance[0]
|
||||
if success is True:
|
||||
if result is False:
|
||||
if balance[4] <= 1: # first or second strike, give them another chance
|
||||
new_expected_balance = (balance[0],
|
||||
balance[1],
|
||||
balance[2],
|
||||
datetime.datetime.now() + self.max_expected_payment_time,
|
||||
balance[4] + 1,
|
||||
balance[5])
|
||||
self.expected_balance_at_time.append(new_expected_balance)
|
||||
peer.update_score(-5.0)
|
||||
else:
|
||||
peer.update_score(-50.0)
|
||||
else:
|
||||
if balance[4] == 0:
|
||||
peer.update_score(balance[5])
|
||||
peer.update_stats('points_received', balance[5])
|
||||
else:
|
||||
logging.warning("Something went wrong checking a balance. Peer: %s, account: %s,"
|
||||
"expected balance: %s, expected time: %s, count: %s, error: %s",
|
||||
str(balance[0]), str(balance[1]), str(balance[2]), str(balance[3]),
|
||||
str(balance[4]), str(result.getErrorMessage()))
|
||||
|
||||
dl.addCallback(handle_checks)
|
||||
return dl
|
||||
|
||||
def _check_expected_balance(self, expected_balance):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Checking balance of address %s", str(expected_balance[1]))
|
||||
balance = rpc_conn.getreceivedbyaddress(expected_balance[1])
|
||||
logging.debug("received balance: %s", str(balance))
|
||||
logging.debug("expected balance: %s", str(expected_balance[2]))
|
||||
return balance >= expected_balance[2]
|
||||
|
||||
def _send_payments(self):
|
||||
logging.info("Trying to send payments, if there are any to be sent")
|
||||
|
||||
def do_send(payments):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.sendmany("", payments)
|
||||
|
||||
payments_to_send = {}
|
||||
for address, points in self.queued_payments.items():
|
||||
logging.info("Should be sending %s points to %s", str(points), str(address))
|
||||
payments_to_send[address] = float(points)
|
||||
self.total_reserved_points -= points
|
||||
self.wallet_balance -= points
|
||||
del self.queued_payments[address]
|
||||
if payments_to_send:
|
||||
logging.info("Creating a transaction with outputs %s", str(payments_to_send))
|
||||
return threads.deferToThread(do_send, payments_to_send)
|
||||
logging.info("There were no payments to send")
|
||||
return defer.succeed(True)
|
||||
|
||||
def _get_wallet_balance(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getbalance("")
|
||||
|
||||
def _get_new_address(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getnewaddress()
|
||||
|
||||
def _get_value_for_name(self, name):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getvalueforname(name)
|
||||
|
||||
def _claim_name(self, name, value, amount):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return str(rpc_conn.claimname(name, value, amount))
|
||||
|
||||
def _rpc_stop(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.stop()
|
||||
self.lbrycrdd.wait()
|
||||
|
||||
|
||||
class LBRYcrdAddressRequester(object):
|
||||
implements([IRequestCreator])
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self._protocols = []
|
||||
|
||||
######### IRequestCreator #########
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
|
||||
if not protocol in self._protocols:
|
||||
r = ClientRequest({'lbrycrd_address': True}, 'lbrycrd_address')
|
||||
d = protocol.add_request(r)
|
||||
d.addCallback(self._handle_address_response, peer, r, protocol)
|
||||
d.addErrback(self._request_failed, peer)
|
||||
self._protocols.append(protocol)
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
return defer.succeed(False)
|
||||
|
||||
######### internal calls #########
|
||||
|
||||
def _handle_address_response(self, response_dict, peer, request, protocol):
|
||||
assert request.response_identifier in response_dict, \
|
||||
"Expected %s in dict but did not get it" % request.response_identifier
|
||||
assert protocol in self._protocols, "Responding protocol is not in our list of protocols"
|
||||
address = response_dict[request.response_identifier]
|
||||
self.wallet.update_peer_address(peer, address)
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = LBRYcrdAddressQueryHandler(self.wallet)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'lbrycrd_address'
|
||||
|
||||
def get_description(self):
|
||||
return "LBRYcrd Address - an address for receiving payments via LBRYcrd"
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandler(object):
|
||||
implements(IQueryHandler)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = ['lbrycrd_address']
|
||||
self.address = None
|
||||
self.peer = None
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
self.peer = peer
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
|
||||
def create_response(address):
|
||||
self.address = address
|
||||
fields = {'lbrycrd_address': address}
|
||||
return fields
|
||||
|
||||
if self.query_identifiers[0] in queries:
|
||||
d = self.wallet.get_new_address_for_peer(self.peer)
|
||||
d.addCallback(create_response)
|
||||
return d
|
||||
if self.address is None:
|
||||
logging.warning("Expected a request for an address, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive an address request")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
@@ -0,0 +1,62 @@
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class Offer(object):
|
||||
"""A rate offer to download blobs from a host."""
|
||||
|
||||
RATE_ACCEPTED = "RATE_ACCEPTED"
|
||||
RATE_TOO_LOW = "RATE_TOO_LOW"
|
||||
RATE_UNSET = "RATE_UNSET"
|
||||
|
||||
def __init__(self, offer):
|
||||
self._state = None
|
||||
self.rate = None
|
||||
if isinstance(offer, Decimal):
|
||||
self.rate = round(offer, 5)
|
||||
elif isinstance(offer, float):
|
||||
self.rate = round(Decimal(offer), 5)
|
||||
if self.rate is None or self.rate < Decimal(0.0):
|
||||
self.unset()
|
||||
|
||||
@property
|
||||
def is_accepted(self):
|
||||
return self._state is Offer.RATE_ACCEPTED
|
||||
|
||||
@property
|
||||
def is_too_low(self):
|
||||
return self._state is Offer.RATE_TOO_LOW
|
||||
|
||||
@property
|
||||
def is_unset(self):
|
||||
return self._state is Offer.RATE_UNSET
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
if self.is_accepted:
|
||||
return Offer.RATE_ACCEPTED
|
||||
elif self.is_too_low:
|
||||
return Offer.RATE_TOO_LOW
|
||||
elif self.is_unset:
|
||||
return Offer.RATE_UNSET
|
||||
return None
|
||||
|
||||
def accept(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_ACCEPTED
|
||||
|
||||
def reject(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_TOO_LOW
|
||||
|
||||
def unset(self):
|
||||
self._state = Offer.RATE_UNSET
|
||||
|
||||
def handle(self, reply_message):
|
||||
if reply_message == Offer.RATE_TOO_LOW:
|
||||
self.reject()
|
||||
elif reply_message == Offer.RATE_ACCEPTED:
|
||||
self.accept()
|
||||
elif reply_message == Offer.RATE_UNSET:
|
||||
self.unset()
|
||||
else:
|
||||
raise Exception("Unknown offer reply %s" % str(reply_message))
|
||||
+46
-31
@@ -1,24 +1,27 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import leveldb
|
||||
import os
|
||||
import unqlite
|
||||
import time
|
||||
from Crypto.Hash import SHA512
|
||||
from Crypto.PublicKey import RSA
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet
|
||||
from lbrynet.pointtraderclient import pointtraderclient
|
||||
from twisted.internet import defer, threads
|
||||
from zope.interface import implements
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.LBRYcrdWallet import ReservedPoints
|
||||
from lbrynet.core.Wallet import ReservedPoints
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PTCWallet(object):
|
||||
"""This class sends payments to peers and also ensures that expected payments are received.
|
||||
This class is only intended to be used for testing."""
|
||||
implements(ILBRYWallet)
|
||||
implements(IWallet)
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
@@ -71,7 +74,7 @@ class PTCWallet(object):
|
||||
|
||||
def save_key(success, private_key):
|
||||
if success is True:
|
||||
threads.deferToThread(self.save_private_key, private_key.exportKey())
|
||||
self._save_private_key(private_key.exportKey())
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -95,8 +98,8 @@ class PTCWallet(object):
|
||||
def start_manage():
|
||||
self.manage()
|
||||
return True
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d.addCallback(lambda _: threads.deferToThread(self.get_wallet_private_key))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._get_wallet_private_key())
|
||||
d.addCallback(ensure_private_key_exists)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
@@ -108,14 +111,16 @@ class PTCWallet(object):
|
||||
return PointTraderKeyQueryHandlerFactory(self)
|
||||
|
||||
def reserve_points(self, peer, amount):
|
||||
"""
|
||||
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
|
||||
"""Ensure a certain amount of points are available to be sent as
|
||||
payment, before the service is rendered
|
||||
|
||||
@param peer: The peer to which the payment will ultimately be sent
|
||||
|
||||
@param amount: The amount of points to reserve
|
||||
|
||||
@return: A ReservedPoints object which is given to send_points once the service has been rendered
|
||||
@return: A ReservedPoints object which is given to send_points
|
||||
once the service has been rendered
|
||||
|
||||
"""
|
||||
if self.wallet_balance >= self.total_reserved_points + amount:
|
||||
self.total_reserved_points += amount
|
||||
@@ -154,14 +159,15 @@ class PTCWallet(object):
|
||||
ds = []
|
||||
for peer, points in self.queued_payments.items():
|
||||
if peer in self.peer_pub_keys:
|
||||
d = pointtraderclient.send_points(self.private_key, self.peer_pub_keys[peer], points)
|
||||
d = pointtraderclient.send_points(
|
||||
self.private_key, self.peer_pub_keys[peer], points)
|
||||
self.wallet_balance -= points
|
||||
self.total_reserved_points -= points
|
||||
ds.append(d)
|
||||
del self.queued_payments[peer]
|
||||
else:
|
||||
logging.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def get_balance(self):
|
||||
@@ -205,22 +211,30 @@ class PTCWallet(object):
|
||||
min_expected_balance = sum([a[0] for a in expected_payments if a[1] < expected_cutoff])
|
||||
received_balance = 0
|
||||
if self.peer_pub_keys[peer] in self.received_payments:
|
||||
received_balance = sum([a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
|
||||
received_balance = sum([
|
||||
a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
|
||||
if min_expected_balance > received_balance:
|
||||
logging.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
log.warning(
|
||||
"Account in bad standing: %s (pub_key: %s), expected amount = %s, "
|
||||
"received_amount = %s",
|
||||
peer, self.peer_pub_keys[peer], min_expected_balance, received_balance)
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
def open_db():
|
||||
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
return threads.deferToThread(open_db)
|
||||
|
||||
def save_private_key(self, private_key):
|
||||
self.db.Put("private_key", private_key)
|
||||
def _save_private_key(self, private_key):
|
||||
def save_key():
|
||||
self.db['private_key'] = private_key
|
||||
return threads.deferToThread(save_key)
|
||||
|
||||
def get_wallet_private_key(self):
|
||||
try:
|
||||
return self.db.Get("private_key")
|
||||
except KeyError:
|
||||
def _get_wallet_private_key(self):
|
||||
def get_key():
|
||||
if 'private_key' in self.db:
|
||||
return self.db['private_key']
|
||||
return None
|
||||
return threads.deferToThread(get_key)
|
||||
|
||||
|
||||
class PointTraderKeyExchanger(object):
|
||||
@@ -256,9 +270,9 @@ class PointTraderKeyExchanger(object):
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
return err
|
||||
|
||||
|
||||
class PointTraderKeyQueryHandlerFactory(object):
|
||||
@@ -277,7 +291,8 @@ class PointTraderKeyQueryHandlerFactory(object):
|
||||
return 'public_key'
|
||||
|
||||
def get_description(self):
|
||||
return "Point Trader Address - an address for receiving payments on the point trader testing network"
|
||||
return ("Point Trader Address - an address for receiving payments on the "
|
||||
"point trader testing network")
|
||||
|
||||
|
||||
class PointTraderKeyQueryHandler(object):
|
||||
@@ -301,15 +316,15 @@ class PointTraderKeyQueryHandler(object):
|
||||
try:
|
||||
RSA.importKey(new_encoded_pub_key)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
logging.warning("Client sent an invalid public key.")
|
||||
log.warning("Client sent an invalid public key.")
|
||||
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
|
||||
self.public_key = new_encoded_pub_key
|
||||
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
|
||||
logging.debug("Received the client's public key: %s", str(self.public_key))
|
||||
log.debug("Received the client's public key: %s", str(self.public_key))
|
||||
fields = {'public_key': self.wallet.encoded_public_key}
|
||||
return defer.succeed(fields)
|
||||
if self.public_key is None:
|
||||
logging.warning("Expected a public key, but did not receive one")
|
||||
log.warning("Expected a public key, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
return defer.succeed({})
|
||||
@@ -1,6 +1,13 @@
|
||||
from lbrynet.core.Strategy import get_default_strategy, OnlyFreeStrategy
|
||||
from lbrynet import conf
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class BasePaymentRateManager(object):
|
||||
def __init__(self, rate):
|
||||
self.min_blob_data_payment_rate = rate
|
||||
def __init__(self, rate=None, info_rate=None):
|
||||
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
|
||||
self.min_blob_info_payment_rate = (
|
||||
info_rate if info_rate is not None else conf.settings['min_info_rate'])
|
||||
|
||||
|
||||
class PaymentRateManager(object):
|
||||
@@ -26,4 +33,94 @@ class PaymentRateManager(object):
|
||||
return self.min_blob_data_payment_rate
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
self.points_paid += amount
|
||||
|
||||
|
||||
class NegotiatedPaymentRateManager(object):
|
||||
def __init__(self, base, availability_tracker, generous=None):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
@param availability_tracker: a BlobAvailabilityTracker
|
||||
@param rate: the min blob data payment rate
|
||||
"""
|
||||
|
||||
self.base = base
|
||||
self.points_paid = 0.0
|
||||
self.blob_tracker = availability_tracker
|
||||
self.generous = generous if generous is not None else conf.settings['is_generous_host']
|
||||
self.strategy = get_default_strategy(self.blob_tracker,
|
||||
base_price=self.base.min_blob_data_payment_rate,
|
||||
is_generous=generous)
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
return (offer.is_too_low and
|
||||
round(Decimal.from_float(offer.rate), 5) >= round(self.strategy.max_rate, 5))
|
||||
return False
|
||||
|
||||
|
||||
class OnlyFreePaymentsManager(object):
|
||||
def __init__(self, **kwargs):
|
||||
"""
|
||||
A payment rate manager that will only ever accept and offer a rate of 0.0,
|
||||
Used for testing
|
||||
"""
|
||||
|
||||
self.base = BasePaymentRateManager(0.0, 0.0)
|
||||
self.points_paid = 0.0
|
||||
self.generous = True
|
||||
self.strategy = OnlyFreeStrategy()
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
if offer.rate > 0.0:
|
||||
return True
|
||||
return False
|
||||
+16
-6
@@ -1,19 +1,23 @@
|
||||
from collections import defaultdict
|
||||
import datetime
|
||||
from collections import defaultdict
|
||||
from lbrynet.core import utils
|
||||
|
||||
|
||||
# Do not create this object except through PeerManager
|
||||
class Peer(object):
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
# If a peer is reported down, we wait till this time till reattempting connection
|
||||
self.attempt_connection_at = None
|
||||
# Number of times this Peer has been reported to be down, resets to 0 when it is up
|
||||
self.down_count = 0
|
||||
# Number of succesful connections (with full protocol completion) to this peer
|
||||
self.success_count = 0
|
||||
self.score = 0
|
||||
self.stats = defaultdict(float) # {string stat_type, float count}
|
||||
|
||||
def is_available(self):
|
||||
if (self.attempt_connection_at is None or
|
||||
datetime.datetime.today() > self.attempt_connection_at):
|
||||
if self.attempt_connection_at is None or utils.today() > self.attempt_connection_at:
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -21,10 +25,13 @@ class Peer(object):
|
||||
self.down_count = 0
|
||||
self.attempt_connection_at = None
|
||||
|
||||
def report_success(self):
|
||||
self.success_count += 1
|
||||
|
||||
def report_down(self):
|
||||
self.down_count += 1
|
||||
timeout_time = datetime.timedelta(seconds=60 * self.down_count)
|
||||
self.attempt_connection_at = datetime.datetime.today() + timeout_time
|
||||
self.attempt_connection_at = utils.today() + timeout_time
|
||||
|
||||
def update_score(self, score_change):
|
||||
self.score += score_change
|
||||
@@ -33,4 +40,7 @@ class Peer(object):
|
||||
self.stats[stat_type] += count
|
||||
|
||||
def __str__(self):
|
||||
return self.host + ":" + str(self.port)
|
||||
return '{}:{}'.format(self.host, self.port)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Peer({!r}, {!r})'.format(self.host, self.port)
|
||||
@@ -16,4 +16,4 @@ class DummyPeerFinder(object):
|
||||
return defer.succeed([])
|
||||
|
||||
def get_most_popular_hashes(self, num_to_return):
|
||||
return []
|
||||
return []
|
||||
@@ -11,4 +11,4 @@ class PeerManager(object):
|
||||
return peer
|
||||
peer = Peer(host, port)
|
||||
self.peers.append(peer)
|
||||
return peer
|
||||
return peer
|
||||
@@ -0,0 +1,62 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.interfaces import IBlobPriceModel
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
def get_default_price_model(blob_tracker, base_price, **kwargs):
|
||||
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
|
||||
|
||||
|
||||
class ZeroPrice(object):
|
||||
def __init__(self):
|
||||
self.base_price = 0.0
|
||||
|
||||
def calculate_price(self, blob):
|
||||
return 0.0
|
||||
|
||||
|
||||
class MeanAvailabilityWeightedPrice(object):
|
||||
"""Calculate mean-blob-availability and stream-position weighted price for a blob
|
||||
|
||||
Attributes:
|
||||
base_price (float): base price
|
||||
alpha (float): constant, > 0.0 and <= 1.0, used to more highly
|
||||
value blobs at the beginning of a stream.
|
||||
alpha defaults to 1.0, which has a null effect
|
||||
blob_tracker (BlobAvailabilityTracker): blob availability tracker
|
||||
|
||||
"""
|
||||
implementer(IBlobPriceModel)
|
||||
|
||||
def __init__(self, tracker, base_price=None, alpha=1.0):
|
||||
self.blob_tracker = tracker
|
||||
base_price = base_price if base_price is not None else conf.settings['data_rate']
|
||||
self.base_price = Decimal(base_price)
|
||||
self.alpha = Decimal(alpha)
|
||||
|
||||
def calculate_price(self, blob):
|
||||
mean_availability = self.blob_tracker.last_mean_availability
|
||||
availability = self.blob_tracker.availability.get(blob, [])
|
||||
index = 0 # blob.index
|
||||
availability_mult = self._get_availability_multiplier(mean_availability, availability)
|
||||
price = self.base_price * availability_mult / self._frontload(index)
|
||||
return round(price, 5)
|
||||
|
||||
def _get_availability_multiplier(self, mean_availability, availability):
|
||||
return Decimal(max(1, mean_availability) / Decimal(max(1, len(availability))))
|
||||
|
||||
def _frontload(self, index):
|
||||
"""Get front-load multiplier, used to weight prices of blobs in a
|
||||
stream towards the front of the stream.
|
||||
|
||||
At index 0, returns 1.0
|
||||
As index increases, return value approaches 2.0
|
||||
|
||||
@param index: blob position in stream
|
||||
@return: front-load multiplier
|
||||
|
||||
"""
|
||||
|
||||
return Decimal(2.0) - (self.alpha ** index)
|
||||
+44
-94
@@ -1,5 +1,11 @@
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IRateLimiter
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyRateLimiter(object):
|
||||
@@ -10,22 +16,20 @@ class DummyRateLimiter(object):
|
||||
self.total_ul_bytes = 0
|
||||
self.target_dl = 0
|
||||
self.target_ul = 0
|
||||
self.ul_delay = 0.00
|
||||
self.dl_delay = 0.00
|
||||
self.next_tick = None
|
||||
self.tick_call = None
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(1)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
pass
|
||||
@@ -33,12 +37,6 @@ class DummyRateLimiter(object):
|
||||
def set_ul_limit(self, limit):
|
||||
pass
|
||||
|
||||
def ul_wait_time(self):
|
||||
return self.ul_delay
|
||||
|
||||
def dl_wait_time(self):
|
||||
return self.dl_delay
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
@@ -58,66 +56,34 @@ class RateLimiter(object):
|
||||
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
|
||||
self.max_dl_bytes = max_dl_bytes
|
||||
self.max_ul_bytes = max_ul_bytes
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.next_tick = None
|
||||
self.next_unthrottle_dl = None
|
||||
self.next_unthrottle_ul = None
|
||||
|
||||
self.next_dl_check = None
|
||||
self.next_ul_check = None
|
||||
|
||||
self.dl_check_interval = 1.0
|
||||
self.ul_check_interval = 1.0
|
||||
self.tick_call = None
|
||||
self.tick_interval = 0.1
|
||||
|
||||
self.dl_throttled = False
|
||||
self.ul_throttled = False
|
||||
|
||||
self.protocols = []
|
||||
|
||||
def start(self):
|
||||
log.info("Starting %s", self)
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(self.tick_interval)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
# happens once per second
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
if self.max_dl_bytes is not None:
|
||||
if self.dl_bytes_this_second == 0:
|
||||
self.dl_check_interval = 1.0
|
||||
else:
|
||||
self.dl_check_interval = min(1.0, self.dl_check_interval *
|
||||
self.max_dl_bytes / self.dl_bytes_this_second)
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
if self.max_ul_bytes is not None:
|
||||
if self.ul_bytes_this_second == 0:
|
||||
self.ul_check_interval = 1.0
|
||||
else:
|
||||
self.ul_check_interval = min(1.0, self.ul_check_interval *
|
||||
self.max_ul_bytes / self.ul_bytes_this_second)
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.unthrottle_dl()
|
||||
self.unthrottle_ul()
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
log.info("Stopping %s", self)
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
self.max_dl_bytes = limit
|
||||
@@ -128,28 +94,18 @@ class RateLimiter(object):
|
||||
#throttling
|
||||
|
||||
def check_dl(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_dl_check = None
|
||||
|
||||
if self.dl_bytes_this_second > self.max_dl_bytes:
|
||||
self.throttle_dl()
|
||||
else:
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
self.dl_check_interval = min(self.dl_check_interval * 2, 1.0)
|
||||
need_throttle = (self.max_dl_bytes is not None and
|
||||
self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_dl)
|
||||
|
||||
def check_ul(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_ul_check = None
|
||||
|
||||
if self.ul_bytes_this_second > self.max_ul_bytes:
|
||||
self.throttle_ul()
|
||||
else:
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.ul_check_interval = min(self.ul_check_interval * 2, 1.0)
|
||||
need_throttle = (self.max_ul_bytes is not None and
|
||||
self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_ul)
|
||||
|
||||
def throttle_dl(self):
|
||||
if self.dl_throttled is False:
|
||||
@@ -175,23 +131,17 @@ class RateLimiter(object):
|
||||
protocol.unthrottle_upload()
|
||||
self.ul_throttled = False
|
||||
|
||||
#deprecated
|
||||
|
||||
def ul_wait_time(self):
|
||||
return 0
|
||||
|
||||
def dl_wait_time(self):
|
||||
return 0
|
||||
|
||||
#called by protocols
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.dl_bytes_this_interval += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
self.check_dl()
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_second += num_bytes
|
||||
self.ul_bytes_this_interval += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
self.check_ul()
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
if protocol not in self.protocols:
|
||||
@@ -203,4 +153,4 @@ class RateLimiter(object):
|
||||
|
||||
def unregister_protocol(self, protocol):
|
||||
if protocol in self.protocols:
|
||||
self.protocols.remove(protocol)
|
||||
self.protocols.remove(protocol)
|
||||
+173
-77
@@ -1,6 +1,5 @@
|
||||
import logging
|
||||
import miniupnpc
|
||||
from lbrynet.core.PTCWallet import PTCWallet
|
||||
from lbrynet.core.BlobManager import DiskBlobManager, TempBlobManager
|
||||
from lbrynet.dht import node
|
||||
from lbrynet.core.PeerManager import PeerManager
|
||||
@@ -9,65 +8,96 @@ from lbrynet.core.client.DHTPeerFinder import DHTPeerFinder
|
||||
from lbrynet.core.HashAnnouncer import DummyHashAnnouncer
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager, NegotiatedPaymentRateManager
|
||||
from lbrynet.core.BlobAvailability import BlobAvailabilityTracker
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
class LBRYSession(object):
|
||||
"""This class manages all important services common to any application that uses the network:
|
||||
the hash announcer, which informs other peers that this peer is associated with some hash. Usually,
|
||||
this means this peer has a blob identified by the hash in question, but it can be used for other
|
||||
purposes.
|
||||
the peer finder, which finds peers that are associated with some hash.
|
||||
the blob manager, which keeps track of which blobs have been downloaded and provides access to them,
|
||||
the rate limiter, which attempts to ensure download and upload rates stay below a set maximum,
|
||||
and upnp, which opens holes in compatible firewalls so that remote peers can connect to this peer."""
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None, lbryid=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None, hash_announcer=None,
|
||||
blob_dir=None, blob_manager=None, peer_port=None, use_upnp=True,
|
||||
rate_limiter=None, wallet=None):
|
||||
"""
|
||||
@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
class Session(object):
|
||||
"""This class manages all important services common to any application that uses the network.
|
||||
|
||||
the hash announcer, which informs other peers that this peer is
|
||||
associated with some hash. Usually, this means this peer has a
|
||||
blob identified by the hash in question, but it can be used for
|
||||
other purposes.
|
||||
|
||||
the peer finder, which finds peers that are associated with some
|
||||
hash.
|
||||
|
||||
the blob manager, which keeps track of which blobs have been
|
||||
downloaded and provides access to them,
|
||||
|
||||
the rate limiter, which attempts to ensure download and upload
|
||||
rates stay below a set maximum
|
||||
|
||||
upnp, which opens holes in compatible firewalls so that remote
|
||||
peers can connect to this peer.
|
||||
"""
|
||||
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None,
|
||||
lbryid=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None,
|
||||
hash_announcer=None, blob_dir=None,
|
||||
blob_manager=None, peer_port=None, use_upnp=True,
|
||||
rate_limiter=None, wallet=None,
|
||||
dht_node_class=node.Node, blob_tracker_class=None,
|
||||
payment_rate_manager_class=None, is_generous=True):
|
||||
"""@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
@param db_dir: The directory in which levelDB files should be stored
|
||||
|
||||
@param lbryid: The unique ID of this node
|
||||
|
||||
@param peer_manager: An object which keeps track of all known peers. If None, a PeerManager will be created
|
||||
@param peer_manager: An object which keeps track of all known
|
||||
peers. If None, a PeerManager will be created
|
||||
|
||||
@param dht_node_port: The port on which the dht node should listen for incoming connections
|
||||
@param dht_node_port: The port on which the dht node should
|
||||
listen for incoming connections
|
||||
|
||||
@param known_dht_nodes: A list of nodes which the dht node should use to bootstrap into the dht
|
||||
@param known_dht_nodes: A list of nodes which the dht node
|
||||
should use to bootstrap into the dht
|
||||
|
||||
@param peer_finder: An object which is used to look up peers that are associated with some hash. If None,
|
||||
a DHTPeerFinder will be used, which looks for peers in the distributed hash table.
|
||||
@param peer_finder: An object which is used to look up peers
|
||||
that are associated with some hash. If None, a
|
||||
DHTPeerFinder will be used, which looks for peers in the
|
||||
distributed hash table.
|
||||
|
||||
@param hash_announcer: An object which announces to other peers that this peer is associated with some hash.
|
||||
If None, and peer_port is not None, a DHTHashAnnouncer will be used. If None and
|
||||
peer_port is None, a DummyHashAnnouncer will be used, which will not actually announce
|
||||
anything.
|
||||
@param hash_announcer: An object which announces to other
|
||||
peers that this peer is associated with some hash. If
|
||||
None, and peer_port is not None, a DHTHashAnnouncer will
|
||||
be used. If None and peer_port is None, a
|
||||
DummyHashAnnouncer will be used, which will not actually
|
||||
announce anything.
|
||||
|
||||
@param blob_dir: The directory in which blobs will be stored. If None and blob_manager is None, blobs will
|
||||
be stored in memory only.
|
||||
@param blob_dir: The directory in which blobs will be
|
||||
stored. If None and blob_manager is None, blobs will be
|
||||
stored in memory only.
|
||||
|
||||
@param blob_manager: An object which keeps track of downloaded blobs and provides access to them. If None,
|
||||
and blob_dir is not None, a DiskBlobManager will be used, with the given blob_dir.
|
||||
If None and blob_dir is None, a TempBlobManager will be used, which stores blobs in
|
||||
memory only.
|
||||
@param blob_manager: An object which keeps track of downloaded
|
||||
blobs and provides access to them. If None, and blob_dir
|
||||
is not None, a DiskBlobManager will be used, with the
|
||||
given blob_dir. If None and blob_dir is None, a
|
||||
TempBlobManager will be used, which stores blobs in memory
|
||||
only.
|
||||
|
||||
@param peer_port: The port on which other peers should connect to this peer
|
||||
@param peer_port: The port on which other peers should connect
|
||||
to this peer
|
||||
|
||||
@param use_upnp: Whether or not to try to open a hole in the firewall so that outside peers can connect to
|
||||
this peer's peer_port and dht_node_port
|
||||
@param use_upnp: Whether or not to try to open a hole in the
|
||||
firewall so that outside peers can connect to this peer's
|
||||
peer_port and dht_node_port
|
||||
|
||||
@param rate_limiter: An object which keeps track of the amount of data transferred to and from this peer,
|
||||
and can limit that rate if desired
|
||||
@param rate_limiter: An object which keeps track of the amount
|
||||
of data transferred to and from this peer, and can limit
|
||||
that rate if desired
|
||||
|
||||
@param wallet: An object which will be used to keep track of expected payments and which will pay peers.
|
||||
If None, a wallet which uses the Point Trader system will be used, which is meant for testing
|
||||
only
|
||||
@param wallet: An object which will be used to keep track of
|
||||
expected payments and which will pay peers. If None, a
|
||||
wallet which uses the Point Trader system will be used,
|
||||
which is meant for testing only
|
||||
|
||||
@return:
|
||||
"""
|
||||
self.db_dir = db_dir
|
||||
|
||||
@@ -85,6 +115,9 @@ class LBRYSession(object):
|
||||
self.blob_dir = blob_dir
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
self.blob_tracker = None
|
||||
self.blob_tracker_class = blob_tracker_class or BlobAvailabilityTracker
|
||||
|
||||
self.peer_port = peer_port
|
||||
|
||||
self.use_upnp = use_upnp
|
||||
@@ -92,24 +125,28 @@ class LBRYSession(object):
|
||||
self.rate_limiter = rate_limiter
|
||||
|
||||
self.external_ip = '127.0.0.1'
|
||||
self.upnp_handler = None
|
||||
self.upnp_redirects_set = False
|
||||
|
||||
self.upnp_redirects = []
|
||||
|
||||
self.wallet = wallet
|
||||
|
||||
self.dht_node_class = dht_node_class
|
||||
self.dht_node = None
|
||||
|
||||
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
|
||||
self.payment_rate_manager = None
|
||||
self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager
|
||||
self.is_generous = is_generous
|
||||
|
||||
def setup(self):
|
||||
"""Create the blob directory and database if necessary, start all desired services"""
|
||||
|
||||
logging.debug("Setting up the lbry session")
|
||||
log.debug("Setting up the lbry session")
|
||||
|
||||
if self.lbryid is None:
|
||||
self.lbryid = generate_id()
|
||||
|
||||
if self.wallet is None:
|
||||
from lbrynet.core.PTCWallet import PTCWallet
|
||||
self.wallet = PTCWallet(self.db_dir)
|
||||
|
||||
if self.peer_manager is None:
|
||||
@@ -124,7 +161,7 @@ class LBRYSession(object):
|
||||
d.addCallback(lambda _: self._setup_dht())
|
||||
else:
|
||||
if self.hash_announcer is None and self.peer_port is not None:
|
||||
logging.warning("The server has no way to advertise its available blobs.")
|
||||
log.warning("The server has no way to advertise its available blobs.")
|
||||
self.hash_announcer = DummyHashAnnouncer()
|
||||
|
||||
d.addCallback(lambda _: self._setup_other_components())
|
||||
@@ -132,44 +169,78 @@ class LBRYSession(object):
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop all services"""
|
||||
log.info('Shutting down %s', self)
|
||||
ds = []
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_tracker.stop))
|
||||
if self.dht_node is not None:
|
||||
ds.append(defer.maybeDeferred(self.dht_node.stop))
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.upnp_redirects_set is True:
|
||||
if self.rate_limiter is not None:
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
if self.peer_finder is not None:
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
if self.hash_announcer is not None:
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
if self.wallet is not None:
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.use_upnp is True:
|
||||
ds.append(defer.maybeDeferred(self._unset_upnp))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _try_upnp(self):
|
||||
|
||||
logging.debug("In _try_upnp")
|
||||
log.debug("In _try_upnp")
|
||||
|
||||
def threaded_try_upnp():
|
||||
if self.use_upnp is False:
|
||||
logging.debug("Not using upnp")
|
||||
log.debug("Not using upnp")
|
||||
return False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
self.upnp_handler = u
|
||||
u.selectigd()
|
||||
external_ip = u.externalipaddress()
|
||||
if external_ip != '0.0.0.0':
|
||||
self.external_ip = external_ip
|
||||
if self.peer_port is not None:
|
||||
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
|
||||
if u.getspecificportmapping(self.peer_port, 'TCP') is None:
|
||||
u.addportmapping(
|
||||
self.peer_port, 'TCP', u.lanaddr, self.peer_port,
|
||||
'LBRY peer port', '')
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
log.info("Set UPnP redirect for TCP port %d", self.peer_port)
|
||||
else:
|
||||
# see comment below
|
||||
log.warning("UPnP redirect already set for TCP port %d", self.peer_port)
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
if self.dht_node_port is not None:
|
||||
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
|
||||
self.upnp_redirects_set = True
|
||||
if u.getspecificportmapping(self.dht_node_port, 'UDP') is None:
|
||||
u.addportmapping(
|
||||
self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port,
|
||||
'LBRY DHT port', '')
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
log.info("Set UPnP redirect for UDP port %d", self.dht_node_port)
|
||||
else:
|
||||
# TODO: check that the existing redirect was
|
||||
# put up by an old lbrynet session before
|
||||
# grabbing it if such a disconnected redirect
|
||||
# exists, then upnp won't work unless the
|
||||
# redirect is appended or is torn down and set
|
||||
# back up. a bad shutdown of lbrynet could
|
||||
# leave such a redirect up and cause problems
|
||||
# on the next start. this could be
|
||||
# problematic if a previous lbrynet session
|
||||
# didn't make the redirect, and it was made by
|
||||
# another application
|
||||
log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port)
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
return True
|
||||
return False
|
||||
|
||||
def upnp_failed(err):
|
||||
logging.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
return False
|
||||
|
||||
d = threads.deferToThread(threaded_try_upnp)
|
||||
@@ -180,10 +251,7 @@ class LBRYSession(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("Starting the dht")
|
||||
|
||||
def match_port(h, p):
|
||||
return h, p
|
||||
log.info("Starting DHT")
|
||||
|
||||
def join_resolved_addresses(result):
|
||||
addresses = []
|
||||
@@ -201,11 +269,14 @@ class LBRYSession(object):
|
||||
ds = []
|
||||
for host, port in self.known_dht_nodes:
|
||||
d = reactor.resolve(host)
|
||||
d.addCallback(match_port, port)
|
||||
d.addCallback(lambda h: (h, port)) # match host to port
|
||||
ds.append(d)
|
||||
|
||||
self.dht_node = node.Node(udpPort=self.dht_node_port, lbryid=self.lbryid,
|
||||
externalIP=self.external_ip)
|
||||
self.dht_node = self.dht_node_class(
|
||||
udpPort=self.dht_node_port,
|
||||
lbryid=self.lbryid,
|
||||
externalIP=self.external_ip
|
||||
)
|
||||
self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager)
|
||||
if self.hash_announcer is None:
|
||||
self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port)
|
||||
@@ -216,7 +287,7 @@ class LBRYSession(object):
|
||||
return dl
|
||||
|
||||
def _setup_other_components(self):
|
||||
logging.debug("Setting up the rest of the components")
|
||||
log.debug("Setting up the rest of the components")
|
||||
|
||||
if self.rate_limiter is None:
|
||||
self.rate_limiter = RateLimiter()
|
||||
@@ -225,21 +296,46 @@ class LBRYSession(object):
|
||||
if self.blob_dir is None:
|
||||
self.blob_manager = TempBlobManager(self.hash_announcer)
|
||||
else:
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer, self.blob_dir, self.db_dir)
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer,
|
||||
self.blob_dir,
|
||||
self.db_dir)
|
||||
|
||||
self.rate_limiter.tick()
|
||||
if self.blob_tracker is None:
|
||||
self.blob_tracker = self.blob_tracker_class(self.blob_manager,
|
||||
self.peer_finder,
|
||||
self.dht_node)
|
||||
if self.payment_rate_manager is None:
|
||||
self.payment_rate_manager = self.payment_rate_manager_class(
|
||||
self.base_payment_rate_manager,
|
||||
self.blob_tracker,
|
||||
self.is_generous)
|
||||
|
||||
self.rate_limiter.start()
|
||||
d1 = self.blob_manager.setup()
|
||||
d2 = self.wallet.start()
|
||||
return defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
|
||||
dl.addCallback(lambda _: self.blob_tracker.start())
|
||||
return dl
|
||||
|
||||
def _unset_upnp(self):
|
||||
log.info("Unsetting upnp for %s", self)
|
||||
|
||||
def threaded_unset_upnp():
|
||||
u = self.upnp_handler
|
||||
if self.peer_port is not None:
|
||||
u.deleteportmapping(self.peer_port, 'TCP')
|
||||
if self.dht_node_port is not None:
|
||||
u.deleteportmapping(self.dht_node_port, 'UDP')
|
||||
self.upnp_redirects_set = False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
for port, protocol in self.upnp_redirects:
|
||||
if u.getspecificportmapping(port, protocol) is None:
|
||||
log.warning(
|
||||
"UPnP redirect for %s %d was removed by something else.",
|
||||
protocol, port)
|
||||
else:
|
||||
u.deleteportmapping(port, protocol)
|
||||
log.info("Removed UPnP redirect for %s %d.", protocol, port)
|
||||
self.upnp_redirects = []
|
||||
|
||||
return threads.deferToThread(threaded_unset_upnp)
|
||||
d = threads.deferToThread(threaded_unset_upnp)
|
||||
d.addErrback(lambda err: str(err))
|
||||
return d
|
||||
@@ -0,0 +1,152 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.interfaces import INegotiationStrategy
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
|
||||
|
||||
|
||||
def get_default_strategy(blob_tracker, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
|
||||
|
||||
|
||||
class Strategy(object):
|
||||
"""
|
||||
Base for negotiation strategies
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
|
||||
self.price_model = price_model
|
||||
self.is_generous = (
|
||||
is_generous if is_generous is not None else conf.settings['is_generous_host'])
|
||||
self.accepted_offers = {}
|
||||
self.pending_sent_offers = {}
|
||||
self.offers_sent = {}
|
||||
self.offers_received = {}
|
||||
self.max_rate = max_rate or Decimal(self.price_model.base_price * 50)
|
||||
self.min_rate = Decimal(min_rate)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def make_offer(self, peer, blobs):
|
||||
offer_count = self.offers_sent.get(peer, 0)
|
||||
self._add_offer_sent(peer)
|
||||
if peer in self.accepted_offers:
|
||||
# if there was a previous accepted offer, use that
|
||||
offer = self.accepted_offers[peer]
|
||||
if peer in self.pending_sent_offers:
|
||||
del self.pending_sent_offers[peer]
|
||||
elif offer_count == 0 and self.is_generous:
|
||||
# Try asking for it for free
|
||||
offer = Offer(Decimal(0.0))
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
else:
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._make_rate_offer(rates, offer_count)
|
||||
offer = Offer(price)
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
return offer
|
||||
|
||||
def respond_to_offer(self, offer, peer, blobs):
|
||||
offer_count = self.offers_received.get(peer, 0)
|
||||
self._add_offer_received(peer)
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._get_response_rate(rates, offer_count)
|
||||
if peer in self.accepted_offers:
|
||||
offer = self.accepted_offers[peer]
|
||||
elif offer.rate == 0.0 and offer_count == 0 and self.is_generous:
|
||||
# give blobs away for free by default on the first request
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
elif offer.rate >= price:
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
else:
|
||||
offer.reject()
|
||||
if peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
return offer
|
||||
|
||||
def update_accepted_offers(self, peer, offer):
|
||||
if not offer.is_accepted and peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
if offer.is_accepted:
|
||||
self.accepted_offers.update({peer: offer})
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
|
||||
def _add_offer_sent(self, peer):
|
||||
turn = self.offers_sent.get(peer, 0) + 1
|
||||
self.offers_sent.update({peer: turn})
|
||||
|
||||
def _add_offer_received(self, peer):
|
||||
turn = self.offers_received.get(peer, 0) + 1
|
||||
self.offers_received.update({peer: turn})
|
||||
|
||||
def _bounded_price(self, price):
|
||||
price_for_return = Decimal(min(self.max_rate, max(price, self.min_rate)))
|
||||
return price_for_return
|
||||
|
||||
|
||||
class BasicAvailabilityWeightedStrategy(Strategy):
|
||||
"""Basic strategy to target blob prices based on supply relative to mean supply
|
||||
|
||||
Discount price target with each incoming request, and raise it
|
||||
with each outgoing from the modeled price until the rate is
|
||||
accepted or a threshold is reached
|
||||
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, blob_tracker, acceleration=1.25,
|
||||
deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0,
|
||||
is_generous=None,
|
||||
base_price=0.0001, alpha=1.0):
|
||||
price_model = MeanAvailabilityWeightedPrice(
|
||||
blob_tracker, base_price=base_price, alpha=alpha)
|
||||
Strategy.__init__(self, price_model, max_rate, min_rate, is_generous)
|
||||
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
|
||||
self._deceleration = Decimal(deceleration)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
mean_rate = Decimal(sum(rates)) / Decimal(max(len(rates), 1))
|
||||
return mean_rate
|
||||
|
||||
def _premium(self, rate, turn):
|
||||
return rate * (self._acceleration ** Decimal(turn))
|
||||
|
||||
def _discount(self, rate, turn):
|
||||
return rate * (self._deceleration ** Decimal(turn))
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
discounted = self._discount(rate, offer_count)
|
||||
rounded_price = round(discounted, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
with_premium = self._premium(rate, offer_count)
|
||||
rounded_price = round(with_premium, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
|
||||
class OnlyFreeStrategy(Strategy):
|
||||
implementer(INegotiationStrategy)
|
||||
def __init__(self, *args, **kwargs):
|
||||
price_model = ZeroPrice()
|
||||
Strategy.__init__(self, price_model, 0.0, 0.0, True)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
return 0.0
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return 0.0
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return 0.0
|
||||
@@ -3,6 +3,9 @@ from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamCreator(object):
|
||||
"""Classes which derive from this class create a 'stream', which can be any
|
||||
collection of associated blobs and associated metadata. These classes
|
||||
@@ -42,7 +45,7 @@ class StreamCreator(object):
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
logging.debug("stop has been called for StreamCreator")
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
current_blob = self.current_blob
|
||||
@@ -61,13 +64,13 @@ class StreamCreator(object):
|
||||
def _finished(self):
|
||||
pass
|
||||
|
||||
# TODO: move the stream creation process to its own thread and
|
||||
# remove the reactor from this process.
|
||||
def write(self, data):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self._write(data)
|
||||
if self.stopped is False and self.streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def _write(self, data):
|
||||
pass
|
||||
pass
|
||||
@@ -3,6 +3,10 @@ import json
|
||||
import logging
|
||||
from twisted.internet import threads
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamDescriptorReader(object):
|
||||
@@ -69,7 +73,9 @@ class StreamDescriptorWriter(object):
|
||||
return self._write_stream_descriptor(json.dumps(sd_info))
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
"""This method must be overridden by subclasses to write raw data to the stream descriptor"""
|
||||
"""This method must be overridden by subclasses to write raw data to
|
||||
the stream descriptor
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -81,7 +87,7 @@ class PlainStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
|
||||
def write_file():
|
||||
logging.debug("Writing the sd file to disk")
|
||||
log.debug("Writing the sd file to disk")
|
||||
with open(self.sd_file_name, 'w') as sd_file:
|
||||
sd_file.write(raw_data)
|
||||
return self.sd_file_name
|
||||
@@ -96,85 +102,137 @@ class BlobStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
logging.debug("Creating the new blob for the stream descriptor")
|
||||
log.debug("Creating the new blob for the stream descriptor")
|
||||
blob_creator = self.blob_manager.get_blob_creator()
|
||||
blob_creator.write(raw_data)
|
||||
logging.debug("Wrote the data to the new blob")
|
||||
log.debug("Wrote the data to the new blob")
|
||||
return blob_creator.close()
|
||||
|
||||
|
||||
class StreamMetadata(object):
|
||||
FROM_BLOB = 1
|
||||
FROM_PLAIN = 2
|
||||
|
||||
def __init__(self, validator, options, factories):
|
||||
self.validator = validator
|
||||
self.options = options
|
||||
self.factories = factories
|
||||
self.metadata_source = None
|
||||
self.source_blob_hash = None
|
||||
self.source_file = None
|
||||
|
||||
|
||||
class StreamDescriptorIdentifier(object):
|
||||
"""Tries to determine the type of stream described by the stream descriptor using the
|
||||
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
|
||||
and returns the appropriate ones based on the type of the stream descriptor given
|
||||
"""
|
||||
def __init__(self):
|
||||
self._sd_info_validators = {} # {stream_type: IStreamDescriptorValidator}
|
||||
self._stream_downloader_factories = defaultdict(list) # {stream_type: [IStreamDownloaderFactory]}
|
||||
# {stream_type: IStreamDescriptorValidator}
|
||||
self._sd_info_validators = {}
|
||||
# {stream_type: IStreamOptions
|
||||
self._stream_options = {}
|
||||
# {stream_type: [IStreamDownloaderFactory]}
|
||||
self._stream_downloader_factories = defaultdict(list)
|
||||
|
||||
def add_stream_info_validator(self, stream_type, sd_info_validator):
|
||||
"""
|
||||
This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
|
||||
"""This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
|
||||
There can only be one StreamDescriptorValidator for each type of stream.
|
||||
|
||||
@param stream_type: A string representing the type of stream descriptor. This must be unique to
|
||||
this stream descriptor.
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor. This must be unique to this stream descriptor.
|
||||
|
||||
@param sd_info_validator: A class implementing the IStreamDescriptorValidator interface. This class's
|
||||
constructor will be passed the raw metadata in the stream descriptor file and its 'validate' method
|
||||
will then be called. If the validation step fails, an exception will be thrown, preventing the stream
|
||||
descriptor from being further processed.
|
||||
@param sd_info_validator: A class implementing the
|
||||
IStreamDescriptorValidator interface. This class's
|
||||
constructor will be passed the raw metadata in the stream
|
||||
descriptor file and its 'validate' method will then be
|
||||
called. If the validation step fails, an exception will be
|
||||
thrown, preventing the stream descriptor from being
|
||||
further processed.
|
||||
|
||||
@param stream_options: A class implementing the IStreamOptions
|
||||
interface. This class's constructor will be passed the
|
||||
sd_info_validator object containing the raw metadata from
|
||||
the stream descriptor file.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._sd_info_validators[stream_type] = sd_info_validator
|
||||
self._stream_options[stream_type] = stream_options
|
||||
|
||||
def add_stream_downloader_factory(self, stream_type, factory):
|
||||
"""
|
||||
Register a stream downloader factory with the StreamDescriptorIdentifier.
|
||||
"""Register a stream downloader factory with the StreamDescriptorIdentifier.
|
||||
|
||||
This is how the StreamDescriptorIdentifier determines what factories may be used to process different stream
|
||||
descriptor files. There must be at least one factory for each type of stream added via
|
||||
"add_stream_info_validator".
|
||||
This is how the StreamDescriptorIdentifier determines what
|
||||
factories may be used to process different stream descriptor
|
||||
files. There must be at least one factory for each type of
|
||||
stream added via "add_stream_info_validator".
|
||||
|
||||
@param stream_type: A string representing the type of stream descriptor which the factory knows how to process.
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor which the factory knows how to process.
|
||||
|
||||
@param factory: An object implementing the IStreamDownloaderFactory interface.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._stream_downloader_factories[stream_type].append(factory)
|
||||
|
||||
def get_info_and_factories_for_sd_file(self, sd_path):
|
||||
def _return_metadata(self, options_validator_factories, source_type, source):
|
||||
validator, options, factories = options_validator_factories
|
||||
m = StreamMetadata(validator, options, factories)
|
||||
m.metadata_source = source_type
|
||||
if source_type == StreamMetadata.FROM_BLOB:
|
||||
m.source_blob_hash = source
|
||||
if source_type == StreamMetadata.FROM_PLAIN:
|
||||
m.source_file = source
|
||||
return m
|
||||
|
||||
def get_metadata_for_sd_file(self, sd_path):
|
||||
sd_reader = PlainStreamDescriptorReader(sd_path)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
|
||||
return d
|
||||
|
||||
def get_info_and_factories_for_sd_blob(self, sd_blob):
|
||||
def get_metadata_for_sd_blob(self, sd_blob):
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
|
||||
return d
|
||||
|
||||
def _get_factories(self, stream_type):
|
||||
assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_downloader_factories[stream_type]
|
||||
|
||||
def _get_validator(self, stream_type):
|
||||
assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._sd_info_validators:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._sd_info_validators[stream_type]
|
||||
|
||||
def _return_info_and_factories(self, sd_info):
|
||||
assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.'
|
||||
def _get_options(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_options[stream_type]
|
||||
|
||||
def _return_options_and_validator_and_factories(self, sd_info):
|
||||
if not 'stream_type' in sd_info:
|
||||
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
|
||||
stream_type = sd_info['stream_type']
|
||||
factories = self._get_factories(stream_type)
|
||||
validator = self._get_validator(stream_type)(sd_info)
|
||||
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
|
||||
|
||||
d = validator.validate()
|
||||
|
||||
d.addCallback(lambda _: (validator, factories))
|
||||
def get_options():
|
||||
options = self._get_options(stream_type)
|
||||
return validator, options, factories
|
||||
|
||||
d.addCallback(lambda _: get_options())
|
||||
return d
|
||||
|
||||
|
||||
@@ -190,6 +248,10 @@ def download_sd_blob(session, blob_hash, payment_rate_manager):
|
||||
|
||||
@return: An object of type HashBlob
|
||||
"""
|
||||
downloader = StandaloneBlobDownloader(blob_hash, session.blob_manager, session.peer_finder,
|
||||
session.rate_limiter, payment_rate_manager, session.wallet)
|
||||
return downloader.download()
|
||||
downloader = StandaloneBlobDownloader(blob_hash,
|
||||
session.blob_manager,
|
||||
session.peer_finder,
|
||||
session.rate_limiter,
|
||||
payment_rate_manager,
|
||||
session.wallet)
|
||||
return downloader.download()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -4,4 +4,4 @@ Classes and functions which can be used by any application wishing to make use o
|
||||
This includes classes for connecting to other peers and downloading blobs from them, listening for
|
||||
connections from peers and responding to their requests, managing locally stored blobs, sending
|
||||
and receiving payments, and locating peers in the DHT.
|
||||
"""
|
||||
"""
|
||||
@@ -1,12 +1,41 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from decimal import Decimal
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
|
||||
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
|
||||
from lbrynet.interfaces import IRequestCreator
|
||||
from lbrynet.core.Offer import Offer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_points(num_bytes, rate):
|
||||
if isinstance(rate, float):
|
||||
return 1.0 * num_bytes * rate / 2**20
|
||||
elif isinstance(rate, Decimal):
|
||||
return 1.0 * num_bytes * float(rate) / 2**20
|
||||
else:
|
||||
raise Exception("Unknown rate type")
|
||||
|
||||
|
||||
def cache(fn):
|
||||
"""Caches the function call for each instance"""
|
||||
attr = '__{}_value'.format(fn.__name__)
|
||||
|
||||
def helper(self):
|
||||
if not hasattr(self, attr):
|
||||
value = fn(self)
|
||||
setattr(self, attr, value)
|
||||
return getattr(self, attr)
|
||||
return helper
|
||||
|
||||
|
||||
class BlobRequester(object):
|
||||
@@ -17,64 +46,23 @@ class BlobRequester(object):
|
||||
self.peer_finder = peer_finder
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.download_manager = download_manager
|
||||
self._download_manager = download_manager
|
||||
self._peers = defaultdict(int) # {Peer: score}
|
||||
self._available_blobs = defaultdict(list) # {Peer: [blob_hash]}
|
||||
self._unavailable_blobs = defaultdict(list) # {Peer: [blob_hash]}}
|
||||
self._protocol_prices = {} # {ClientProtocol: price}
|
||||
self._protocol_offers = {}
|
||||
self._price_disagreements = [] # [Peer]
|
||||
self._protocol_tries = {}
|
||||
self._maxed_out_peers = []
|
||||
self._incompatible_peers = []
|
||||
|
||||
######## IRequestCreator #########
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
sent_request = False
|
||||
if self._blobs_to_download() and self._should_send_request_to(peer):
|
||||
a_r = self._get_availability_request(peer)
|
||||
d_r = self._get_download_request(peer)
|
||||
p_r = None
|
||||
|
||||
if a_r is not None or d_r is not None:
|
||||
p_r = self._get_price_request(peer, protocol)
|
||||
|
||||
if a_r is not None:
|
||||
d1 = protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, peer, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request", peer)
|
||||
sent_request = True
|
||||
if d_r is not None:
|
||||
reserved_points = self._reserve_points(peer, protocol, d_r.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
d_r.finished_deferred.addCallbacks(self._download_succeeded, self._download_failed,
|
||||
callbackArgs=(peer, d_r.blob),
|
||||
errbackArgs=(peer,))
|
||||
d_r.finished_deferred.addBoth(self._pay_or_cancel_payment, protocol, reserved_points, d_r.blob)
|
||||
d_r.finished_deferred.addErrback(self._handle_download_error, peer, d_r.blob)
|
||||
|
||||
d2 = protocol.add_blob_request(d_r)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d2.addCallback(self._handle_incoming_blob, peer, d_r)
|
||||
d2.addErrback(self._request_failed, "download request", peer)
|
||||
|
||||
sent_request = True
|
||||
else:
|
||||
d_r.cancel(InsufficientFundsError())
|
||||
return defer.fail(InsufficientFundsError())
|
||||
if sent_request is True:
|
||||
if p_r is not None:
|
||||
d3 = protocol.add_request(p_r)
|
||||
d3.addCallback(self._handle_price_response, peer, p_r, protocol)
|
||||
d3.addErrback(self._request_failed, "price request", peer)
|
||||
return defer.succeed(sent_request)
|
||||
"""Makes an availability request, download request and price request"""
|
||||
if not self.should_send_next_request(peer):
|
||||
return defer.succeed(False)
|
||||
return self._send_next_request(peer, protocol)
|
||||
|
||||
def get_new_peers(self):
|
||||
d = self._get_hash_for_peer_search()
|
||||
@@ -82,36 +70,35 @@ class BlobRequester(object):
|
||||
return d
|
||||
|
||||
######### internal calls #########
|
||||
def should_send_next_request(self, peer):
|
||||
return (
|
||||
self._blobs_to_download() and
|
||||
self._should_send_request_to(peer)
|
||||
)
|
||||
|
||||
def _download_succeeded(self, arg, peer, blob):
|
||||
logging.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
|
||||
self._update_local_score(peer, 5.0)
|
||||
peer.update_stats('blobs_downloaded', 1)
|
||||
peer.update_score(5.0)
|
||||
self.blob_manager.blob_completed(blob)
|
||||
return arg
|
||||
def _send_next_request(self, peer, protocol):
|
||||
log.debug('Sending a blob request for %s and %s', peer, protocol)
|
||||
availability = AvailabilityRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
download = DownloadRequest(self, peer, protocol, self.payment_rate_manager, self.wallet)
|
||||
price = PriceRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
|
||||
def _download_failed(self, reason, peer):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self._update_local_score(peer, -10.0)
|
||||
return reason
|
||||
sent_request = False
|
||||
if availability.can_make_request():
|
||||
availability.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if price.can_make_request():
|
||||
# TODO: document why a PriceRequest is only made if an
|
||||
# Availability or Download request was made
|
||||
price.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if download.can_make_request():
|
||||
try:
|
||||
download.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
except InsufficientFundsError as err:
|
||||
return defer.fail(err)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, protocol, reserved_points, blob):
|
||||
if blob.length != 0 and (not isinstance(arg, Failure) or arg.check(DownloadCanceledError)):
|
||||
self._pay_peer(protocol, blob.length, reserved_points)
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _handle_download_error(self, err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
logging.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
return defer.succeed(sent_request)
|
||||
|
||||
def _get_hash_for_peer_search(self):
|
||||
r = None
|
||||
@@ -123,7 +110,7 @@ class BlobRequester(object):
|
||||
else:
|
||||
blob_hash = blobs_without_sources[0].blob_hash
|
||||
r = blob_hash
|
||||
logging.debug("Blob requester peer search response: %s", str(r))
|
||||
log.debug("Blob requester peer search response: %s", str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -134,9 +121,18 @@ class BlobRequester(object):
|
||||
|
||||
def choose_best_peers(peers):
|
||||
bad_peers = self._get_bad_peers()
|
||||
return [p for p in peers if not p in bad_peers]
|
||||
without_bad_peers = [p for p in peers if not p in bad_peers]
|
||||
without_maxed_out_peers = [
|
||||
p for p in without_bad_peers if p not in self._maxed_out_peers]
|
||||
return without_maxed_out_peers
|
||||
|
||||
d.addCallback(choose_best_peers)
|
||||
|
||||
def lookup_failed(err):
|
||||
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
|
||||
return []
|
||||
|
||||
d.addErrback(lookup_failed)
|
||||
return d
|
||||
|
||||
def _should_send_request_to(self, peer):
|
||||
@@ -163,145 +159,411 @@ class BlobRequester(object):
|
||||
return False
|
||||
|
||||
def _blobs_to_download(self):
|
||||
needed_blobs = self.download_manager.needed_blobs()
|
||||
needed_blobs = self._download_manager.needed_blobs()
|
||||
return sorted(needed_blobs, key=lambda b: b.is_downloading())
|
||||
|
||||
def _blobs_without_sources(self):
|
||||
return [b for b in self.download_manager.needed_blobs() if not self._hash_available(b.blob_hash)]
|
||||
|
||||
def _get_availability_request(self, peer):
|
||||
all_needed = [b.blob_hash for b in self._blobs_to_download() if not b.blob_hash in self._available_blobs[peer]]
|
||||
# sort them so that the peer will be asked first for blobs it hasn't said it doesn't have
|
||||
to_request = sorted(all_needed, key=lambda b: b in self._unavailable_blobs[peer])[:20]
|
||||
if to_request:
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
return None
|
||||
|
||||
def _get_download_request(self, peer):
|
||||
request = None
|
||||
to_download = [b for b in self._blobs_to_download() if self._hash_available_on(b.blob_hash, peer)]
|
||||
while to_download and request is None:
|
||||
blob_to_download = to_download[0]
|
||||
to_download = to_download[1:]
|
||||
if not blob_to_download.is_validated():
|
||||
d, write_func, cancel_func = blob_to_download.open_for_writing(peer)
|
||||
|
||||
def counting_write_func(data):
|
||||
peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return write_func(data)
|
||||
|
||||
if d is not None:
|
||||
|
||||
request_dict = {'requested_blob': blob_to_download.blob_hash}
|
||||
response_identifier = 'incoming_blob'
|
||||
|
||||
request = ClientBlobRequest(request_dict, response_identifier, counting_write_func, d,
|
||||
cancel_func, blob_to_download)
|
||||
|
||||
logging.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
|
||||
return request
|
||||
return [
|
||||
b for b in self._download_manager.needed_blobs()
|
||||
if not self._hash_available(b.blob_hash)
|
||||
]
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
if protocol in self._protocol_prices:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_price_request(self, peer, protocol):
|
||||
request = None
|
||||
if not protocol in self._protocol_prices:
|
||||
self._protocol_prices[protocol] = self.payment_rate_manager.get_rate_blob_data(peer)
|
||||
request_dict = {'blob_data_payment_rate': self._protocol_prices[protocol]}
|
||||
request = ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
|
||||
|
||||
class RequestHelper(object):
|
||||
def __init__(self, requestor, peer, protocol, payment_rate_manager):
|
||||
self.requestor = requestor
|
||||
self.peer = peer
|
||||
self.protocol = protocol
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
|
||||
@property
|
||||
def protocol_prices(self):
|
||||
return self.requestor._protocol_prices
|
||||
|
||||
@property
|
||||
def protocol_offers(self):
|
||||
return self.requestor._protocol_offers
|
||||
|
||||
@property
|
||||
def available_blobs(self):
|
||||
return self.requestor._available_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def unavailable_blobs(self):
|
||||
return self.requestor._unavailable_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def maxed_out_peers(self):
|
||||
return self.requestor._maxed_out_peers
|
||||
|
||||
def update_local_score(self, score):
|
||||
self.requestor._update_local_score(self.peer, score)
|
||||
|
||||
def _request_failed(self, reason, request_type):
|
||||
if reason.check(RequestCanceledError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self.requestor._incompatible_peers.append(self.peer)
|
||||
log.warning("A request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
request_type, reason.getErrorMessage(), reason.type)
|
||||
self.update_local_score(-10.0)
|
||||
if isinstance(reason, InvalidResponseError) or isinstance(reason, NoResponseError):
|
||||
self.peer.update_score(-10.0)
|
||||
else:
|
||||
self.peer.update_score(-2.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
|
||||
def get_rate(self):
|
||||
if self.payment_rate_manager.price_limit_reached(self.peer):
|
||||
if self.peer not in self.maxed_out_peers:
|
||||
self.maxed_out_peers.append(self.peer)
|
||||
return None
|
||||
rate = self.protocol_prices.get(self.protocol)
|
||||
if rate is None:
|
||||
if self.peer in self.payment_rate_manager.strategy.pending_sent_offers:
|
||||
pending = self.payment_rate_manager.strategy.pending_sent_offers[self.peer]
|
||||
if not pending.is_too_low and not pending.is_accepted:
|
||||
return pending.rate
|
||||
rate = self.payment_rate_manager.get_rate_blob_data(self.peer, self.available_blobs)
|
||||
return rate
|
||||
|
||||
|
||||
def _handle_incoming_blob(response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not type(response_dict[request.response_identifier]) == dict:
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
type(response_dict[request.response_identifier]))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if not 'blob_hash' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError(
|
||||
"Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob'])
|
||||
)
|
||||
if not 'length' in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
|
||||
|
||||
def _handle_download_error(err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
log.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
|
||||
|
||||
class AvailabilityRequest(RequestHelper):
|
||||
"""Ask a peer what blobs it has available.
|
||||
|
||||
Results are saved in `_available_blobs` and `_unavailable_blobs`
|
||||
on the parent BlobRequester.
|
||||
"""
|
||||
def can_make_request(self):
|
||||
return self.get_top_needed_blobs()
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
to_request = self.get_top_needed_blobs()
|
||||
if not to_request:
|
||||
raise Exception('Unable to make a request without available blobs')
|
||||
return self._make_request(to_request)
|
||||
|
||||
@cache
|
||||
def get_top_needed_blobs(self, limit=20):
|
||||
all_needed = [
|
||||
b.blob_hash for b in self.requestor._blobs_to_download()
|
||||
if not self.is_available(b)
|
||||
]
|
||||
# sort them so that the peer will be asked first for blobs it
|
||||
# hasn't said it doesn't have
|
||||
sorted_needed = sorted(
|
||||
all_needed,
|
||||
key=lambda b: b in self.unavailable_blobs
|
||||
)
|
||||
return sorted_needed[:limit]
|
||||
|
||||
def is_available(self, blob):
|
||||
return blob.blob_hash in self.available_blobs
|
||||
|
||||
def _make_request(self, to_request):
|
||||
log.debug('Requesting blobs: %s', to_request)
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
def _handle_request(self, a_r):
|
||||
log.debug('making an availability request')
|
||||
d1 = self.protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request")
|
||||
|
||||
def _reserve_points(self, peer, protocol, max_bytes):
|
||||
assert protocol in self._protocol_prices
|
||||
points_to_reserve = 1.0 * max_bytes * self._protocol_prices[protocol] / 2**20
|
||||
return self.wallet.reserve_points(peer, points_to_reserve)
|
||||
def _handle_availability(self, response_dict, request):
|
||||
assert request.response_identifier == 'available_blobs'
|
||||
if 'available_blobs' not in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
log.debug("Received a response to the availability request")
|
||||
# save available blobs
|
||||
blob_hashes = response_dict['available_blobs']
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
self.process_available_blob_hash(blob_hash, request)
|
||||
# everything left in the request is missing
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self.unavailable_blobs.append(blob_hash)
|
||||
return True
|
||||
|
||||
def _pay_peer(self, protocol, num_bytes, reserved_points):
|
||||
def process_available_blob_hash(self, blob_hash, request):
|
||||
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self.available_blobs.append(blob_hash)
|
||||
self.remove_from_unavailable_blobs(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
|
||||
def remove_from_unavailable_blobs(self, blob_hash):
|
||||
if blob_hash in self.unavailable_blobs:
|
||||
self.unavailable_blobs.remove(blob_hash)
|
||||
|
||||
|
||||
class PriceRequest(RequestHelper):
|
||||
"""Ask a peer if a certain price is acceptable"""
|
||||
def can_make_request(self):
|
||||
if len(self.available_blobs) and not self.protocol in self.protocol_prices:
|
||||
return self.get_rate() is not None
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_price_request()
|
||||
self._handle_price_request(request)
|
||||
|
||||
def _get_price_request(self):
|
||||
rate = self.get_rate()
|
||||
if rate is None:
|
||||
log.debug("No blobs to request from %s", self.peer)
|
||||
raise Exception('Cannot make a price request without a payment rate')
|
||||
log.debug("Offer rate %s to %s for %i blobs", rate, self.peer, len(self.available_blobs))
|
||||
|
||||
request_dict = {'blob_data_payment_rate': rate}
|
||||
assert self.protocol not in self.protocol_offers
|
||||
self.protocol_offers[self.protocol] = rate
|
||||
return ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
|
||||
def _handle_price_request(self, price_request):
|
||||
d = self.protocol.add_request(price_request)
|
||||
d.addCallback(self._handle_price_response, price_request)
|
||||
d.addErrback(self._request_failed, "price request")
|
||||
|
||||
def _handle_price_response(self, response_dict, request):
|
||||
assert request.response_identifier == 'blob_data_payment_rate'
|
||||
if 'blob_data_payment_rate' not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
offer_value = self.protocol_offers.pop(self.protocol)
|
||||
offer = Offer(offer_value)
|
||||
offer.handle(response_dict['blob_data_payment_rate'])
|
||||
self.payment_rate_manager.record_offer_reply(self.peer, offer)
|
||||
if offer.is_accepted:
|
||||
log.info("Offered rate %f/mb accepted by %s", offer.rate, self.peer.host)
|
||||
self.protocol_prices[self.protocol] = offer.rate
|
||||
return True
|
||||
elif offer.is_too_low:
|
||||
log.debug("Offered rate %f/mb rejected by %s", offer.rate, self.peer.host)
|
||||
return not self.payment_rate_manager.price_limit_reached(self.peer)
|
||||
else:
|
||||
log.warning("Price disagreement")
|
||||
self.requestor._price_disagreements.append(self.peer)
|
||||
return False
|
||||
|
||||
|
||||
class DownloadRequest(RequestHelper):
|
||||
"""Choose a blob and download it from a peer and also pay the peer for the data."""
|
||||
def __init__(self, requester, peer, protocol, payment_rate_manager, wallet):
|
||||
RequestHelper.__init__(self, requester, peer, protocol, payment_rate_manager)
|
||||
self.wallet = wallet
|
||||
|
||||
def can_make_request(self):
|
||||
if self.protocol in self.protocol_prices:
|
||||
return self.get_blob_details()
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_download_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
blob_details = self.get_blob_details()
|
||||
if not blob_details:
|
||||
raise Exception('No blobs available to download')
|
||||
return self._make_request(blob_details)
|
||||
|
||||
@cache
|
||||
def get_blob_details(self):
|
||||
"""Open a blob for writing and return the details.
|
||||
|
||||
If no blob can be opened, returns None.
|
||||
"""
|
||||
to_download = self.get_available_blobs()
|
||||
return self.find_blob(to_download)
|
||||
|
||||
def get_available_blobs(self):
|
||||
available_blobs = [
|
||||
b for b in self.requestor._blobs_to_download()
|
||||
if self.requestor._hash_available_on(b.blob_hash, self.peer)
|
||||
]
|
||||
log.debug('available blobs: %s', available_blobs)
|
||||
return available_blobs
|
||||
|
||||
def find_blob(self, to_download):
|
||||
"""Return the first blob in `to_download` that is successfully opened for write."""
|
||||
for blob in to_download:
|
||||
if blob.is_validated():
|
||||
log.debug('Skipping blob %s as its already validated', blob)
|
||||
continue
|
||||
d, write_func, cancel_func = blob.open_for_writing(self.peer)
|
||||
if d is not None:
|
||||
return BlobDownloadDetails(blob, d, write_func, cancel_func, self.peer)
|
||||
log.debug('Skipping blob %s as there was an issue opening it for writing', blob)
|
||||
return None
|
||||
|
||||
def _make_request(self, blob_details):
|
||||
blob = blob_details.blob
|
||||
request = ClientBlobRequest(
|
||||
{'requested_blob': blob.blob_hash},
|
||||
'incoming_blob',
|
||||
blob_details.counting_write_func,
|
||||
blob_details.deferred,
|
||||
blob_details.cancel_func,
|
||||
blob
|
||||
)
|
||||
log.debug("Requesting blob %s from %s", blob.blob_hash, self.peer)
|
||||
return request
|
||||
|
||||
def _handle_download_request(self, client_blob_request):
|
||||
reserved_points = self.reserve_funds_or_cancel(client_blob_request)
|
||||
self.add_callbacks_to_download_request(client_blob_request, reserved_points)
|
||||
self.create_add_blob_request(client_blob_request)
|
||||
|
||||
def reserve_funds_or_cancel(self, client_blob_request):
|
||||
reserved_points = self._reserve_points(client_blob_request.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
return reserved_points
|
||||
client_blob_request.cancel(InsufficientFundsError())
|
||||
client_blob_request.finished_deferred.addErrback(lambda _: True)
|
||||
raise InsufficientFundsError()
|
||||
|
||||
def add_callbacks_to_download_request(self, client_blob_request, reserved_points):
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
client_blob_request.finished_deferred.addCallbacks(
|
||||
self._download_succeeded,
|
||||
self._download_failed,
|
||||
callbackArgs=(client_blob_request.blob,),
|
||||
)
|
||||
client_blob_request.finished_deferred.addBoth(
|
||||
self._pay_or_cancel_payment, reserved_points, client_blob_request.blob)
|
||||
client_blob_request.finished_deferred.addErrback(
|
||||
_handle_download_error, self.peer, client_blob_request.blob)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, reserved_points, blob):
|
||||
if self._can_pay_peer(blob, arg):
|
||||
self._pay_peer(blob.length, reserved_points)
|
||||
d = self.requestor.blob_manager.add_blob_to_download_history(
|
||||
str(blob), str(self.peer.host), float(self.protocol_prices[self.protocol]))
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _can_pay_peer(self, blob, arg):
|
||||
return (
|
||||
blob.length != 0 and
|
||||
(not isinstance(arg, Failure) or arg.check(DownloadCanceledError))
|
||||
)
|
||||
|
||||
def _pay_peer(self, num_bytes, reserved_points):
|
||||
assert num_bytes != 0
|
||||
assert protocol in self._protocol_prices
|
||||
point_amount = 1.0 * num_bytes * self._protocol_prices[protocol] / 2**20
|
||||
rate = self.get_rate()
|
||||
point_amount = get_points(num_bytes, rate)
|
||||
self.wallet.send_points(reserved_points, point_amount)
|
||||
self.payment_rate_manager.record_points_paid(point_amount)
|
||||
|
||||
def _cancel_points(self, reserved_points):
|
||||
self.wallet.cancel_point_reservation(reserved_points)
|
||||
|
||||
def _handle_availability(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
logging.debug("Received a response to the availability request")
|
||||
blob_hashes = response_dict[request.response_identifier]
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
logging.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self._available_blobs[peer].append(blob_hash)
|
||||
if blob_hash in self._unavailable_blobs[peer]:
|
||||
self._unavailable_blobs[peer].remove(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self._unavailable_blobs[peer].append(blob_hash)
|
||||
return True
|
||||
def create_add_blob_request(self, client_blob_request):
|
||||
d = self.protocol.add_blob_request(client_blob_request)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d.addCallback(_handle_incoming_blob, self.peer, client_blob_request)
|
||||
d.addErrback(self._request_failed, "download request")
|
||||
|
||||
def _handle_incoming_blob(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not type(response_dict[request.response_identifier]) == dict:
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
(type(response_dict[request.response_identifier]),))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if not 'blob_hash' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError("Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob']))
|
||||
if not 'length' in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
def _reserve_points(self, num_bytes):
|
||||
# jobevers: there was an assertion here, but I don't think it
|
||||
# was a valid assertion to make. It is possible for a rate to
|
||||
# not yet been set for this protocol or for it to have been
|
||||
# removed so instead I switched it to check if a rate has been set
|
||||
# and calculate it if it has not
|
||||
rate = self.get_rate()
|
||||
points_to_reserve = get_points(num_bytes, rate)
|
||||
return self.wallet.reserve_points(self.peer, points_to_reserve)
|
||||
|
||||
def _handle_price_response(self, response_dict, peer, request, protocol):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
assert protocol in self._protocol_prices
|
||||
response = response_dict[request.response_identifier]
|
||||
if response == "RATE_ACCEPTED":
|
||||
return True
|
||||
else:
|
||||
del self._protocol_prices[protocol]
|
||||
self._price_disagreements.append(peer)
|
||||
return True
|
||||
def _download_succeeded(self, arg, blob):
|
||||
log.info("Blob %s has been successfully downloaded from %s", blob, self.peer)
|
||||
self.update_local_score(5.0)
|
||||
self.peer.update_stats('blobs_downloaded', 1)
|
||||
self.peer.update_score(5.0)
|
||||
self.requestor.blob_manager.blob_completed(blob)
|
||||
return arg
|
||||
|
||||
def _request_failed(self, reason, request_type, peer):
|
||||
if reason.check(RequestCanceledError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
str(request_type), reason.getErrorMessage(), reason.type)
|
||||
self._update_local_score(peer, -10.0)
|
||||
if isinstance(reason, InvalidResponseError):
|
||||
peer.update_score(-10.0)
|
||||
else:
|
||||
peer.update_score(-2.0)
|
||||
return reason
|
||||
def _download_failed(self, reason):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self.update_local_score(-10.0)
|
||||
return reason
|
||||
|
||||
|
||||
class BlobDownloadDetails(object):
|
||||
"""Contains the information needed to make a ClientBlobRequest from an open blob"""
|
||||
def __init__(self, blob, deferred, write_func, cancel_func, peer):
|
||||
self.blob = blob
|
||||
self.deferred = deferred
|
||||
self.write_func = write_func
|
||||
self.cancel_func = cancel_func
|
||||
self.peer = peer
|
||||
|
||||
def counting_write_func(self, data):
|
||||
self.peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return self.write_func(data)
|
||||
@@ -1,9 +1,12 @@
|
||||
import json
|
||||
import logging
|
||||
from twisted.internet import error, defer, reactor
|
||||
from decimal import Decimal
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.protocols.policies import TimeoutMixin
|
||||
from twisted.python import failure
|
||||
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
|
||||
from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
@@ -11,12 +14,22 @@ from lbrynet.interfaces import IRequestSender, IRateLimited
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
class ClientProtocol(Protocol):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_decimal(obj):
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
raise TypeError(repr(obj) + " is not JSON serializable")
|
||||
|
||||
|
||||
class ClientProtocol(Protocol, TimeoutMixin):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
######### Protocol #########
|
||||
PROTOCOL_TIMEOUT = 30
|
||||
|
||||
def connectionMade(self):
|
||||
log.debug("Connection made to %s", self.factory.peer)
|
||||
self._connection_manager = self.factory.connection_manager
|
||||
self._rate_limiter = self.factory.rate_limiter
|
||||
self.peer = self.factory.peer
|
||||
@@ -27,19 +40,23 @@ class ClientProtocol(Protocol):
|
||||
self._next_request = {}
|
||||
self.connection_closed = False
|
||||
self.connection_closing = False
|
||||
|
||||
# This needs to be set for TimeoutMixin
|
||||
self.callLater = utils.call_later
|
||||
self.peer.report_up()
|
||||
|
||||
self._ask_for_request()
|
||||
|
||||
def dataReceived(self, data):
|
||||
log.debug("Data receieved from %s", self.peer)
|
||||
self.setTimeout(None)
|
||||
self._rate_limiter.report_dl_bytes(len(data))
|
||||
if self._downloading_blob is True:
|
||||
self._blob_download_request.write(data)
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > MAX_RESPONSE_SIZE:
|
||||
logging.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
|
||||
log.warning("Response is too large from %s. Size %s",
|
||||
self.peer, len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
response, extra_data = self._get_valid_response(self._response_buff)
|
||||
if response is not None:
|
||||
@@ -48,29 +65,34 @@ class ClientProtocol(Protocol):
|
||||
if self._downloading_blob is True and len(extra_data) != 0:
|
||||
self._blob_download_request.write(extra_data)
|
||||
|
||||
def timeoutConnection(self):
|
||||
log.info("Connection timed out to %s", self.peer)
|
||||
self.peer.report_down()
|
||||
self.transport.abortConnection()
|
||||
|
||||
def connectionLost(self, reason):
|
||||
log.debug("Connection lost to %s: %s", self.peer, reason)
|
||||
self.setTimeout(None)
|
||||
self.connection_closed = True
|
||||
if reason.check(error.ConnectionDone):
|
||||
err = failure.Failure(ConnectionClosedBeforeResponseError())
|
||||
else:
|
||||
err = reason
|
||||
#if self._response_deferreds:
|
||||
# logging.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
for key, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
if self._blob_download_request is not None:
|
||||
self._blob_download_request.cancel(err)
|
||||
self._connection_manager.protocol_disconnected(self.peer, self)
|
||||
self.factory.connection_was_made_deferred.callback(True)
|
||||
|
||||
######### IRequestSender #########
|
||||
|
||||
def add_request(self, request):
|
||||
if request.response_identifier in self._response_deferreds:
|
||||
return defer.fail(failure.Failure(ValueError("There is already a request for that response active")))
|
||||
raise ValueError("There is already a request for that response active")
|
||||
self._next_request.update(request.request_dict)
|
||||
d = defer.Deferred()
|
||||
logging.debug("Adding a request. Request: %s", str(request))
|
||||
log.debug("Adding a request for %s. Request: %s", self.peer, request.request_dict)
|
||||
self._response_deferreds[request.response_identifier] = d
|
||||
return d
|
||||
|
||||
@@ -83,7 +105,7 @@ class ClientProtocol(Protocol):
|
||||
blob_request.finished_deferred.addErrback(self._handle_response_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(failure.Failure(ValueError("There is already a blob download request active")))
|
||||
raise ValueError("There is already a blob download request active")
|
||||
|
||||
def cancel_requests(self):
|
||||
self.connection_closing = True
|
||||
@@ -102,12 +124,12 @@ class ClientProtocol(Protocol):
|
||||
######### Internal request handling #########
|
||||
|
||||
def _handle_request_error(self, err):
|
||||
logging.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
log.error(
|
||||
"An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
self.peer, err.getTraceback())
|
||||
self.transport.loseConnection()
|
||||
|
||||
def _ask_for_request(self):
|
||||
|
||||
if self.connection_closed is True or self.connection_closing is True:
|
||||
return
|
||||
|
||||
@@ -117,17 +139,20 @@ class ClientProtocol(Protocol):
|
||||
self._send_request_message(request_msg)
|
||||
else:
|
||||
# The connection manager has indicated that this connection should be terminated
|
||||
logging.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
log.info(
|
||||
"Closing the connection to %s due to having no further requests to send",
|
||||
self.peer)
|
||||
self.peer.report_success()
|
||||
self.transport.loseConnection()
|
||||
|
||||
d = self._connection_manager.get_next_request(self.peer, self)
|
||||
d.addCallback(send_request_or_close)
|
||||
d.addErrback(self._handle_request_error)
|
||||
|
||||
def _send_request_message(self, request_msg):
|
||||
self.setTimeout(self.PROTOCOL_TIMEOUT)
|
||||
# TODO: compare this message to the last one. If they're the same,
|
||||
# TODO: incrementally delay this message.
|
||||
m = json.dumps(request_msg)
|
||||
m = json.dumps(request_msg, default=encode_decimal)
|
||||
self.transport.write(m)
|
||||
|
||||
def _get_valid_response(self, response_msg):
|
||||
@@ -151,20 +176,24 @@ class ClientProtocol(Protocol):
|
||||
|
||||
def _handle_response_error(self, err):
|
||||
# If an error gets to this point, log it and kill the connection.
|
||||
if not err.check(MisbehavingPeerError, ConnectionClosedBeforeResponseError, DownloadCanceledError,
|
||||
RequestCanceledError):
|
||||
logging.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
expected_errors = (MisbehavingPeerError, ConnectionClosedBeforeResponseError,
|
||||
DownloadCanceledError, RequestCanceledError)
|
||||
if not err.check(expected_errors):
|
||||
log.error("The connection to %s is closing due to an unexpected error: %s",
|
||||
self.peer, err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError):
|
||||
self.transport.loseConnection()
|
||||
# The connection manager is closing the connection, so
|
||||
# there's no need to do it here.
|
||||
return err
|
||||
|
||||
def _handle_response(self, response):
|
||||
ds = []
|
||||
logging.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
log.debug(
|
||||
"Handling a response from %s. Expected responses: %s. Actual responses: %s",
|
||||
self.peer, self._response_deferreds.keys(), response.keys())
|
||||
for key, val in response.items():
|
||||
if key in self._response_deferreds:
|
||||
d = self._response_deferreds[key]
|
||||
del self._response_deferreds[key]
|
||||
d = self._response_deferreds.pop(key)
|
||||
d.callback({key: val})
|
||||
ds.append(d)
|
||||
for k, d in self._response_deferreds.items():
|
||||
@@ -178,12 +207,31 @@ class ClientProtocol(Protocol):
|
||||
d.addErrback(self._handle_response_error)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
# TODO: are we sure we want to consume errors here
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
|
||||
dl.addCallback(lambda _: self._ask_for_request())
|
||||
def get_next_request(results):
|
||||
failed = False
|
||||
for success, result in results:
|
||||
if success is False:
|
||||
failed = True
|
||||
if not isinstance(result.value, DownloadCanceledError):
|
||||
log.info(result.value)
|
||||
log.info("The connection to %s is closing due to an error: %s",
|
||||
self.peer, result.getTraceback())
|
||||
|
||||
self.peer.report_down()
|
||||
if failed is False:
|
||||
log.debug("Asking for another request from %s", self.peer)
|
||||
self._ask_for_request()
|
||||
else:
|
||||
log.debug("Not asking for another request from %s", self.peer)
|
||||
self.transport.loseConnection()
|
||||
|
||||
dl.addCallback(get_next_request)
|
||||
|
||||
def _downloading_finished(self, arg):
|
||||
logging.debug("The blob has finished downloading")
|
||||
log.debug("The blob has finished downloading from %s", self.peer)
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return arg
|
||||
@@ -194,10 +242,8 @@ class ClientProtocol(Protocol):
|
||||
# TODO: always be this way. it's done this way now because the client has no other way
|
||||
# TODO: of telling the server it wants the download to stop. It would be great if the
|
||||
# TODO: protocol had such a mechanism.
|
||||
logging.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
#self.transport.loseConnection()
|
||||
#return True
|
||||
log.debug("Closing the connection to %s because the download of blob %s was canceled",
|
||||
self.peer, self._blob_download_request.blob)
|
||||
return err
|
||||
|
||||
######### IRateLimited #########
|
||||
@@ -223,13 +269,18 @@ class ClientProtocolFactory(ClientFactory):
|
||||
self.rate_limiter = rate_limiter
|
||||
self.connection_manager = connection_manager
|
||||
self.p = None
|
||||
# This defer fires and returns True when connection was
|
||||
# made and completed, or fires and returns False if
|
||||
# connection failed
|
||||
self.connection_was_made_deferred = defer.Deferred()
|
||||
|
||||
def clientConnectionFailed(self, connector, reason):
|
||||
log.debug("Connection failed to %s: %s", self.peer, reason)
|
||||
self.peer.report_down()
|
||||
self.connection_manager.protocol_disconnected(self.peer, connector)
|
||||
self.connection_was_made_deferred.callback(False)
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = self.protocol()
|
||||
p.factory = self
|
||||
self.p = p
|
||||
return p
|
||||
return p
|
||||
@@ -1,4 +1,4 @@
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
class ClientRequest(object):
|
||||
@@ -17,11 +17,11 @@ class ClientBlobRequest(ClientPaidRequest):
|
||||
def __init__(self, request_dict, response_identifier, write_func, finished_deferred,
|
||||
cancel_func, blob):
|
||||
if blob.length is None:
|
||||
max_pay_units = BLOB_SIZE
|
||||
max_pay_units = conf.settings['BLOB_SIZE']
|
||||
else:
|
||||
max_pay_units = blob.length
|
||||
ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units)
|
||||
self.write = write_func
|
||||
self.finished_deferred = finished_deferred
|
||||
self.cancel = cancel_func
|
||||
self.blob = blob
|
||||
self.blob = blob
|
||||
@@ -1,177 +1,227 @@
|
||||
import random
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import defer, reactor
|
||||
from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
from lbrynet.conf import MAX_CONNECTIONS_PER_STREAM
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnectionHandler(object):
|
||||
def __init__(self, request_creators, factory):
|
||||
self.request_creators = request_creators
|
||||
self.factory = factory
|
||||
self.connection = None
|
||||
|
||||
|
||||
class ConnectionManager(object):
|
||||
implements(interfaces.IConnectionManager)
|
||||
MANAGE_CALL_INTERVAL_SEC = 5
|
||||
TCP_CONNECT_TIMEOUT = 15
|
||||
|
||||
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
|
||||
def __init__(self, downloader, rate_limiter,
|
||||
primary_request_creators, secondary_request_creators):
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
self.primary_request_creators = primary_request_creators
|
||||
self.secondary_request_creators = secondary_request_creators
|
||||
self.peer_connections = {} # {Peer: {'connection': connection,
|
||||
# 'request_creators': [IRequestCreator if using this connection]}}
|
||||
self.connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self.next_manage_call = None
|
||||
self._primary_request_creators = primary_request_creators
|
||||
self._secondary_request_creators = secondary_request_creators
|
||||
self._peer_connections = {} # {Peer: PeerConnectionHandler}
|
||||
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self._next_manage_call = None
|
||||
# a deferred that gets fired when a _manage call is set
|
||||
self._manage_deferred = None
|
||||
self.stopped = True
|
||||
log.info("%s initialized", self._get_log_name())
|
||||
|
||||
# this identifies what the connection manager is for,
|
||||
# used for logging purposes only
|
||||
def _get_log_name(self):
|
||||
out = 'Connection Manager Unknown'
|
||||
if hasattr(self.downloader, 'stream_name'):
|
||||
out = 'Connection Manager '+self.downloader.stream_name
|
||||
elif hasattr(self.downloader, 'blob_hash'):
|
||||
out = 'Connection Manager '+self.downloader.blob_hash
|
||||
return out
|
||||
|
||||
def _start(self):
|
||||
self.stopped = False
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
|
||||
def start(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = reactor.callLater(0, self._manage)
|
||||
log.debug("%s starting", self._get_log_name())
|
||||
self._start()
|
||||
self._next_manage_call = utils.call_later(0, self.manage)
|
||||
return defer.succeed(True)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
closing_deferreds = []
|
||||
for peer in self.peer_connections.keys():
|
||||
log.debug("%s stopping", self._get_log_name())
|
||||
self.stopped = True
|
||||
# wait for the current manage call to finish
|
||||
if self._manage_deferred:
|
||||
yield self._manage_deferred
|
||||
# in case we stopped between manage calls, cancel the next one
|
||||
if self._next_manage_call and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
yield self._close_peers()
|
||||
|
||||
def close_connection(p):
|
||||
logging.info("Abruptly closing a connection to %s due to downloading being paused",
|
||||
str(p))
|
||||
def num_peer_connections(self):
|
||||
return len(self._peer_connections)
|
||||
|
||||
if self.peer_connections[p]['factory'].p is not None:
|
||||
d = self.peer_connections[p]['factory'].p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
def _close_peers(self):
|
||||
def disconnect_peer(p):
|
||||
d = defer.Deferred()
|
||||
self._connections_closing[p] = d
|
||||
self._peer_connections[p].connection.disconnect()
|
||||
if p in self._peer_connections:
|
||||
del self._peer_connections[p]
|
||||
return d
|
||||
|
||||
def disconnect_peer():
|
||||
self.peer_connections[p]['connection'].disconnect()
|
||||
if p in self.peer_connections:
|
||||
del self.peer_connections[p]
|
||||
d = defer.Deferred()
|
||||
self.connections_closing[p] = d
|
||||
return d
|
||||
def close_connection(p):
|
||||
log.debug("%s Abruptly closing a connection to %s due to downloading being paused",
|
||||
self._get_log_name(), p)
|
||||
if self._peer_connections[p].factory.p is not None:
|
||||
d = self._peer_connections[p].factory.p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addBoth(lambda _: disconnect_peer(p))
|
||||
return d
|
||||
|
||||
d.addBoth(lambda _: disconnect_peer())
|
||||
return d
|
||||
|
||||
closing_deferreds.append(close_connection(peer))
|
||||
closing_deferreds = [close_connection(peer) for peer in self._peer_connections.keys()]
|
||||
return defer.DeferredList(closing_deferreds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_next_request(self, peer, protocol):
|
||||
log.debug("%s Trying to get the next request for peer %s", self._get_log_name(), peer)
|
||||
if not peer in self._peer_connections or self.stopped is True:
|
||||
log.debug("%s The peer %s has already been told to shut down.",
|
||||
self._get_log_name(), peer)
|
||||
defer.returnValue(False)
|
||||
requests = yield self._send_primary_requests(peer, protocol)
|
||||
have_request = any(r[1] for r in requests if r[0] is True)
|
||||
if have_request:
|
||||
yield self._send_secondary_requests(peer, protocol)
|
||||
defer.returnValue(have_request)
|
||||
|
||||
logging.debug("Trying to get the next request for peer %s", str(peer))
|
||||
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("The peer has already been told to shut down.")
|
||||
return defer.succeed(False)
|
||||
|
||||
def _send_primary_requests(self, peer, protocol):
|
||||
def handle_error(err):
|
||||
if err.check(InsufficientFundsError):
|
||||
self.downloader.insufficient_funds()
|
||||
return False
|
||||
else:
|
||||
return err
|
||||
err.trap(InsufficientFundsError)
|
||||
self.downloader.insufficient_funds(err)
|
||||
return False
|
||||
|
||||
def check_if_request_sent(request_sent, request_creator):
|
||||
if peer not in self._peer_connections:
|
||||
# This can happen if the connection is told to close
|
||||
return False
|
||||
if request_sent is False:
|
||||
if request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].remove(request_creator)
|
||||
if request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.remove(request_creator)
|
||||
else:
|
||||
if not request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].append(request_creator)
|
||||
if not request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.append(request_creator)
|
||||
return request_sent
|
||||
|
||||
def check_requests(requests):
|
||||
have_request = True in [r[1] for r in requests if r[0] is True]
|
||||
return have_request
|
||||
|
||||
def get_secondary_requests_if_necessary(have_request):
|
||||
if have_request is True:
|
||||
ds = []
|
||||
for s_r_c in self.secondary_request_creators:
|
||||
d = s_r_c.send_next_request(peer, protocol)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
else:
|
||||
dl = defer.succeed(None)
|
||||
dl.addCallback(lambda _: have_request)
|
||||
return dl
|
||||
|
||||
ds = []
|
||||
|
||||
for p_r_c in self.primary_request_creators:
|
||||
for p_r_c in self._primary_request_creators:
|
||||
d = p_r_c.send_next_request(peer, protocol)
|
||||
d.addErrback(handle_error)
|
||||
d.addCallback(check_if_request_sent, p_r_c)
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds, fireOnOneErrback=True)
|
||||
|
||||
dl = defer.DeferredList(ds, fireOnOneErrback=True)
|
||||
dl.addCallback(check_requests)
|
||||
dl.addCallback(get_secondary_requests_if_necessary)
|
||||
return dl
|
||||
def _send_secondary_requests(self, peer, protocol):
|
||||
ds = [
|
||||
s_r_c.send_next_request(peer, protocol)
|
||||
for s_r_c in self._secondary_request_creators
|
||||
]
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def protocol_disconnected(self, peer, protocol):
|
||||
if peer in self.peer_connections:
|
||||
del self.peer_connections[peer]
|
||||
if peer in self.connections_closing:
|
||||
d = self.connections_closing[peer]
|
||||
del self.connections_closing[peer]
|
||||
d.callback(True)
|
||||
@defer.inlineCallbacks
|
||||
def manage(self, schedule_next_call=True):
|
||||
self._manage_deferred = defer.Deferred()
|
||||
if len(self._peer_connections) < conf.settings['max_connections_per_stream']:
|
||||
log.debug("%s have %d connections, looking for %d",
|
||||
self._get_log_name(), len(self._peer_connections),
|
||||
conf.settings['max_connections_per_stream'])
|
||||
ordered_request_creators = self._rank_request_creator_connections()
|
||||
peers = yield self._get_new_peers(ordered_request_creators)
|
||||
new_conns = conf.settings['max_connections_per_stream'] - len(self._peer_connections)
|
||||
peers = self._pick_best_peers(peers, new_conns)
|
||||
for peer in peers:
|
||||
self._connect_to_peer(peer)
|
||||
self._manage_deferred.callback(None)
|
||||
self._manage_deferred = None
|
||||
if not self.stopped and schedule_next_call:
|
||||
self._next_manage_call = utils.call_later(self.MANAGE_CALL_INTERVAL_SEC, self.manage)
|
||||
|
||||
def _rank_request_creator_connections(self):
|
||||
"""
|
||||
@return: an ordered list of our request creators, ranked according to which has the least number of
|
||||
connections open that it likes
|
||||
"""Returns an ordered list of our request creators, ranked according
|
||||
to which has the least number of connections open that it
|
||||
likes
|
||||
"""
|
||||
def count_peers(request_creator):
|
||||
return len([p for p in self.peer_connections.itervalues() if request_creator in p['request_creators']])
|
||||
return len([
|
||||
p for p in self._peer_connections.itervalues()
|
||||
if request_creator in p.request_creators])
|
||||
|
||||
return sorted(self._primary_request_creators, key=count_peers)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _get_new_peers(self, request_creators):
|
||||
log.debug("%s Trying to get a new peer to connect to", self._get_log_name())
|
||||
if not request_creators:
|
||||
defer.returnValue(None)
|
||||
new_peers = yield request_creators[0].get_new_peers()
|
||||
if not new_peers:
|
||||
new_peers = yield self._get_new_peers(request_creators[1:])
|
||||
defer.returnValue(new_peers)
|
||||
|
||||
def _pick_best_peers(self, peers, num_peers_to_pick):
|
||||
# TODO: Eventually rank them based on past performance/reputation. For now
|
||||
# TODO: just pick the first to which we don't have an open connection
|
||||
log.debug("%s Got a list of peers to choose from: %s",
|
||||
self._get_log_name(), peers)
|
||||
log.debug("%s Current connections: %s",
|
||||
self._get_log_name(), self._peer_connections.keys())
|
||||
log.debug("%s List of connection states: %s", self._get_log_name(),
|
||||
[p_c_h.connection.state for p_c_h in self._peer_connections.values()])
|
||||
if peers is None:
|
||||
return []
|
||||
out = [peer for peer in peers if peer not in self._peer_connections]
|
||||
random.shuffle(out)
|
||||
return out[0:num_peers_to_pick]
|
||||
|
||||
return sorted(self.primary_request_creators, key=count_peers)
|
||||
|
||||
def _connect_to_peer(self, peer):
|
||||
if peer is None or self.stopped:
|
||||
return
|
||||
|
||||
from twisted.internet import reactor
|
||||
log.debug("%s Trying to connect to %s", self._get_log_name(), peer)
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
factory.connection_was_made_deferred.addCallback(
|
||||
lambda c_was_made: self._peer_disconnected(c_was_made, peer))
|
||||
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
|
||||
factory)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory,
|
||||
timeout=self.TCP_CONNECT_TIMEOUT)
|
||||
self._peer_connections[peer].connection = connection
|
||||
|
||||
if peer is not None:
|
||||
logging.debug("Trying to connect to %s", str(peer))
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory)
|
||||
self.peer_connections[peer] = {'connection': connection,
|
||||
'request_creators': self.primary_request_creators[:],
|
||||
'factory': factory}
|
||||
def _peer_disconnected(self, connection_was_made, peer):
|
||||
log.debug("%s protocol disconnected for %s",
|
||||
self._get_log_name(), peer)
|
||||
if peer in self._peer_connections:
|
||||
del self._peer_connections[peer]
|
||||
if peer in self._connections_closing:
|
||||
d = self._connections_closing[peer]
|
||||
del self._connections_closing[peer]
|
||||
d.callback(True)
|
||||
return connection_was_made
|
||||
|
||||
def _manage(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
def get_new_peers(request_creators):
|
||||
logging.debug("Trying to get a new peer to connect to")
|
||||
if len(request_creators) > 0:
|
||||
logging.debug("Got a creator to check: %s", str(request_creators[0]))
|
||||
d = request_creators[0].get_new_peers()
|
||||
d.addCallback(lambda h: h if h is not None else get_new_peers(request_creators[1:]))
|
||||
return d
|
||||
else:
|
||||
return defer.succeed(None)
|
||||
|
||||
def pick_best_peer(peers):
|
||||
# TODO: Eventually rank them based on past performance/reputation. For now
|
||||
# TODO: just pick the first to which we don't have an open connection
|
||||
logging.debug("Got a list of peers to choose from: %s", str(peers))
|
||||
if peers is None:
|
||||
return None
|
||||
for peer in peers:
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("Got a good peer. Returning peer %s", str(peer))
|
||||
return peer
|
||||
logging.debug("Couldn't find a good peer to connect to")
|
||||
return None
|
||||
|
||||
if len(self.peer_connections) < MAX_CONNECTIONS_PER_STREAM:
|
||||
ordered_request_creators = self._rank_request_creator_connections()
|
||||
d = get_new_peers(ordered_request_creators)
|
||||
d.addCallback(pick_best_peer)
|
||||
d.addCallback(self._connect_to_peer)
|
||||
|
||||
self.next_manage_call = reactor.callLater(1, self._manage)
|
||||
@@ -1,6 +1,13 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer, reactor
|
||||
from lbrynet.interfaces import IPeerFinder
|
||||
from lbrynet.core.utils import short_hash
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTPeerFinder(object):
|
||||
@@ -21,6 +28,7 @@ class DHTPeerFinder(object):
|
||||
self.next_manage_call = reactor.callLater(60, self.run_manage_loop)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self.next_manage_call is not None and self.next_manage_call.active():
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
@@ -28,20 +36,29 @@ class DHTPeerFinder(object):
|
||||
def _manage_peers(self):
|
||||
pass
|
||||
|
||||
def find_peers_for_blob(self, blob_hash):
|
||||
@defer.inlineCallbacks
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None):
|
||||
def _trigger_timeout():
|
||||
if not finished_deferred.called:
|
||||
log.debug("Peer search for %s timed out", short_hash(blob_hash))
|
||||
finished_deferred.cancel()
|
||||
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = self.dht_node.getPeersForBlob(bin_hash)
|
||||
|
||||
def filter_peers(peer_list):
|
||||
good_peers = []
|
||||
for host, port in peer_list:
|
||||
peer = self.peer_manager.get_peer(host, port)
|
||||
if peer.is_available() is True:
|
||||
good_peers.append(peer)
|
||||
return good_peers
|
||||
if timeout is not None:
|
||||
reactor.callLater(timeout, _trigger_timeout)
|
||||
|
||||
d = self.dht_node.getPeersForBlob(bin_hash)
|
||||
d.addCallback(filter_peers)
|
||||
return d
|
||||
peer_list = yield finished_deferred
|
||||
|
||||
peers = set(peer_list)
|
||||
good_peers = []
|
||||
for host, port in peers:
|
||||
peer = self.peer_manager.get_peer(host, port)
|
||||
if peer.is_available() is True:
|
||||
good_peers.append(peer)
|
||||
|
||||
defer.returnValue(good_peers)
|
||||
|
||||
def get_most_popular_hashes(self, num_to_return):
|
||||
return self.dht_node.get_most_popular_hashes(num_to_return)
|
||||
return self.dht_node.get_most_popular_hashes(num_to_return)
|
||||
@@ -5,18 +5,18 @@ from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadManager(object):
|
||||
implements(interfaces.IDownloadManager)
|
||||
|
||||
def __init__(self, blob_manager, upload_allowed):
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.upload_allowed = upload_allowed
|
||||
self.blob_requester = None
|
||||
self.blob_info_finder = None
|
||||
self.progress_manager = None
|
||||
self.blob_handler = None
|
||||
self.connection_manager = None
|
||||
|
||||
self.blobs = {}
|
||||
self.blob_infos = {}
|
||||
|
||||
@@ -24,7 +24,7 @@ class DownloadManager(object):
|
||||
|
||||
def start_downloading(self):
|
||||
d = self.blob_info_finder.get_initial_blobs()
|
||||
logging.debug("Requested the initial blobs from the info finder")
|
||||
log.debug("Requested the initial blobs from the info finder")
|
||||
d.addCallback(self.add_blobs_to_download)
|
||||
d.addCallback(lambda _: self.resume_downloading())
|
||||
return d
|
||||
@@ -33,7 +33,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_start(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
log.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -49,7 +49,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_stop(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to stop the %s: %s", manager. result.getErrorMessage())
|
||||
log.error("Failed to stop the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -57,28 +57,31 @@ class DownloadManager(object):
|
||||
d1.addBoth(check_stop, "progress manager")
|
||||
d2 = self.connection_manager.stop()
|
||||
d2.addBoth(check_stop, "connection manager")
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
dl.addCallback(lambda xs: False not in xs)
|
||||
dl = defer.DeferredList([d1, d2], consumeErrors=True)
|
||||
dl.addCallback(lambda results: all([success for success, val in results]))
|
||||
return dl
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
logging.debug("Adding %s to blobs", str(blob_infos))
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
logging.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
log.debug(
|
||||
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
|
||||
|
||||
def error_during_add(err):
|
||||
logging.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
log.warning(
|
||||
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
logging.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, self.upload_allowed, blob_info.length)
|
||||
log.debug(
|
||||
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
ds.append(d)
|
||||
@@ -105,11 +108,14 @@ class DownloadManager(object):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
to_be_outputted = [b for n, b in self.blobs.iteritems() if n >= self.progress_manager.last_blob_outputted]
|
||||
to_be_outputted = [
|
||||
b for n, b in self.blobs.iteritems()
|
||||
if n >= self.progress_manager.last_blob_outputted
|
||||
]
|
||||
return sum([b.length for b in to_be_outputted if b.length is not None])
|
||||
|
||||
def calculate_bytes_left_to_download(self):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
@@ -5,9 +5,15 @@ from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.Error import InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleBlobMetadataHandler(object):
|
||||
implements(interfaces.IMetadataHandler)
|
||||
|
||||
@@ -18,7 +24,7 @@ class SingleBlobMetadataHandler(object):
|
||||
######## IMetadataHandler #########
|
||||
|
||||
def get_initial_blobs(self):
|
||||
logging.debug("Returning the blob info")
|
||||
log.debug("Returning the blob info")
|
||||
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
|
||||
|
||||
def final_blob_num(self):
|
||||
@@ -72,7 +78,7 @@ class SingleProgressManager(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
log.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
if self.finished is False:
|
||||
self.finished = True
|
||||
reactor.callLater(0, self.finished_callback, blob)
|
||||
@@ -87,8 +93,8 @@ class DummyBlobHandler(object):
|
||||
|
||||
|
||||
class StandaloneBlobDownloader(object):
|
||||
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet):
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, wallet):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
@@ -99,11 +105,14 @@ class StandaloneBlobDownloader(object):
|
||||
self.finished_deferred = None
|
||||
|
||||
def download(self):
|
||||
if not is_valid_blobhash(self.blob_hash):
|
||||
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
|
||||
|
||||
def cancel_download(d):
|
||||
self.stop()
|
||||
|
||||
self.finished_deferred = defer.Deferred(canceller=cancel_download)
|
||||
self.download_manager = DownloadManager(self.blob_manager, True)
|
||||
self.download_manager = DownloadManager(self.blob_manager)
|
||||
self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
self.download_manager)
|
||||
@@ -127,7 +136,10 @@ class StandaloneBlobDownloader(object):
|
||||
|
||||
def _blob_downloaded(self, blob):
|
||||
self.stop()
|
||||
self.finished_deferred.callback(blob)
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.callback(blob)
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(err)
|
||||
@@ -4,10 +4,14 @@ from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamProgressManager(object):
|
||||
implements(IProgressManager)
|
||||
|
||||
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False):
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
self.finished_callback = finished_callback
|
||||
self.blob_manager = blob_manager
|
||||
self.delete_blob_after_finished = delete_blob_after_finished
|
||||
@@ -61,41 +65,55 @@ class StreamProgressManager(object):
|
||||
return defer.succeed(None)
|
||||
|
||||
def _finished_with_blob(self, blob_num):
|
||||
logging.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
if self.delete_blob_after_finished is True:
|
||||
logging.debug("delete_blob_after_finished is True")
|
||||
log.debug("delete_blob_after_finished is True")
|
||||
blobs = self.download_manager.blobs
|
||||
if blob_num in blobs:
|
||||
logging.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
log.debug("Telling the blob manager, %s, to delete blob %s",
|
||||
self.blob_manager, blobs[blob_num].blob_hash)
|
||||
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
|
||||
else:
|
||||
logging.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
log.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
else:
|
||||
logging.debug("delete_blob_after_finished is False")
|
||||
log.debug("delete_blob_after_finished is False")
|
||||
|
||||
|
||||
class FullStreamProgressManager(StreamProgressManager):
|
||||
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False):
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
StreamProgressManager.__init__(self, finished_callback, blob_manager, download_manager,
|
||||
delete_blob_after_finished)
|
||||
self.outputting_d = None
|
||||
|
||||
######### IProgressManager #########
|
||||
|
||||
def _done(self, i, blobs):
|
||||
"""Return true if `i` is a blob number we don't have"""
|
||||
return (
|
||||
i not in blobs or
|
||||
(
|
||||
not blobs[i].is_validated() and
|
||||
i not in self.provided_blob_nums
|
||||
)
|
||||
)
|
||||
|
||||
def stream_position(self):
|
||||
blobs = self.download_manager.blobs
|
||||
if not blobs:
|
||||
return 0
|
||||
else:
|
||||
for i in xrange(max(blobs.iterkeys())):
|
||||
if not i in blobs or (not blobs[i].is_validated() and not i in self.provided_blob_nums):
|
||||
if self._done(i, blobs):
|
||||
return i
|
||||
return max(blobs.iterkeys()) + 1
|
||||
|
||||
def needed_blobs(self):
|
||||
blobs = self.download_manager.blobs
|
||||
return [b for n, b in blobs.iteritems() if not b.is_validated() and not n in self.provided_blob_nums]
|
||||
return [
|
||||
b for n, b in blobs.iteritems()
|
||||
if not b.is_validated() and not n in self.provided_blob_nums
|
||||
]
|
||||
|
||||
######### internal #########
|
||||
|
||||
@@ -115,6 +133,8 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
|
||||
def finished_outputting_blob():
|
||||
self.last_blob_outputted += 1
|
||||
|
||||
def check_if_finished():
|
||||
final_blob_num = self.download_manager.final_blob_num()
|
||||
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
|
||||
self._finished_outputting()
|
||||
@@ -126,16 +146,20 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
current_blob_num = self.last_blob_outputted + 1
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
log.debug("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
self.provided_blob_nums.append(self.last_blob_outputted + 1)
|
||||
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
d.addCallback(lambda _: check_if_finished())
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
log.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
if self.outputting_d is not None and not self.outputting_d.called:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
|
||||
d.addErrback(log_error)
|
||||
else:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
self.outputting_d = None
|
||||
@@ -15,4 +15,4 @@ def sign_with_pass_phrase(m, pass_phrase):
|
||||
|
||||
|
||||
def verify_signature(m, signature, pub_key):
|
||||
return seccure.verify(m, signature, pub_key, curve="brainpoolp384r1")
|
||||
return seccure.verify(m, signature, pub_key, curve="brainpoolp384r1")
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_read_handle(path):
|
||||
"""
|
||||
Get os independent read handle for a file
|
||||
"""
|
||||
|
||||
if os.name == "nt":
|
||||
file_mode = 'rb'
|
||||
else:
|
||||
file_mode = 'r'
|
||||
read_handle = open(path, file_mode)
|
||||
yield read_handle
|
||||
read_handle.close()
|
||||
@@ -0,0 +1,382 @@
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from txrequests import Session
|
||||
import twisted.python.log
|
||||
|
||||
from lbrynet import __version__ as lbrynet_version, analytics, build_type, conf
|
||||
from lbrynet.core import utils
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
#####
|
||||
|
||||
|
||||
TRACE = 5
|
||||
|
||||
|
||||
def bg_cb(sess, resp):
|
||||
""" Don't do anything with the response """
|
||||
pass
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None, session=None):
|
||||
logging.Handler.__init__(self)
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.session = session if session is not None else Session()
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
payload = self.format(record)
|
||||
self.session.post(self.url, data=payload, background_callback=bg_cb)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except:
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
for handler in log.handlers:
|
||||
if handler.name == handler_name:
|
||||
log.removeHandler(handler)
|
||||
|
||||
|
||||
def _log_decorator(fn):
|
||||
"""Configure a logging handler.
|
||||
|
||||
`fn` is a function that returns a logging handler. The returned
|
||||
handler has its log-level set and is attached to the specified
|
||||
logger or the root logger.
|
||||
"""
|
||||
|
||||
def helper(*args, **kwargs):
|
||||
log = kwargs.pop('log', logging.getLogger())
|
||||
level = kwargs.pop('level', logging.INFO)
|
||||
if not isinstance(level, int):
|
||||
# despite the name, getLevelName returns
|
||||
# the numeric level when passed a text level
|
||||
level = logging.getLevelName(level)
|
||||
handler = fn(*args, **kwargs)
|
||||
configure_handler(handler, log, level)
|
||||
return handler
|
||||
|
||||
return helper
|
||||
|
||||
|
||||
def configure_handler(handler, log, level):
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
return handler
|
||||
|
||||
|
||||
def disable_third_party_loggers():
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
logging.getLogger('urllib3').setLevel(logging.WARNING)
|
||||
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
|
||||
logging.getLogger('lbryum').setLevel(logging.WARNING)
|
||||
logging.getLogger('twisted').setLevel(logging.WARNING)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_console(**kwargs):
|
||||
"""Convenience function to configure a log-handler that outputs to stdout"""
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'console'
|
||||
return handler
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_file_handler(file_name, **kwargs):
|
||||
"""Convenience function to configure a log-handler that writes to `file_name`"""
|
||||
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'file'
|
||||
return handler
|
||||
|
||||
|
||||
def configure_analytics_handler(analytics_manager):
|
||||
handler = analytics.Handler(analytics_manager)
|
||||
handler.name = 'analytics'
|
||||
return configure_handler(handler, logging.getLogger(), logging.ERROR)
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
|
||||
version = version or lbrynet_version
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
def configure_loggly_handler(*args, **kwargs):
|
||||
if build_type.BUILD == 'dev':
|
||||
return
|
||||
level = kwargs.pop('level', logging.WARNING)
|
||||
_configure_loggly_handler(*args, level=level, **kwargs)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def _configure_loggly_handler(url=None, **kwargs):
|
||||
url = url or get_loggly_url()
|
||||
formatter = JsonFormatter(**kwargs)
|
||||
handler = HTTPSHandler(url)
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
|
||||
def failure(failure, log, msg, *args):
|
||||
"""Log a failure message from a deferred.
|
||||
|
||||
Args:
|
||||
failure: twisted.python.failure.Failure
|
||||
log: a python logger instance
|
||||
msg: the message to log. Can use normal logging string interpolation.
|
||||
the last argument will be set to the error message from the failure.
|
||||
args: values to substitute into `msg`
|
||||
"""
|
||||
args += (failure.getErrorMessage(),)
|
||||
exc_info = (failure.type, failure.value, failure.getTracebackObject())
|
||||
log.error(msg, *args, exc_info=exc_info)
|
||||
|
||||
|
||||
def convert_verbose(verbose):
|
||||
"""Convert the results of the --verbose flag into a list of logger names
|
||||
|
||||
if --verbose is not provided, args.verbose will be None and logging
|
||||
should be at the info level.
|
||||
if --verbose is provided, but not followed by any arguments, then
|
||||
args.verbose = [] and debug logging should be enabled for all of lbrynet
|
||||
along with info logging on lbryum.
|
||||
if --verbose is provided and followed by arguments, those arguments
|
||||
will be in a list
|
||||
"""
|
||||
if verbose is None:
|
||||
return []
|
||||
if verbose == []:
|
||||
return ['lbrynet', 'lbryum']
|
||||
return verbose
|
||||
|
||||
|
||||
def configure_logging(file_name, console, verbose=None):
|
||||
"""Apply the default logging configuration.
|
||||
|
||||
Enables two log-handlers at the INFO level: a file logger and a loggly logger.
|
||||
Optionally turns on a console logger that defaults to the INFO level, with
|
||||
specified loggers being set to the DEBUG level.
|
||||
|
||||
Args:
|
||||
file_name: the file to which logs should be saved
|
||||
console: If true, enable a console logger
|
||||
verbose: a list of loggers to set to debug level.
|
||||
See `convert_verbose` for more details.
|
||||
"""
|
||||
verbose = convert_verbose(verbose)
|
||||
configure_twisted()
|
||||
configure_file_handler(file_name)
|
||||
configure_loggly_handler()
|
||||
disable_third_party_loggers()
|
||||
if console:
|
||||
# if there are some loggers at the debug level, we need
|
||||
# to enable the console to allow debug. Otherwise, only
|
||||
# allow info.
|
||||
level = 'DEBUG' if verbose else 'INFO'
|
||||
handler = configure_console(level=level)
|
||||
if 'lbryum' in verbose:
|
||||
# TODO: this enables lbryum logging on the other handlers
|
||||
# too which isn't consistent with how verbose logging
|
||||
# happens with other loggers. Should change the configuration
|
||||
# so that its only logging at the INFO level for the console.
|
||||
logging.getLogger('lbryum').setLevel(logging.INFO)
|
||||
verbose.remove('lbryum')
|
||||
if verbose:
|
||||
handler.addFilter(LoggerNameFilter(verbose))
|
||||
|
||||
|
||||
def configure_twisted():
|
||||
"""Setup twisted logging to output events to the python stdlib logger"""
|
||||
# I tried using the new logging api
|
||||
# https://twistedmatrix.com/documents/current/core/howto/logger.html#compatibility-with-standard-library-logging
|
||||
# and it simply didn't work
|
||||
observer = twisted.python.log.PythonLoggingObserver()
|
||||
observer.start()
|
||||
|
||||
|
||||
class LoggerNameFilter(object):
|
||||
"""Filter a log record based on its name.
|
||||
|
||||
Allows all info level and higher records to pass thru.
|
||||
Debug records pass if the log record name (or a parent) match
|
||||
the input list of logger names.
|
||||
"""
|
||||
|
||||
def __init__(self, logger_names):
|
||||
self.logger_names = logger_names
|
||||
|
||||
def filter(self, record):
|
||||
if record.levelno >= logging.INFO:
|
||||
return True
|
||||
name = record.name
|
||||
while name:
|
||||
if name in self.logger_names:
|
||||
return True
|
||||
name = get_parent(name)
|
||||
return False
|
||||
|
||||
|
||||
def get_parent(logger_name):
|
||||
names = logger_name.split('.')
|
||||
if len(names) == 1:
|
||||
return ''
|
||||
names = names[:-1]
|
||||
return '.'.join(names)
|
||||
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to subtitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
@@ -0,0 +1,20 @@
|
||||
class LoopingCallManager(object):
|
||||
def __init__(self, calls=None):
|
||||
self.calls = calls or {}
|
||||
|
||||
def register_looping_call(self, name, call):
|
||||
assert name not in self.calls, '{} is already registered'.format(name)
|
||||
self.calls[name] = call
|
||||
|
||||
def start(self, name, *args):
|
||||
lcall = self.calls[name]
|
||||
if not lcall.running:
|
||||
lcall.start(*args)
|
||||
|
||||
def stop(self, name):
|
||||
self.calls[name].stop()
|
||||
|
||||
def shutdown(self):
|
||||
for lcall in self.calls.itervalues():
|
||||
if lcall.running:
|
||||
lcall.stop()
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -37,10 +40,11 @@ class BlobAvailabilityHandler(object):
|
||||
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
logging.debug("Received the client's list of requested blobs")
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
@@ -52,4 +56,4 @@ class BlobAvailabilityHandler(object):
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
return d
|
||||
return d
|
||||
@@ -1,23 +1,31 @@
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet import analytics
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequestHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager):
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, track):
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.track = track
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = BlobRequestHandler(self.blob_manager, self.wallet, self.payment_rate_manager)
|
||||
q_h = BlobRequestHandler(
|
||||
self.blob_manager, self.wallet, self.payment_rate_manager, self.track)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
@@ -29,18 +37,23 @@ class BlobRequestHandlerFactory(object):
|
||||
|
||||
class BlobRequestHandler(object):
|
||||
implements(IQueryHandler, IBlobSender)
|
||||
PAYMENT_RATE_QUERY = 'blob_data_payment_rate'
|
||||
BLOB_QUERY = 'requested_blob'
|
||||
AVAILABILITY_QUERY = 'requested_blobs'
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager):
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, track):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = ['blob_data_payment_rate', 'requested_blob']
|
||||
self.query_identifiers = [self.PAYMENT_RATE_QUERY, self.BLOB_QUERY, self.AVAILABILITY_QUERY]
|
||||
self.track = track
|
||||
self.peer = None
|
||||
self.blob_data_payment_rate = None
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
self.blob_bytes_uploaded = 0
|
||||
self._blobs_requested = []
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
@@ -50,44 +63,22 @@ class BlobRequestHandler(object):
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = {}
|
||||
if self.query_identifiers[0] in queries:
|
||||
if not self.handle_blob_data_payment_rate(queries[self.query_identifiers[0]]):
|
||||
response['blob_data_payment_rate'] = "RATE_TOO_LOW"
|
||||
else:
|
||||
response['blob_data_payment_rate'] = 'RATE_ACCEPTED'
|
||||
response = defer.succeed({})
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
logging.debug("Received the client's request to send a blob")
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response_fields['error'] = "RATE_UNSET"
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
|
||||
d = self.blob_manager.get_blob(queries[self.query_identifiers[1]], True)
|
||||
|
||||
def open_blob_for_reading(blob):
|
||||
if blob.is_validated():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
logging.debug("Sending %s to client", str(blob))
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
return response
|
||||
logging.debug("We can not send %s", str(blob))
|
||||
response_fields['error'] = "BLOB_UNAVAILABLE"
|
||||
return response
|
||||
|
||||
d.addCallback(open_blob_for_reading)
|
||||
|
||||
return d
|
||||
else:
|
||||
return defer.succeed(response)
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
|
||||
@@ -105,12 +96,92 @@ class BlobRequestHandler(object):
|
||||
|
||||
######### internal #########
|
||||
|
||||
def handle_blob_data_payment_rate(self, requested_payment_rate):
|
||||
if not self.payment_rate_manager.accept_rate_blob_data(self.peer, requested_payment_rate):
|
||||
return False
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
log.debug("Offered rate %f LBC/mb for %i blobs", offer.rate, len(blobs))
|
||||
reply = self.payment_rate_manager.reply_to_offer(self.peer, blobs, offer)
|
||||
if reply.is_accepted:
|
||||
self.blob_data_payment_rate = offer.rate
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_ACCEPTED"
|
||||
log.debug("Accepted rate: %f", offer.rate)
|
||||
elif reply.is_too_low:
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_TOO_LOW"
|
||||
log.debug("Reject rate: %f", offer.rate)
|
||||
elif reply.is_unset:
|
||||
log.warning("Rate unset")
|
||||
request['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
log.debug("Returning rate query result: %s", str(request))
|
||||
|
||||
return request
|
||||
|
||||
def _handle_blob_query(self, response, query):
|
||||
log.debug("Received the client's request to send a blob")
|
||||
response['incoming_blob'] = {}
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response['incoming_blob'] = {'error': "RATE_UNSET"}
|
||||
return response
|
||||
else:
|
||||
self.blob_data_payment_rate = requested_payment_rate
|
||||
return True
|
||||
return self._send_blob(response, query)
|
||||
|
||||
def _send_blob(self, response, query):
|
||||
d = self.blob_manager.get_blob(query)
|
||||
d.addCallback(self.open_blob_for_reading, response)
|
||||
return d
|
||||
|
||||
def open_blob_for_reading(self, blob, response):
|
||||
response_fields = {}
|
||||
d = defer.succeed(None)
|
||||
if blob.is_validated():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
log.info("Sending %s to %s", str(blob), self.peer)
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
response['incoming_blob'] = response_fields
|
||||
d.addCallback(lambda _: self.record_transaction(blob))
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
log.debug("We can not send %s", str(blob))
|
||||
response['incoming_blob'] = {'error': 'BLOB_UNAVAILABLE'}
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
|
||||
def record_transaction(self, blob):
|
||||
d = self.blob_manager.add_blob_to_upload_history(
|
||||
str(blob), self.peer.host, self.blob_data_payment_rate)
|
||||
return d
|
||||
|
||||
def _reply_to_send_request(self, response, incoming):
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
log.debug("Rate not set yet")
|
||||
response['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
log.debug("Requested blob: %s", str(incoming))
|
||||
d = self.blob_manager.get_blob(incoming)
|
||||
d.addCallback(lambda blob: self.open_blob_for_reading(blob, response))
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
@@ -123,23 +194,30 @@ class BlobRequestHandler(object):
|
||||
return inner_d
|
||||
|
||||
def count_bytes(data):
|
||||
self.blob_bytes_uploaded += len(data)
|
||||
self.peer.update_stats('blob_bytes_uploaded', len(data))
|
||||
uploaded = len(data)
|
||||
self.blob_bytes_uploaded += uploaded
|
||||
self.peer.update_stats('blob_bytes_uploaded', uploaded)
|
||||
self.track.add_observation(analytics.BLOB_BYTES_UPLOADED, uploaded)
|
||||
return data
|
||||
|
||||
def start_transfer():
|
||||
self.file_sender = FileSender()
|
||||
logging.info("Starting the file upload")
|
||||
assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer"
|
||||
log.debug("Starting the file upload")
|
||||
assert self.read_handle is not None, \
|
||||
"self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
|
||||
return d
|
||||
|
||||
def set_expected_payment():
|
||||
logging.info("Setting expected payment")
|
||||
if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None:
|
||||
log.debug("Setting expected payment")
|
||||
if (
|
||||
self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None
|
||||
and self.blob_data_payment_rate > 0
|
||||
):
|
||||
# TODO: explain why 2**20
|
||||
self.wallet.add_expected_payment(self.peer,
|
||||
self.currently_uploading.length * 1.0 *
|
||||
self.blob_data_payment_rate / 2**20)
|
||||
self.blob_data_payment_rate / 2 ** 20)
|
||||
self.blob_bytes_uploaded = 0
|
||||
self.peer.update_stats('blobs_uploaded', 1)
|
||||
return None
|
||||
@@ -151,6 +229,6 @@ class BlobRequestHandler(object):
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
if reason is not None and isinstance(reason, Failure):
|
||||
logging.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
return _send_file()
|
||||
return _send_file()
|
||||
@@ -1,9 +1,18 @@
|
||||
import binascii
|
||||
from twisted.internet import defer, task, reactor
|
||||
import collections
|
||||
import logging
|
||||
import time
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTHashAnnouncer(object):
|
||||
ANNOUNCE_CHECK_INTERVAL = 60
|
||||
CONCURRENT_ANNOUNCERS = 5
|
||||
|
||||
"""This class announces to the DHT that this peer has certain blobs"""
|
||||
def __init__(self, dht_node, peer_port):
|
||||
self.dht_node = dht_node
|
||||
@@ -14,14 +23,12 @@ class DHTHashAnnouncer(object):
|
||||
self._concurrent_announcers = 0
|
||||
|
||||
def run_manage_loop(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.peer_port is not None:
|
||||
self._announce_available_hashes()
|
||||
self.next_manage_call = reactor.callLater(60, self.run_manage_loop)
|
||||
self.next_manage_call = utils.call_later(self.ANNOUNCE_CHECK_INTERVAL, self.run_manage_loop)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self.next_manage_call is not None:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
@@ -31,11 +38,15 @@ class DHTHashAnnouncer(object):
|
||||
|
||||
def immediate_announce(self, blob_hashes):
|
||||
if self.peer_port is not None:
|
||||
return self._announce_hashes(blob_hashes)
|
||||
return self._announce_hashes(blob_hashes, immediate=True)
|
||||
else:
|
||||
return defer.succeed(False)
|
||||
|
||||
def hash_queue_size(self):
|
||||
return len(self.hash_queue)
|
||||
|
||||
def _announce_available_hashes(self):
|
||||
log.debug('Announcing available hashes')
|
||||
ds = []
|
||||
for supplier in self.suppliers:
|
||||
d = supplier.hashes_to_announce()
|
||||
@@ -44,38 +55,75 @@ class DHTHashAnnouncer(object):
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
|
||||
def _announce_hashes(self, hashes):
|
||||
|
||||
def _announce_hashes(self, hashes, immediate=False):
|
||||
if not hashes:
|
||||
return
|
||||
log.debug('Announcing %s hashes', len(hashes))
|
||||
# TODO: add a timeit decorator
|
||||
start = time.time()
|
||||
ds = []
|
||||
|
||||
for h in hashes:
|
||||
announce_deferred = defer.Deferred()
|
||||
ds.append(announce_deferred)
|
||||
self.hash_queue.append((h, announce_deferred))
|
||||
if immediate:
|
||||
self.hash_queue.appendleft((h, announce_deferred))
|
||||
else:
|
||||
self.hash_queue.append((h, announce_deferred))
|
||||
log.debug('There are now %s hashes remaining to be announced', self.hash_queue_size())
|
||||
|
||||
def announce():
|
||||
if len(self.hash_queue):
|
||||
h, announce_deferred = self.hash_queue.popleft()
|
||||
log.debug('Announcing blob %s to dht', h)
|
||||
d = self.dht_node.announceHaveBlob(binascii.unhexlify(h), self.peer_port)
|
||||
d.chainDeferred(announce_deferred)
|
||||
d.addBoth(lambda _: reactor.callLater(0, announce))
|
||||
d.addBoth(lambda _: utils.call_later(0, announce))
|
||||
else:
|
||||
self._concurrent_announcers -= 1
|
||||
|
||||
for i in range(self._concurrent_announcers, 5):
|
||||
# TODO: maybe make the 5 configurable
|
||||
for i in range(self._concurrent_announcers, self.CONCURRENT_ANNOUNCERS):
|
||||
self._concurrent_announcers += 1
|
||||
announce()
|
||||
return defer.DeferredList(ds)
|
||||
d = defer.DeferredList(ds)
|
||||
d.addCallback(lambda _: log.debug('Took %s seconds to announce %s hashes',
|
||||
time.time() - start, len(hashes)))
|
||||
return d
|
||||
|
||||
|
||||
class DHTHashSupplier(object):
|
||||
# 1 hour is the min time hash will be reannounced
|
||||
MIN_HASH_REANNOUNCE_TIME = 60*60
|
||||
# conservative assumption of the time it takes to announce
|
||||
# a single hash
|
||||
SINGLE_HASH_ANNOUNCE_DURATION = 1
|
||||
|
||||
"""Classes derived from this class give hashes to a hash announcer"""
|
||||
def __init__(self, announcer):
|
||||
if announcer is not None:
|
||||
announcer.add_supplier(self)
|
||||
self.hash_announcer = announcer
|
||||
self.hash_reannounce_time = 60 * 60 # 1 hour
|
||||
|
||||
def hashes_to_announce(self):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
def get_next_announce_time(self, num_hashes_to_announce=1):
|
||||
"""
|
||||
Hash reannounce time is set to current time + MIN_HASH_REANNOUNCE_TIME,
|
||||
unless we are announcing a lot of hashes at once which could cause the
|
||||
the announce queue to pile up. To prevent pile up, reannounce
|
||||
only after a conservative estimate of when it will finish
|
||||
to announce all the hashes.
|
||||
|
||||
Args:
|
||||
num_hashes_to_announce: number of hashes that will be added to the queue
|
||||
Returns:
|
||||
timestamp for next announce time
|
||||
"""
|
||||
queue_size = self.hash_announcer.hash_queue_size()+num_hashes_to_announce
|
||||
reannounce = max(self.MIN_HASH_REANNOUNCE_TIME,
|
||||
queue_size*self.SINGLE_HASH_ANNOUNCE_DURATION)
|
||||
return time.time() + reannounce
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ from zope.interface import implements
|
||||
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerProtocol(Protocol):
|
||||
"""ServerProtocol needs to:
|
||||
|
||||
@@ -26,15 +29,14 @@ class ServerProtocol(Protocol):
|
||||
#Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
logging.debug("Got a connection")
|
||||
log.debug("Got a connection")
|
||||
peer_info = self.transport.getPeer()
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.request_handler = ServerRequestHandler(self)
|
||||
for query_handler_factory, enabled in self.factory.query_handler_factories.iteritems():
|
||||
if enabled is True:
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
logging.debug("Setting the request handler")
|
||||
for query_handler_factory in self.factory.query_handler_factories.values():
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
log.debug("Setting the request handler")
|
||||
self.factory.rate_limiter.register_protocol(self)
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
@@ -42,10 +44,10 @@ class ServerProtocol(Protocol):
|
||||
self.request_handler.stopProducing()
|
||||
self.factory.rate_limiter.unregister_protocol(self)
|
||||
if not reason.check(error.ConnectionDone):
|
||||
logging.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
def dataReceived(self, data):
|
||||
logging.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
self.factory.rate_limiter.report_dl_bytes(len(data))
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.data_received(data)
|
||||
@@ -53,7 +55,7 @@ class ServerProtocol(Protocol):
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
logging.debug("Registering the producer")
|
||||
log.debug("Registering the producer")
|
||||
assert streaming is True
|
||||
|
||||
def unregisterProducer(self):
|
||||
@@ -61,7 +63,7 @@ class ServerProtocol(Protocol):
|
||||
self.transport.loseConnection()
|
||||
|
||||
def write(self, data):
|
||||
logging.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
log.trace("Writing %s bytes of data to the transport", len(data))
|
||||
self.transport.write(data)
|
||||
self.factory.rate_limiter.report_ul_bytes(len(data))
|
||||
|
||||
@@ -88,4 +90,4 @@ class ServerProtocolFactory(ServerFactory):
|
||||
def __init__(self, rate_limiter, query_handler_factories, peer_manager):
|
||||
self.rate_limiter = rate_limiter
|
||||
self.query_handler_factories = query_handler_factories
|
||||
self.peer_manager = peer_manager
|
||||
self.peer_manager = peer_manager
|
||||
@@ -5,10 +5,14 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IRequestHandler
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and return request for information about
|
||||
more blobs that are associated with streams"""
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and
|
||||
return request for information about more blobs that are
|
||||
associated with streams.
|
||||
"""
|
||||
implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
|
||||
|
||||
def __init__(self, consumer):
|
||||
@@ -48,18 +52,19 @@ class ServerRequestHandler(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.production_paused is False:
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk != '':
|
||||
logging.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
if self.production_paused:
|
||||
return
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk == '':
|
||||
return
|
||||
log.trace("writing %s bytes to the client", len(chunk))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
#assert self.file_sender == producer
|
||||
self.producer = producer
|
||||
assert streaming is False
|
||||
producer.resumeProducing()
|
||||
@@ -76,7 +81,7 @@ class ServerRequestHandler(object):
|
||||
|
||||
def get_more_data():
|
||||
if self.producer is not None:
|
||||
logging.debug("Requesting more data from the producer")
|
||||
log.trace("Requesting more data from the producer")
|
||||
self.producer.resumeProducing()
|
||||
|
||||
reactor.callLater(0, get_more_data)
|
||||
@@ -84,22 +89,30 @@ class ServerRequestHandler(object):
|
||||
#From Protocol
|
||||
|
||||
def data_received(self, data):
|
||||
logging.debug("Received data")
|
||||
logging.debug("%s", str(data))
|
||||
log.debug("Received data")
|
||||
log.debug("%s", str(data))
|
||||
if self.request_received is False:
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg is not None:
|
||||
self.request_buff = ''
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender is not None:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
else:
|
||||
logging.info("Request buff not a valid json message")
|
||||
logging.info("Request buff: %s", str(self.request_buff))
|
||||
return self._parse_data_and_maybe_send_blob(data)
|
||||
else:
|
||||
logging.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
log.warning(
|
||||
"The client sent data when we were uploading a file. This should not happen")
|
||||
|
||||
def _parse_data_and_maybe_send_blob(self, data):
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg:
|
||||
self.request_buff = ''
|
||||
self._process_msg(msg)
|
||||
else:
|
||||
log.debug("Request buff not a valid json message")
|
||||
log.debug("Request buff: %s", self.request_buff)
|
||||
|
||||
def _process_msg(self, msg):
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
|
||||
|
||||
######### IRequestHandler #########
|
||||
|
||||
@@ -112,7 +125,7 @@ class ServerRequestHandler(object):
|
||||
#response handling
|
||||
|
||||
def request_failure_handler(self, err):
|
||||
logging.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
self.stopProducing()
|
||||
return err
|
||||
|
||||
@@ -122,15 +135,15 @@ class ServerRequestHandler(object):
|
||||
|
||||
def send_response(self, msg):
|
||||
m = json.dumps(msg)
|
||||
logging.info("Sending a response of length %s", str(len(m)))
|
||||
logging.debug("Response: %s", str(m))
|
||||
log.debug("Sending a response of length %s", str(len(m)))
|
||||
log.debug("Response: %s", str(m))
|
||||
self.response_buff = self.response_buff + m
|
||||
self._produce_more()
|
||||
return True
|
||||
|
||||
def handle_request(self, msg):
|
||||
logging.debug("Handling a request")
|
||||
logging.debug(str(msg))
|
||||
log.debug("Handling a request")
|
||||
log.debug(str(msg))
|
||||
|
||||
def create_response_message(results):
|
||||
response = {}
|
||||
@@ -140,11 +153,13 @@ class ServerRequestHandler(object):
|
||||
else:
|
||||
# result is a Failure
|
||||
return result
|
||||
logging.debug("Finished making the response message. Response: %s", str(response))
|
||||
log.debug("Finished making the response message. Response: %s", str(response))
|
||||
return response
|
||||
|
||||
def log_errors(err):
|
||||
logging.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
log.warning(
|
||||
"An error occurred handling a client request. Error message: %s",
|
||||
err.getErrorMessage())
|
||||
return err
|
||||
|
||||
def send_response(response):
|
||||
@@ -168,4 +183,4 @@ class ServerRequestHandler(object):
|
||||
msg = json.loads(request_buff)
|
||||
return msg
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -0,0 +1,23 @@
|
||||
import sqlite3
|
||||
from twisted.internet import task, reactor
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
|
||||
def rerun(err, *args, **kwargs):
|
||||
if err.check(sqlite3.OperationalError) and err.value.message == "database is locked":
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
return task.deferLater(reactor, 0, wrapper, *args, **kwargs)
|
||||
return err
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addErrback(rerun, *args, **kwargs)
|
||||
return d
|
||||
|
||||
return wrapper
|
||||
@@ -0,0 +1,29 @@
|
||||
import platform
|
||||
import json
|
||||
|
||||
from urllib2 import urlopen
|
||||
|
||||
from lbrynet import __version__ as lbrynet_version
|
||||
from lbrynet import build_type
|
||||
from lbryum.version import LBRYUM_VERSION as lbryum_version
|
||||
|
||||
|
||||
def get_platform(get_ip=True):
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"lbryum_version": lbryum_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
|
||||
if get_ip:
|
||||
try:
|
||||
p['ip'] = json.load(urlopen('http://jsonip.com'))['ip']
|
||||
except:
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
+103
-9
@@ -1,8 +1,53 @@
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
import base64
|
||||
import datetime
|
||||
import logging
|
||||
import random
|
||||
import socket
|
||||
import string
|
||||
import json
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from lbryschema.claim import ClaimDict
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2 # digest_size is in bytes, and blob hashes are hex encoded
|
||||
# defining these time functions here allows for easier overriding in testing
|
||||
def now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def utcnow():
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def isonow():
|
||||
"""Return utc now in isoformat with timezone"""
|
||||
return utcnow().isoformat() + 'Z'
|
||||
|
||||
|
||||
def today():
|
||||
return datetime.datetime.today()
|
||||
|
||||
|
||||
def timedelta(**kwargs):
|
||||
return datetime.timedelta(**kwargs)
|
||||
|
||||
|
||||
def datetime_obj(*args, **kwargs):
|
||||
return datetime.datetime(*args, **kwargs)
|
||||
|
||||
|
||||
def call_later(delay, func, *args, **kwargs):
|
||||
# Import here to ensure that it gets called after installing a reactor
|
||||
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
|
||||
from twisted.internet import reactor
|
||||
return reactor.callLater(delay, func, *args, **kwargs)
|
||||
|
||||
|
||||
def generate_id(num=None):
|
||||
@@ -14,15 +59,64 @@ def generate_id(num=None):
|
||||
return h.digest()
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: Whether the blobhash is the correct length and contains only valid characters (0-9, a-f)
|
||||
@return: True/False
|
||||
"""
|
||||
if len(blobhash) != blobhash_length:
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def version_is_greater_than(a, b):
|
||||
"""Returns True if version a is more recent than version b"""
|
||||
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
|
||||
|
||||
|
||||
def deobfuscate(obfustacated):
|
||||
return base64.b64decode(obfustacated.decode('rot13'))
|
||||
|
||||
|
||||
def obfuscate(plain):
|
||||
return base64.b64encode(plain).encode('rot13')
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80):
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
try:
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
host = socket.gethostbyname(server)
|
||||
s = socket.create_connection((host, port), 2)
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except Exception as ex:
|
||||
log.info(
|
||||
"Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port, exc_info=True)
|
||||
return False
|
||||
for l in blobhash:
|
||||
if l not in "0123456789abcdef":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
|
||||
def short_hash(hash_str):
|
||||
return hash_str[:6]
|
||||
|
||||
|
||||
def get_sd_hash(stream_info):
|
||||
if not stream_info:
|
||||
return None
|
||||
if isinstance(stream_info, ClaimDict):
|
||||
return stream_info.source_hash
|
||||
return stream_info['stream']['source']['source']
|
||||
|
||||
|
||||
def json_dumps_pretty(obj, **kwargs):
|
||||
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
|
||||
@@ -1,10 +1,13 @@
|
||||
import binascii
|
||||
import logging
|
||||
from Crypto.Cipher import AES
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
@@ -32,8 +35,8 @@ class StreamBlobDecryptor(object):
|
||||
|
||||
def write_bytes():
|
||||
if self.len_read < self.length:
|
||||
num_bytes_to_decrypt = (len(self.buff) // self.cipher.block_size) * self.cipher.block_size
|
||||
data_to_decrypt, self.buff = self.buff[:num_bytes_to_decrypt], self.buff[num_bytes_to_decrypt:]
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), self.cipher.block_size)
|
||||
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
|
||||
write_func(self.cipher.decrypt(data_to_decrypt))
|
||||
|
||||
def finish_decrypt():
|
||||
@@ -45,7 +48,6 @@ class StreamBlobDecryptor(object):
|
||||
self.buff += data
|
||||
self.len_read += len(data)
|
||||
write_bytes()
|
||||
#write_func(remove_padding(self.cipher.decrypt(self.buff)))
|
||||
|
||||
d = self.blob.read(decrypt_bytes)
|
||||
d.addCallback(lambda _: finish_decrypt())
|
||||
@@ -64,7 +66,7 @@ class CryptStreamBlobMaker(object):
|
||||
self.length = 0
|
||||
|
||||
def write(self, data):
|
||||
max_bytes_to_write = BLOB_SIZE - self.length - 1
|
||||
max_bytes_to_write = conf.settings['BLOB_SIZE'] - self.length - 1
|
||||
done = False
|
||||
if max_bytes_to_write <= len(data):
|
||||
num_bytes_to_write = max_bytes_to_write
|
||||
@@ -78,17 +80,17 @@ class CryptStreamBlobMaker(object):
|
||||
return done, num_bytes_to_write
|
||||
|
||||
def close(self):
|
||||
logging.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self._close_buffer()
|
||||
d = self.blob.close()
|
||||
d.addCallback(self._return_info)
|
||||
logging.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
return d
|
||||
|
||||
def _write_buffer(self):
|
||||
num_bytes_to_encrypt = (len(self.buff) // AES.block_size) * AES.block_size
|
||||
data_to_encrypt, self.buff = self.buff[:num_bytes_to_encrypt], self.buff[num_bytes_to_encrypt:]
|
||||
data_to_encrypt, self.buff = split(self.buff, num_bytes_to_encrypt)
|
||||
encrypted_data = self.cipher.encrypt(data_to_encrypt)
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
@@ -103,4 +105,13 @@ class CryptStreamBlobMaker(object):
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
def _return_info(self, blob_hash):
|
||||
return CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
return CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
|
||||
|
||||
def greatest_multiple(a, b):
|
||||
"""return the largest value `c`, that is a multiple of `b` and is <= `a`"""
|
||||
return (a // b) * b
|
||||
|
||||
|
||||
def split(buff, cutoff):
|
||||
return buff[:cutoff], buff[cutoff:]
|
||||
@@ -12,25 +12,29 @@ from lbrynet.core.StreamCreator import StreamCreator
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(StreamCreator):
|
||||
"""Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
Each blob is encrypted with the same key, but each blob has its own initialization vector
|
||||
which is associated with the blob when the blob is associated with the stream."""
|
||||
Each blob is encrypted with the same key, but each blob has its
|
||||
own initialization vector which is associated with the blob when
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""
|
||||
@param blob_manager: Object that stores and provides access to blobs.
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@type blob_manager: BlobManager
|
||||
|
||||
@param name: the name of the stream, which will be presented to the user
|
||||
@type name: string
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
|
||||
be generated.
|
||||
@param key: the raw AES key which will be used to encrypt the
|
||||
blobs. If None, a random key will be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
|
||||
once for each blob.
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: None
|
||||
@@ -57,22 +61,20 @@ class CryptStreamCreator(StreamCreator):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _finalize(self):
|
||||
logging.debug("_finalize has been called")
|
||||
log.debug("_finalize has been called")
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
final_blob_creator = self.blob_manager.get_blob_creator()
|
||||
logging.debug("Created the finished_deferred")
|
||||
log.debug("Created the finished_deferred")
|
||||
final_blob = self._get_blob_maker(iv, final_blob_creator)
|
||||
logging.debug("Created the final blob")
|
||||
logging.debug("Calling close on final blob")
|
||||
log.debug("Created the final blob")
|
||||
log.debug("Calling close on final blob")
|
||||
d = final_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
self.finished_deferreds.append(d)
|
||||
logging.debug("called close on final blob, returning from make_final_blob")
|
||||
return d
|
||||
log.debug("called close on final blob, returning from make_final_blob")
|
||||
|
||||
def _write(self, data):
|
||||
|
||||
def close_blob(blob):
|
||||
d = blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
@@ -91,4 +93,4 @@ class CryptStreamCreator(StreamCreator):
|
||||
self.current_blob = None
|
||||
|
||||
def _get_blob_maker(self, iv, blob_creator):
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
@@ -5,4 +5,4 @@ Crypt Streams are encrypted blobs and metadata tying those blobs together. At le
|
||||
metadata is generally stored in a Stream Descriptor File, for example containing a public key
|
||||
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
|
||||
may or may not be present.
|
||||
"""
|
||||
"""
|
||||
@@ -14,6 +14,7 @@ class CryptBlobHandler(object):
|
||||
######## IBlobHandler #########
|
||||
|
||||
def handle_blob(self, blob, blob_info):
|
||||
blob_decryptor = StreamBlobDecryptor(blob, self.key, binascii.unhexlify(blob_info.iv), blob_info.length)
|
||||
blob_decryptor = StreamBlobDecryptor(
|
||||
blob, self.key, binascii.unhexlify(blob_info.iv), blob_info.length)
|
||||
d = blob_decryptor.decrypt(self.write_func)
|
||||
return d
|
||||
return d
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IStreamDownloader
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
@@ -9,6 +10,9 @@ from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartFailedError(Exception):
|
||||
pass
|
||||
|
||||
@@ -34,21 +38,22 @@ class CryptStreamDownloader(object):
|
||||
implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, upload_allowed):
|
||||
"""
|
||||
Initialize a CryptStreamDownloader
|
||||
payment_rate_manager, wallet):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@param peer_finder: An object which implements the IPeerFinder interface. Used to look up peers by a hashsum.
|
||||
@param peer_finder: An object which implements the IPeerFinder
|
||||
interface. Used to look up peers by a hashsum.
|
||||
|
||||
@param rate_limiter: An object which implements the IRateLimiter interface
|
||||
|
||||
@param blob_manager: A BlobManager object
|
||||
|
||||
@param payment_rate_manager: A PaymentRateManager object
|
||||
@param payment_rate_manager: A NegotiatedPaymentRateManager object
|
||||
|
||||
@param wallet: An object which implements the ILBRYWallet interface
|
||||
@param wallet: An object which implements the IWallet interface
|
||||
|
||||
@return:
|
||||
|
||||
"""
|
||||
|
||||
self.peer_finder = peer_finder
|
||||
@@ -56,20 +61,19 @@ class CryptStreamDownloader(object):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.upload_allowed = upload_allowed
|
||||
|
||||
self.key = None
|
||||
self.stream_name = None
|
||||
|
||||
self.completed = False
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
self.starting = False
|
||||
|
||||
self.download_manager = None
|
||||
self.finished_deferred = None
|
||||
|
||||
self.points_paid = 0.0
|
||||
self.blob_requester = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stream_name)
|
||||
|
||||
def toggle_running(self):
|
||||
if self.stopped is True:
|
||||
@@ -78,11 +82,6 @@ class CryptStreamDownloader(object):
|
||||
return self.stop()
|
||||
|
||||
def start(self):
|
||||
|
||||
def set_finished_deferred():
|
||||
self.finished_deferred = defer.Deferred()
|
||||
return self.finished_deferred
|
||||
|
||||
if self.starting is True:
|
||||
raise CurrentlyStartingError()
|
||||
if self.stopping is True:
|
||||
@@ -92,29 +91,25 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is None
|
||||
self.starting = True
|
||||
self.completed = False
|
||||
self.finished_deferred = defer.Deferred()
|
||||
d = self._start()
|
||||
d.addCallback(lambda _: set_finished_deferred())
|
||||
d.addCallback(lambda _: self.finished_deferred)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
|
||||
def check_if_stop_succeeded(success):
|
||||
self.stopping = False
|
||||
if success is True:
|
||||
self.stopped = True
|
||||
self._remove_download_manager()
|
||||
return success
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None):
|
||||
if self.stopped is True:
|
||||
raise AlreadyStoppedError()
|
||||
if self.stopping is True:
|
||||
raise CurrentlyStoppingError()
|
||||
assert self.download_manager is not None
|
||||
self.stopping = True
|
||||
d = self.download_manager.stop_downloading()
|
||||
self._fire_completed_deferred()
|
||||
d.addCallback(check_if_stop_succeeded)
|
||||
return d
|
||||
success = yield self.download_manager.stop_downloading()
|
||||
self.stopping = False
|
||||
if success is True:
|
||||
self.stopped = True
|
||||
self._remove_download_manager()
|
||||
yield self._fire_completed_deferred(err)
|
||||
|
||||
def _start_failed(self):
|
||||
|
||||
@@ -149,28 +144,32 @@ class CryptStreamDownloader(object):
|
||||
return d
|
||||
|
||||
def _get_download_manager(self):
|
||||
download_manager = DownloadManager(self.blob_manager, self.upload_allowed)
|
||||
assert self.blob_requester is None
|
||||
download_manager = DownloadManager(self.blob_manager)
|
||||
# TODO: can we get rid of these circular references. I'm not
|
||||
# smart enough to handle thinking about the interactions
|
||||
# between them and have hope that there is a simpler way
|
||||
# to accomplish what we want
|
||||
download_manager.blob_info_finder = self._get_metadata_handler(download_manager)
|
||||
download_manager.blob_requester = self._get_blob_requester(download_manager)
|
||||
download_manager.progress_manager = self._get_progress_manager(download_manager)
|
||||
download_manager.blob_handler = self._get_blob_handler(download_manager)
|
||||
download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
|
||||
# blob_requester needs to be set before the connection manager is setup
|
||||
self.blob_requester = self._get_blob_requester(download_manager)
|
||||
download_manager.connection_manager = self._get_connection_manager(download_manager)
|
||||
#return DownloadManager(self.blob_manager, self.blob_requester, self.metadata_handler,
|
||||
# self.progress_manager, self.blob_handler, self.connection_manager)
|
||||
return download_manager
|
||||
|
||||
def _remove_download_manager(self):
|
||||
self.download_manager.blob_info_finder = None
|
||||
self.download_manager.blob_requester = None
|
||||
self.download_manager.progress_manager = None
|
||||
self.download_manager.blob_handler = None
|
||||
self.download_manager.wallet_info_exchanger = None
|
||||
self.blob_requester = None
|
||||
self.download_manager.connection_manager = None
|
||||
self.download_manager = None
|
||||
|
||||
def _get_primary_request_creators(self, download_manager):
|
||||
return [download_manager.blob_requester]
|
||||
return [self.blob_requester]
|
||||
|
||||
def _get_secondary_request_creators(self, download_manager):
|
||||
return [download_manager.wallet_info_exchanger]
|
||||
@@ -179,11 +178,13 @@ class CryptStreamDownloader(object):
|
||||
pass
|
||||
|
||||
def _get_blob_requester(self, download_manager):
|
||||
return BlobRequester(self.blob_manager, self.peer_finder, self.payment_rate_manager, self.wallet,
|
||||
return BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
download_manager)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _get_write_func(self):
|
||||
pass
|
||||
@@ -196,10 +197,16 @@ class CryptStreamDownloader(object):
|
||||
self._get_primary_request_creators(download_manager),
|
||||
self._get_secondary_request_creators(download_manager))
|
||||
|
||||
def _fire_completed_deferred(self):
|
||||
def _fire_completed_deferred(self, err=None):
|
||||
self.finished_deferred, d = None, self.finished_deferred
|
||||
if d is not None:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
if err is not None:
|
||||
d.errback(err)
|
||||
else:
|
||||
value = self._get_finished_deferred_callback_value()
|
||||
d.callback(value)
|
||||
else:
|
||||
log.debug("Not firing the completed deferred because d is None")
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
return None
|
||||
@@ -209,5 +216,5 @@ class CryptStreamDownloader(object):
|
||||
self.completed = True
|
||||
return self.stop()
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop(err=err)
|
||||
File renamed without changes.
@@ -0,0 +1,27 @@
|
||||
import logging
|
||||
|
||||
|
||||
def migrate_db(db_dir, start, end):
|
||||
current = start
|
||||
while current < end:
|
||||
if current == 1:
|
||||
from lbrynet.db_migrator.migrate1to2 import do_migration
|
||||
do_migration(db_dir)
|
||||
else:
|
||||
raise Exception(
|
||||
"DB migration of version {} to {} is not available".format(current, current+1))
|
||||
current += 1
|
||||
return None
|
||||
|
||||
|
||||
def run_migration_script():
|
||||
import sys
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
|
||||
sys.stdout = open("migrator.out.log", 'w')
|
||||
sys.stderr = open("migrator.err.log", 'w')
|
||||
migrate_db(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_migration_script()
|
||||
@@ -0,0 +1,77 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
UNSET_NOUT = -1
|
||||
|
||||
def do_migration(db_dir):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
def migrate_blockchainname_db(db_dir):
|
||||
blockchainname_db = os.path.join(db_dir, "blockchainname.db")
|
||||
# skip migration on fresh installs
|
||||
if not os.path.isfile(blockchainname_db):
|
||||
return
|
||||
temp_db = sqlite3.connect(":memory:")
|
||||
db_file = sqlite3.connect(blockchainname_db)
|
||||
file_cursor = db_file.cursor()
|
||||
mem_cursor = temp_db.cursor()
|
||||
|
||||
mem_cursor.execute("create table if not exists name_metadata ("
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer, "
|
||||
" sd_hash text)")
|
||||
mem_cursor.execute("create table if not exists claim_ids ("
|
||||
" claimId text, "
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer)")
|
||||
temp_db.commit()
|
||||
|
||||
name_metadata = file_cursor.execute("select * from name_metadata").fetchall()
|
||||
claim_metadata = file_cursor.execute("select * from claim_ids").fetchall()
|
||||
|
||||
# fill n as V1_UNSET_NOUT, Wallet.py will be responsible for filling in correct n
|
||||
for name, txid, sd_hash in name_metadata:
|
||||
mem_cursor.execute(
|
||||
"insert into name_metadata values (?, ?, ?, ?) ",
|
||||
(name, txid, UNSET_NOUT, sd_hash))
|
||||
|
||||
for claim_id, name, txid in claim_metadata:
|
||||
mem_cursor.execute(
|
||||
"insert into claim_ids values (?, ?, ?, ?)",
|
||||
(claim_id, name, txid, UNSET_NOUT))
|
||||
temp_db.commit()
|
||||
|
||||
new_name_metadata = mem_cursor.execute("select * from name_metadata").fetchall()
|
||||
new_claim_metadata = mem_cursor.execute("select * from claim_ids").fetchall()
|
||||
|
||||
file_cursor.execute("drop table name_metadata")
|
||||
file_cursor.execute("create table name_metadata ("
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer, "
|
||||
" sd_hash text)")
|
||||
|
||||
for name, txid, n, sd_hash in new_name_metadata:
|
||||
file_cursor.execute(
|
||||
"insert into name_metadata values (?, ?, ?, ?) ", (name, txid, n, sd_hash))
|
||||
|
||||
file_cursor.execute("drop table claim_ids")
|
||||
file_cursor.execute("create table claim_ids ("
|
||||
" claimId text, "
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer)")
|
||||
|
||||
for claim_id, name, txid, n in new_claim_metadata:
|
||||
file_cursor.execute("insert into claim_ids values (?, ?, ?, ?)", (claim_id, name, txid, n))
|
||||
|
||||
db_file.commit()
|
||||
db_file.close()
|
||||
temp_db.close()
|
||||
+2
-2
@@ -10,7 +10,7 @@
|
||||
the terms and conditions of version 3 of the GNU General Public
|
||||
License, supplemented by the additional permissions listed below.
|
||||
|
||||
0. Additional Definitions.
|
||||
0. Additional Definitions.
|
||||
|
||||
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||
@@ -111,7 +111,7 @@ the following:
|
||||
a copy of the Library already present on the user's computer
|
||||
system, and (b) will operate properly with a modified version
|
||||
of the Library that is interface-compatible with the Linked
|
||||
Version.
|
||||
Version.
|
||||
|
||||
e) Provide Installation Information, but only if you would otherwise
|
||||
be required to provide such information under section 6 of the
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
""" This module defines the charaterizing constants of the Kademlia network
|
||||
|
||||
C{checkRefreshInterval} and C{udpDatagramMaxSize} are implementation-specific
|
||||
constants, and do not affect general Kademlia operation.
|
||||
constants, and do not affect general Kademlia operation.
|
||||
"""
|
||||
|
||||
######### KADEMLIA CONSTANTS ###########
|
||||
@@ -28,14 +28,14 @@ rpcTimeout = 5
|
||||
iterativeLookupDelay = rpcTimeout / 2
|
||||
|
||||
#: If a k-bucket has not been used for this amount of time, refresh it (in seconds)
|
||||
refreshTimeout = 3600 # 1 hour
|
||||
refreshTimeout = 3600 # 1 hour
|
||||
#: The interval at which nodes replicate (republish/refresh) data they are holding
|
||||
replicateInterval = refreshTimeout
|
||||
# The time it takes for data to expire in the network; the original publisher of the data
|
||||
# will also republish the data at this time if it is still valid
|
||||
dataExpireTimeout = 86400 # 24 hours
|
||||
dataExpireTimeout = 86400 # 24 hours
|
||||
|
||||
tokenSecretChangeInterval = 300 # 5 minutes
|
||||
tokenSecretChangeInterval = 300 # 5 minutes
|
||||
|
||||
peer_request_timeout = 10
|
||||
|
||||
@@ -43,10 +43,13 @@ peer_request_timeout = 10
|
||||
|
||||
#: The interval in which the node should check its whether any buckets need refreshing,
|
||||
#: or whether any data needs to be republished (in seconds)
|
||||
checkRefreshInterval = refreshTimeout/5
|
||||
checkRefreshInterval = refreshTimeout / 5
|
||||
|
||||
#: Max size of a single UDP datagram, in bytes. If a message is larger than this, it will
|
||||
#: be spread accross several UDP packets.
|
||||
udpDatagramMaxSize = 8192 # 8 KB
|
||||
#: be spread across several UDP packets.
|
||||
udpDatagramMaxSize = 8192 # 8 KB
|
||||
|
||||
key_bits = 384
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
h = get_lbry_hash_obj()
|
||||
key_bits = h.digest_size * 8 # 384 bits
|
||||
+14
-9
@@ -10,17 +10,18 @@
|
||||
|
||||
class Contact(object):
|
||||
""" Encapsulation for remote contact
|
||||
|
||||
|
||||
This class contains information on a single remote contact, and also
|
||||
provides a direct RPC API to the remote node which it represents
|
||||
"""
|
||||
|
||||
def __init__(self, id, ipAddress, udpPort, networkProtocol, firstComm=0):
|
||||
self.id = id
|
||||
self.address = ipAddress
|
||||
self.port = udpPort
|
||||
self._networkProtocol = networkProtocol
|
||||
self.commTime = firstComm
|
||||
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, Contact):
|
||||
return self.id == other.id
|
||||
@@ -28,7 +29,7 @@ class Contact(object):
|
||||
return self.id == other
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def __ne__(self, other):
|
||||
if isinstance(other, Contact):
|
||||
return self.id != other.id
|
||||
@@ -38,26 +39,30 @@ class Contact(object):
|
||||
return True
|
||||
|
||||
def compact_ip(self):
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), self.address.split('.'), bytearray())
|
||||
compact_ip = reduce(
|
||||
lambda buff, x: buff + bytearray([int(x)]), self.address.split('.'), bytearray())
|
||||
return str(compact_ip)
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return '<%s.%s object; IP address: %s, UDP port: %d>' % (self.__module__, self.__class__.__name__, self.address, self.port)
|
||||
|
||||
return '<%s.%s object; IP address: %s, UDP port: %d>' % (
|
||||
self.__module__, self.__class__.__name__, self.address, self.port)
|
||||
|
||||
def __getattr__(self, name):
|
||||
""" This override allows the host node to call a method of the remote
|
||||
node (i.e. this contact) as if it was a local function.
|
||||
|
||||
|
||||
For instance, if C{remoteNode} is a instance of C{Contact}, the
|
||||
following will result in C{remoteNode}'s C{test()} method to be
|
||||
called with argument C{123}::
|
||||
remoteNode.test(123)
|
||||
|
||||
|
||||
Such a RPC method call will return a Deferred, which will callback
|
||||
when the contact responds with the result (or an error occurs).
|
||||
This happens via this contact's C{_networkProtocol} object (i.e. the
|
||||
host Node's C{_protocol} object).
|
||||
"""
|
||||
|
||||
def _sendRPC(*args, **kwargs):
|
||||
return self._networkProtocol.sendRPC(self, name, args, **kwargs)
|
||||
|
||||
return _sendRPC
|
||||
+5
-152
@@ -8,63 +8,27 @@
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import UserDict
|
||||
#import sqlite3
|
||||
import cPickle as pickle
|
||||
import time
|
||||
import os
|
||||
import constants
|
||||
|
||||
|
||||
|
||||
class DataStore(UserDict.DictMixin):
|
||||
""" Interface for classes implementing physical storage (for data
|
||||
published via the "STORE" RPC) for the Kademlia DHT
|
||||
|
||||
|
||||
@note: This provides an interface for a dict-like object
|
||||
"""
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
|
||||
# def lastPublished(self, key):
|
||||
# """ Get the time the C{(key, value)} pair identified by C{key}
|
||||
# was last published """
|
||||
|
||||
# def originalPublisherID(self, key):
|
||||
# """ Get the original publisher of the data's node ID
|
||||
#
|
||||
# @param key: The key that identifies the stored data
|
||||
# @type key: str
|
||||
#
|
||||
# @return: Return the node ID of the original publisher of the
|
||||
# C{(key, value)} pair identified by C{key}.
|
||||
# """
|
||||
|
||||
# def originalPublishTime(self, key):
|
||||
# """ Get the time the C{(key, value)} pair identified by C{key}
|
||||
# was originally published """
|
||||
|
||||
# def setItem(self, key, value, lastPublished, originallyPublished, originalPublisherID):
|
||||
# """ Set the value of the (key, value) pair identified by C{key};
|
||||
# this should set the "last published" value for the (key, value)
|
||||
# pair to the current time
|
||||
# """
|
||||
|
||||
def addPeerToBlob(self, key, value, lastPublished, originallyPublished, originalPublisherID):
|
||||
pass
|
||||
|
||||
# def __getitem__(self, key):
|
||||
# """ Get the value identified by C{key} """
|
||||
|
||||
# def __setitem__(self, key, value):
|
||||
# """ Convenience wrapper to C{setItem}; this accepts a tuple in the
|
||||
# format: (value, lastPublished, originallyPublished, originalPublisherID) """
|
||||
# self.setItem(key, *value)
|
||||
|
||||
# def __delitem__(self, key):
|
||||
# """ Delete the specified key (and its value) """
|
||||
|
||||
class DictDataStore(DataStore):
|
||||
""" A datastore using an in-memory Python dictionary """
|
||||
|
||||
def __init__(self):
|
||||
# Dictionary format:
|
||||
# { <key>: (<value>, <lastPublished>, <originallyPublished> <originalPublisherID>) }
|
||||
@@ -74,33 +38,14 @@ class DictDataStore(DataStore):
|
||||
""" Return a list of the keys in this data store """
|
||||
return self._dict.keys()
|
||||
|
||||
# def lastPublished(self, key):
|
||||
# """ Get the time the C{(key, value)} pair identified by C{key}
|
||||
# was last published """
|
||||
# return self._dict[key][1]
|
||||
|
||||
# def originalPublisherID(self, key):
|
||||
# """ Get the original publisher of the data's node ID
|
||||
#
|
||||
# @param key: The key that identifies the stored data
|
||||
# @type key: str
|
||||
#
|
||||
# @return: Return the node ID of the original publisher of the
|
||||
# C{(key, value)} pair identified by C{key}.
|
||||
# """
|
||||
# return self._dict[key][3]
|
||||
|
||||
# def originalPublishTime(self, key):
|
||||
# """ Get the time the C{(key, value)} pair identified by C{key}
|
||||
# was originally published """
|
||||
# return self._dict[key][2]
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
now = int(time.time())
|
||||
|
||||
def notExpired(peer):
|
||||
if (now - peer[2]) > constants.dataExpireTimeout:
|
||||
return False
|
||||
return True
|
||||
|
||||
for key in self._dict.keys():
|
||||
unexpired_peers = filter(notExpired, self._dict[key])
|
||||
self._dict[key] = unexpired_peers
|
||||
@@ -119,95 +64,3 @@ class DictDataStore(DataStore):
|
||||
def getPeersForBlob(self, key):
|
||||
if key in self._dict:
|
||||
return [val[0] for val in self._dict[key]]
|
||||
|
||||
# def setItem(self, key, value, lastPublished, originallyPublished, originalPublisherID):
|
||||
# """ Set the value of the (key, value) pair identified by C{key};
|
||||
# this should set the "last published" value for the (key, value)
|
||||
# pair to the current time
|
||||
# """
|
||||
# self._dict[key] = (value, lastPublished, originallyPublished, originalPublisherID)
|
||||
|
||||
# def __getitem__(self, key):
|
||||
# """ Get the value identified by C{key} """
|
||||
# return self._dict[key][0]
|
||||
|
||||
# def __delitem__(self, key):
|
||||
# """ Delete the specified key (and its value) """
|
||||
# del self._dict[key]
|
||||
|
||||
|
||||
#class SQLiteDataStore(DataStore):
|
||||
# """ Example of a SQLite database-based datastore
|
||||
# """
|
||||
# def __init__(self, dbFile=':memory:'):
|
||||
# """
|
||||
# @param dbFile: The name of the file containing the SQLite database; if
|
||||
# unspecified, an in-memory database is used.
|
||||
# @type dbFile: str
|
||||
# """
|
||||
# createDB = not os.path.exists(dbFile)
|
||||
# self._db = sqlite3.connect(dbFile)
|
||||
# self._db.isolation_level = None
|
||||
# self._db.text_factory = str
|
||||
# if createDB:
|
||||
# self._db.execute('CREATE TABLE data(key, value, lastPublished, originallyPublished, originalPublisherID)')
|
||||
# self._cursor = self._db.cursor()
|
||||
|
||||
# def keys(self):
|
||||
# """ Return a list of the keys in this data store """
|
||||
# keys = []
|
||||
# try:
|
||||
# self._cursor.execute("SELECT key FROM data")
|
||||
# for row in self._cursor:
|
||||
# keys.append(row[0].decode('hex'))
|
||||
# finally:
|
||||
# return keys
|
||||
|
||||
# def lastPublished(self, key):
|
||||
# """ Get the time the C{(key, value)} pair identified by C{key}
|
||||
# was last published """
|
||||
# return int(self._dbQuery(key, 'lastPublished'))
|
||||
|
||||
# def originalPublisherID(self, key):
|
||||
# """ Get the original publisher of the data's node ID
|
||||
|
||||
# @param key: The key that identifies the stored data
|
||||
# @type key: str
|
||||
|
||||
# @return: Return the node ID of the original publisher of the
|
||||
# C{(key, value)} pair identified by C{key}.
|
||||
# """
|
||||
# return self._dbQuery(key, 'originalPublisherID')
|
||||
|
||||
# def originalPublishTime(self, key):
|
||||
# """ Get the time the C{(key, value)} pair identified by C{key}
|
||||
# was originally published """
|
||||
# return int(self._dbQuery(key, 'originallyPublished'))
|
||||
|
||||
# def setItem(self, key, value, lastPublished, originallyPublished, originalPublisherID):
|
||||
# # Encode the key so that it doesn't corrupt the database
|
||||
# encodedKey = key.encode('hex')
|
||||
# self._cursor.execute("select key from data where key=:reqKey", {'reqKey': encodedKey})
|
||||
# if self._cursor.fetchone() == None:
|
||||
# self._cursor.execute('INSERT INTO data(key, value, lastPublished, originallyPublished, originalPublisherID) VALUES (?, ?, ?, ?, ?)', (encodedKey, buffer(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)), lastPublished, originallyPublished, originalPublisherID))
|
||||
# else:
|
||||
# self._cursor.execute('UPDATE data SET value=?, lastPublished=?, originallyPublished=?, originalPublisherID=? WHERE key=?', (buffer(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)), lastPublished, originallyPublished, originalPublisherID, encodedKey))
|
||||
|
||||
# def _dbQuery(self, key, columnName, unpickle=False):
|
||||
# try:
|
||||
# self._cursor.execute("SELECT %s FROM data WHERE key=:reqKey" % columnName, {'reqKey': key.encode('hex')})
|
||||
# row = self._cursor.fetchone()
|
||||
# value = str(row[0])
|
||||
# except TypeError:
|
||||
# raise KeyError, key
|
||||
# else:
|
||||
# if unpickle:
|
||||
# return pickle.loads(value)
|
||||
# else:
|
||||
# return value
|
||||
|
||||
# def __getitem__(self, key):
|
||||
# return self._dbQuery(key, 'value', unpickle=True)
|
||||
|
||||
# def __delitem__(self, key):
|
||||
# self._cursor.execute("DELETE FROM data WHERE key=:reqKey", {'reqKey': key.encode('hex')})
|
||||
+39
-29
@@ -7,54 +7,59 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
|
||||
class DecodeError(Exception):
|
||||
""" Should be raised by an C{Encoding} implementation if decode operation
|
||||
fails
|
||||
"""
|
||||
|
||||
|
||||
class Encoding(object):
|
||||
""" Interface for RPC message encoders/decoders
|
||||
|
||||
|
||||
All encoding implementations used with this library should inherit and
|
||||
implement this.
|
||||
"""
|
||||
|
||||
def encode(self, data):
|
||||
""" Encode the specified data
|
||||
|
||||
|
||||
@param data: The data to encode
|
||||
This method has to support encoding of the following
|
||||
types: C{str}, C{int} and C{long}
|
||||
Any additional data types may be supported as long as the
|
||||
implementing class's C{decode()} method can successfully
|
||||
decode them.
|
||||
|
||||
|
||||
@return: The encoded data
|
||||
@rtype: str
|
||||
"""
|
||||
|
||||
def decode(self, data):
|
||||
""" Decode the specified data string
|
||||
|
||||
|
||||
@param data: The data (byte string) to decode.
|
||||
@type data: str
|
||||
|
||||
|
||||
@return: The decoded data (in its correct type)
|
||||
"""
|
||||
|
||||
|
||||
class Bencode(Encoding):
|
||||
""" Implementation of a Bencode-based algorithm (Bencode is the encoding
|
||||
algorithm used by Bittorrent).
|
||||
|
||||
|
||||
@note: This algorithm differs from the "official" Bencode algorithm in
|
||||
that it can encode/decode floating point values in addition to
|
||||
integers.
|
||||
"""
|
||||
|
||||
|
||||
def encode(self, data):
|
||||
""" Encoder implementation of the Bencode algorithm
|
||||
|
||||
|
||||
@param data: The data to encode
|
||||
@type data: int, long, tuple, list, dict or str
|
||||
|
||||
|
||||
@return: The encoded data
|
||||
@rtype: str
|
||||
"""
|
||||
@@ -76,47 +81,51 @@ class Bencode(Encoding):
|
||||
encodedDictItems += self.encode(data[key])
|
||||
return 'd%se' % encodedDictItems
|
||||
elif type(data) == float:
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
return 'f%fe' % data
|
||||
elif data == None:
|
||||
# This (None/NULL data type) is a non-standard extension to the original Bencode algorithm
|
||||
# This (None/NULL data type) is a non-standard extension
|
||||
# to the original Bencode algorithm
|
||||
return 'n'
|
||||
else:
|
||||
print data
|
||||
raise TypeError, "Cannot bencode '%s' object" % type(data)
|
||||
|
||||
|
||||
def decode(self, data):
|
||||
""" Decoder implementation of the Bencode algorithm
|
||||
|
||||
""" Decoder implementation of the Bencode algorithm
|
||||
|
||||
@param data: The encoded data
|
||||
@type data: str
|
||||
|
||||
|
||||
@note: This is a convenience wrapper for the recursive decoding
|
||||
algorithm, C{_decodeRecursive}
|
||||
|
||||
|
||||
@return: The decoded data, as a native Python type
|
||||
@rtype: int, list, dict or str
|
||||
"""
|
||||
if len(data) == 0:
|
||||
raise DecodeError, 'Cannot decode empty string'
|
||||
return self._decodeRecursive(data)[0]
|
||||
|
||||
try:
|
||||
return self._decodeRecursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError, e.message
|
||||
|
||||
@staticmethod
|
||||
def _decodeRecursive(data, startIndex=0):
|
||||
""" Actual implementation of the recursive Bencode algorithm
|
||||
|
||||
|
||||
Do not call this; use C{decode()} instead
|
||||
"""
|
||||
if data[startIndex] == 'i':
|
||||
endPos = data[startIndex:].find('e')+startIndex
|
||||
return (int(data[startIndex+1:endPos]), endPos+1)
|
||||
endPos = data[startIndex:].find('e') + startIndex
|
||||
return (int(data[startIndex + 1:endPos]), endPos + 1)
|
||||
elif data[startIndex] == 'l':
|
||||
startIndex += 1
|
||||
decodedList = []
|
||||
while data[startIndex] != 'e':
|
||||
listData, startIndex = Bencode._decodeRecursive(data, startIndex)
|
||||
decodedList.append(listData)
|
||||
return (decodedList, startIndex+1)
|
||||
return (decodedList, startIndex + 1)
|
||||
elif data[startIndex] == 'd':
|
||||
startIndex += 1
|
||||
decodedDict = {}
|
||||
@@ -127,18 +136,19 @@ class Bencode(Encoding):
|
||||
return (decodedDict, startIndex)
|
||||
elif data[startIndex] == 'f':
|
||||
# This (float data type) is a non-standard extension to the original Bencode algorithm
|
||||
endPos = data[startIndex:].find('e')+startIndex
|
||||
return (float(data[startIndex+1:endPos]), endPos+1)
|
||||
endPos = data[startIndex:].find('e') + startIndex
|
||||
return (float(data[startIndex + 1:endPos]), endPos + 1)
|
||||
elif data[startIndex] == 'n':
|
||||
# This (None/NULL data type) is a non-standard extension to the original Bencode algorithm
|
||||
return (None, startIndex+1)
|
||||
# This (None/NULL data type) is a non-standard extension
|
||||
# to the original Bencode algorithm
|
||||
return (None, startIndex + 1)
|
||||
else:
|
||||
splitPos = data[startIndex:].find(':')+startIndex
|
||||
splitPos = data[startIndex:].find(':') + startIndex
|
||||
try:
|
||||
length = int(data[startIndex:splitPos])
|
||||
except ValueError, e:
|
||||
raise DecodeError, e
|
||||
startIndex = splitPos+1
|
||||
endPos = startIndex+length
|
||||
startIndex = splitPos + 1
|
||||
endPos = startIndex + length
|
||||
bytes = data[startIndex:endPos]
|
||||
return (bytes, endPos)
|
||||
@@ -1,10 +1,9 @@
|
||||
|
||||
from collections import Counter
|
||||
import datetime
|
||||
|
||||
|
||||
class HashWatcher():
|
||||
def __init__(self, ttl=600):
|
||||
class HashWatcher(object):
|
||||
def __init__(self):
|
||||
self.ttl = 600
|
||||
self.hashes = []
|
||||
self.next_tick = None
|
||||
@@ -32,4 +31,4 @@ class HashWatcher():
|
||||
|
||||
def _remove_old_hashes(self):
|
||||
remove_time = datetime.datetime.now() - datetime.timedelta(minutes=10)
|
||||
self.hashes = [h for h in self.hashes if h[1] < remove_time]
|
||||
self.hashes = [h for h in self.hashes if h[1] < remove_time]
|
||||
+17
-14
@@ -9,6 +9,7 @@
|
||||
|
||||
import constants
|
||||
|
||||
|
||||
class BucketFull(Exception):
|
||||
""" Raised when the bucket is full """
|
||||
|
||||
@@ -16,6 +17,7 @@ class BucketFull(Exception):
|
||||
class KBucket(object):
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
def __init__(self, rangeMin, rangeMax):
|
||||
"""
|
||||
@param rangeMin: The lower boundary for the range in the n-bit ID
|
||||
@@ -31,17 +33,18 @@ class KBucket(object):
|
||||
def addContact(self, contact):
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
contact to the end of the k-bucket if it is already present.
|
||||
|
||||
|
||||
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
|
||||
the contact isn't in the bucket
|
||||
already
|
||||
|
||||
|
||||
@param contact: The contact to add
|
||||
@type contact: kademlia.contact.Contact
|
||||
"""
|
||||
if contact in self._contacts:
|
||||
# Move the existing contact to the end of the list
|
||||
# - using the new contact to allow add-on data (e.g. optimization-specific stuff) to pe updated as well
|
||||
# - using the new contact to allow add-on data
|
||||
# (e.g. optimization-specific stuff) to pe updated as well
|
||||
self._contacts.remove(contact)
|
||||
self._contacts.append(contact)
|
||||
elif len(self._contacts) < constants.k:
|
||||
@@ -56,7 +59,7 @@ class KBucket(object):
|
||||
|
||||
def getContacts(self, count=-1, excludeContact=None):
|
||||
""" Returns a list containing up to the first count number of contacts
|
||||
|
||||
|
||||
@param count: The amount of contacts to return (if 0 or less, return
|
||||
all contacts)
|
||||
@type count: int
|
||||
@@ -64,12 +67,12 @@ class KBucket(object):
|
||||
the list of returned values, it will be
|
||||
discarded before returning. If a C{str} is
|
||||
passed as this argument, it must be the
|
||||
contact's ID.
|
||||
contact's ID.
|
||||
@type excludeContact: kademlia.contact.Contact or str
|
||||
|
||||
|
||||
|
||||
|
||||
@raise IndexError: If the number of requested contacts is too large
|
||||
|
||||
|
||||
@return: Return up to the first count number of contacts in a list
|
||||
If no contacts are present an empty is returned
|
||||
@rtype: list
|
||||
@@ -96,7 +99,7 @@ class KBucket(object):
|
||||
# enough contacts in list
|
||||
else:
|
||||
contactList = self._contacts[0:count]
|
||||
|
||||
|
||||
if excludeContact in contactList:
|
||||
contactList.remove(excludeContact)
|
||||
|
||||
@@ -104,24 +107,24 @@ class KBucket(object):
|
||||
|
||||
def removeContact(self, contact):
|
||||
""" Remove given contact from list
|
||||
|
||||
|
||||
@param contact: The contact to remove, or a string containing the
|
||||
contact's node ID
|
||||
@type contact: kademlia.contact.Contact or str
|
||||
|
||||
|
||||
@raise ValueError: The specified contact is not in this bucket
|
||||
"""
|
||||
self._contacts.remove(contact)
|
||||
|
||||
|
||||
def keyInRange(self, key):
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
of the n-bit ID space covered by this k-bucket (in otherwords, it
|
||||
returns whether or not the specified key should be placed in this
|
||||
k-bucket)
|
||||
|
||||
|
||||
@param key: The key to test
|
||||
@type key: str or int
|
||||
|
||||
|
||||
@return: C{True} if the key is in this k-bucket's range, or C{False}
|
||||
if not.
|
||||
@rtype: bool
|
||||
|
||||
+25
-14
@@ -9,54 +9,65 @@
|
||||
|
||||
import msgtypes
|
||||
|
||||
|
||||
class MessageTranslator(object):
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
|
||||
Classes inheriting from this should provide a translation services between
|
||||
the classes used internally by this Kademlia implementation and the actual
|
||||
data that is transmitted between nodes.
|
||||
"""
|
||||
|
||||
def fromPrimitive(self, msgPrimitive):
|
||||
""" Create an RPC Message from a message's string representation
|
||||
|
||||
|
||||
@param msgPrimitive: The unencoded primitive representation of a message
|
||||
@type msgPrimitive: str, int, list or dict
|
||||
|
||||
|
||||
@return: The translated message object
|
||||
@rtype: entangled.kademlia.msgtypes.Message
|
||||
"""
|
||||
|
||||
|
||||
def toPrimitive(self, message):
|
||||
""" Create a string representation of a message
|
||||
|
||||
|
||||
@param message: The message object
|
||||
@type message: msgtypes.Message
|
||||
|
||||
|
||||
@return: The message's primitive representation in a particular
|
||||
messaging format
|
||||
@rtype: str, int, list or dict
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class DefaultFormat(MessageTranslator):
|
||||
""" The default on-the-wire message format for this library """
|
||||
typeRequest, typeResponse, typeError = range(3)
|
||||
headerType, headerMsgID, headerNodeID, headerPayload, headerArgs = range(5)
|
||||
|
||||
|
||||
def fromPrimitive(self, msgPrimitive):
|
||||
msgType = msgPrimitive[self.headerType]
|
||||
if msgType == self.typeRequest:
|
||||
msg = msgtypes.RequestMessage(msgPrimitive[self.headerNodeID], msgPrimitive[self.headerPayload], msgPrimitive[self.headerArgs], msgPrimitive[self.headerMsgID])
|
||||
msg = msgtypes.RequestMessage(msgPrimitive[self.headerNodeID],
|
||||
msgPrimitive[self.headerPayload],
|
||||
msgPrimitive[self.headerArgs],
|
||||
msgPrimitive[self.headerMsgID])
|
||||
elif msgType == self.typeResponse:
|
||||
msg = msgtypes.ResponseMessage(msgPrimitive[self.headerMsgID], msgPrimitive[self.headerNodeID], msgPrimitive[self.headerPayload])
|
||||
msg = msgtypes.ResponseMessage(msgPrimitive[self.headerMsgID],
|
||||
msgPrimitive[self.headerNodeID],
|
||||
msgPrimitive[self.headerPayload])
|
||||
elif msgType == self.typeError:
|
||||
msg = msgtypes.ErrorMessage(msgPrimitive[self.headerMsgID], msgPrimitive[self.headerNodeID], msgPrimitive[self.headerPayload], msgPrimitive[self.headerArgs])
|
||||
msg = msgtypes.ErrorMessage(msgPrimitive[self.headerMsgID],
|
||||
msgPrimitive[self.headerNodeID],
|
||||
msgPrimitive[self.headerPayload],
|
||||
msgPrimitive[self.headerArgs])
|
||||
else:
|
||||
# Unknown message, no payload
|
||||
msg = msgtypes.Message(msgPrimitive[self.headerMsgID], msgPrimitive[self.headerNodeID])
|
||||
return msg
|
||||
|
||||
def toPrimitive(self, message):
|
||||
msg = {self.headerMsgID: message.id,
|
||||
|
||||
def toPrimitive(self, message):
|
||||
msg = {self.headerMsgID: message.id,
|
||||
self.headerNodeID: message.nodeID}
|
||||
if isinstance(message, msgtypes.RequestMessage):
|
||||
msg[self.headerType] = self.typeRequest
|
||||
|
||||
@@ -7,11 +7,12 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import hashlib
|
||||
import random
|
||||
from lbrynet.core.utils import generate_id
|
||||
|
||||
|
||||
class Message(object):
|
||||
""" Base class for messages - all "unknown" messages use this class """
|
||||
|
||||
def __init__(self, rpcID, nodeID):
|
||||
self.id = rpcID
|
||||
self.nodeID = nodeID
|
||||
@@ -19,11 +20,10 @@ class Message(object):
|
||||
|
||||
class RequestMessage(Message):
|
||||
""" Message containing an RPC request """
|
||||
|
||||
def __init__(self, nodeID, method, methodArgs, rpcID=None):
|
||||
if rpcID == None:
|
||||
hash = hashlib.sha384()
|
||||
hash.update(str(random.getrandbits(255)))
|
||||
rpcID = hash.digest()
|
||||
rpcID = generate_id()
|
||||
Message.__init__(self, rpcID, nodeID)
|
||||
self.request = method
|
||||
self.args = methodArgs
|
||||
@@ -31,6 +31,7 @@ class RequestMessage(Message):
|
||||
|
||||
class ResponseMessage(Message):
|
||||
""" Message containing the result from a successful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, response):
|
||||
Message.__init__(self, rpcID, nodeID)
|
||||
self.response = response
|
||||
@@ -38,6 +39,7 @@ class ResponseMessage(Message):
|
||||
|
||||
class ErrorMessage(ResponseMessage):
|
||||
""" Message containing the error from an unsuccessful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
|
||||
ResponseMessage.__init__(self, rpcID, nodeID, errorMessage)
|
||||
if isinstance(exceptionType, type):
|
||||
|
||||
+361
-471
File diff suppressed because it is too large.
Load diff
+140
-76
@@ -7,10 +7,15 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import logging
|
||||
import binascii
|
||||
import time
|
||||
import socket
|
||||
import errno
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
from twisted.python import failure
|
||||
from twisted.internet import error
|
||||
import twisted.internet.reactor
|
||||
|
||||
import constants
|
||||
@@ -20,25 +25,57 @@ import msgformat
|
||||
from contact import Contact
|
||||
|
||||
reactor = twisted.internet.reactor
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TimeoutError(Exception):
|
||||
""" Raised when a RPC times out """
|
||||
|
||||
def __init__(self, remote_contact_id):
|
||||
# remote_contact_id is a binary blob so we need to convert it
|
||||
# into something more readable
|
||||
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
|
||||
Exception.__init__(self, msg)
|
||||
self.remote_contact_id = remote_contact_id
|
||||
|
||||
|
||||
class Delay(object):
|
||||
maxToSendDelay = 10 ** -3 # 0.05
|
||||
minToSendDelay = 10 ** -5 # 0.01
|
||||
|
||||
def __init__(self, start=0):
|
||||
self._next = start
|
||||
|
||||
# TODO: explain why this logic is like it is. And add tests that
|
||||
# show that it actually does what it needs to do.
|
||||
def __call__(self):
|
||||
ts = time.time()
|
||||
delay = 0
|
||||
if ts >= self._next:
|
||||
delay = self.minToSendDelay
|
||||
self._next = ts + self.minToSendDelay
|
||||
else:
|
||||
delay = (self._next - ts) + self.maxToSendDelay
|
||||
self._next += self.maxToSendDelay
|
||||
return delay
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
msgSizeLimit = constants.udpDatagramMaxSize-26
|
||||
maxToSendDelay = 10**-3#0.05
|
||||
minToSendDelay = 10**-5#0.01
|
||||
msgSizeLimit = constants.udpDatagramMaxSize - 26
|
||||
|
||||
def __init__(self, node, msgEncoder=encoding.Bencode(), msgTranslator=msgformat.DefaultFormat()):
|
||||
def __init__(self, node, msgEncoder=encoding.Bencode(),
|
||||
msgTranslator=msgformat.DefaultFormat()):
|
||||
self._node = node
|
||||
self._encoder = msgEncoder
|
||||
self._translator = msgTranslator
|
||||
self._sentMessages = {}
|
||||
self._partialMessages = {}
|
||||
self._partialMessagesProgress = {}
|
||||
self._next = 0
|
||||
self._callLaterList = {}
|
||||
self._delay = Delay()
|
||||
# keep track of outstanding writes so that they
|
||||
# can be cancelled on shutdown
|
||||
self._call_later_list = {}
|
||||
|
||||
def sendRPC(self, contact, method, args, rawResponse=False):
|
||||
""" Sends an RPC to the specified contact
|
||||
@@ -71,12 +108,15 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = self._encoder.encode(msgPrimitive)
|
||||
|
||||
log.debug("DHT SEND: %s(%s)", method, args)
|
||||
|
||||
df = defer.Deferred()
|
||||
if rawResponse:
|
||||
df._rpcRawResponse = True
|
||||
|
||||
# Set the RPC timeout timer
|
||||
timeoutCall = reactor.callLater(constants.rpcTimeout, self._msgTimeout, msg.id) #IGNORE:E1101
|
||||
timeoutCall = reactor.callLater(
|
||||
constants.rpcTimeout, self._msgTimeout, msg.id) # IGNORE:E1101
|
||||
# Transmit the data
|
||||
self._send(encodedMsg, msg.id, (contact.address, contact.port))
|
||||
self._sentMessages[msg.id] = (contact.id, df, timeoutCall)
|
||||
@@ -110,10 +150,13 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
except encoding.DecodeError:
|
||||
# We received some rubbish here
|
||||
return
|
||||
|
||||
except IndexError:
|
||||
log.warning("Couldn't decode dht datagram from %s", address)
|
||||
return
|
||||
|
||||
message = self._translator.fromPrimitive(msgPrimitive)
|
||||
remoteContact = Contact(message.nodeID, address[0], address[1], self)
|
||||
|
||||
|
||||
# Refresh the remote node's details in the local node's k-buckets
|
||||
self._node.addContact(remoteContact)
|
||||
|
||||
@@ -122,14 +165,16 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
self._handleRPC(remoteContact, message.id, message.request, message.args)
|
||||
elif isinstance(message, msgtypes.ResponseMessage):
|
||||
# Find the message that triggered this response
|
||||
if self._sentMessages.has_key(message.id):
|
||||
if message.id in self._sentMessages:
|
||||
# Cancel timeout timer for this RPC
|
||||
df, timeoutCall = self._sentMessages[message.id][1:3]
|
||||
timeoutCall.cancel()
|
||||
del self._sentMessages[message.id]
|
||||
|
||||
if hasattr(df, '_rpcRawResponse'):
|
||||
# The RPC requested that the raw response message and originating address be returned; do not interpret it
|
||||
# The RPC requested that the raw response message
|
||||
# and originating address be returned; do not
|
||||
# interpret it
|
||||
df.callback((message, address))
|
||||
elif isinstance(message, msgtypes.ErrorMessage):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
@@ -138,7 +183,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
else:
|
||||
localModuleHierarchy = self.__module__.split('.')
|
||||
remoteHierarchy = message.exceptionType.split('.')
|
||||
#strip the remote hierarchy
|
||||
# strip the remote hierarchy
|
||||
while remoteHierarchy[0] == localModuleHierarchy[0]:
|
||||
remoteHierarchy.pop(0)
|
||||
localModuleHierarchy.pop(0)
|
||||
@@ -155,13 +200,13 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df.callback(message.response)
|
||||
else:
|
||||
# If the original message isn't found, it must have timed out
|
||||
#TODO: we should probably do something with this...
|
||||
# TODO: we should probably do something with this...
|
||||
pass
|
||||
|
||||
def _send(self, data, rpcID, address):
|
||||
""" Transmit the specified data over UDP, breaking it up into several
|
||||
packets if necessary
|
||||
|
||||
|
||||
If the data is spread over multiple UDP datagrams, the packets have the
|
||||
following structure::
|
||||
| | | | | |||||||||||| 0x00 |
|
||||
@@ -169,15 +214,19 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
| type ID | of packets |of this packet | | indicator|
|
||||
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
|
||||
| | | | | |||||||||||| |
|
||||
|
||||
|
||||
@note: The header used for breaking up large data segments will
|
||||
possibly be moved out of the KademliaProtocol class in the
|
||||
future, into something similar to a message translator/encoder
|
||||
class (see C{kademlia.msgformat} and C{kademlia.encoding}).
|
||||
class (see C{kademlia.msgformat} and C{kademlia.encoding}).
|
||||
"""
|
||||
if len(data) > self.msgSizeLimit:
|
||||
# We have to spread the data over multiple UDP datagrams, and provide sequencing information
|
||||
# 1st byte is transmission type id, bytes 2 & 3 are the total number of packets in this transmission, bytes 4 & 5 are the sequence number for this specific packet
|
||||
# We have to spread the data over multiple UDP datagrams,
|
||||
# and provide sequencing information
|
||||
#
|
||||
# 1st byte is transmission type id, bytes 2 & 3 are the
|
||||
# total number of packets in this transmission, bytes 4 &
|
||||
# 5 are the sequence number for this specific packet
|
||||
totalPackets = len(data) / self.msgSizeLimit
|
||||
if len(data) % self.msgSizeLimit > 0:
|
||||
totalPackets += 1
|
||||
@@ -185,33 +234,35 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
seqNumber = 0
|
||||
startPos = 0
|
||||
while seqNumber < totalPackets:
|
||||
#reactor.iterate() #IGNORE:E1101
|
||||
packetData = data[startPos:startPos+self.msgSizeLimit]
|
||||
packetData = data[startPos:startPos + self.msgSizeLimit]
|
||||
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
|
||||
txData = '\x00%s%s%s\x00%s' % (encTotalPackets, encSeqNumber, rpcID, packetData)
|
||||
self._sendNext(txData, address)
|
||||
self._scheduleSendNext(txData, address)
|
||||
|
||||
startPos += self.msgSizeLimit
|
||||
seqNumber += 1
|
||||
else:
|
||||
self._sendNext(data, address)
|
||||
self._scheduleSendNext(data, address)
|
||||
|
||||
def _sendNext(self, txData, address):
|
||||
""" Send the next UDP packet """
|
||||
ts = time.time()
|
||||
delay = 0
|
||||
if ts >= self._next:
|
||||
delay = self.minToSendDelay
|
||||
self._next = ts + self.minToSendDelay
|
||||
else:
|
||||
delay = (self._next-ts) + self.maxToSendDelay
|
||||
self._next += self.maxToSendDelay
|
||||
def _scheduleSendNext(self, txData, address):
|
||||
"""Schedule the sending of the next UDP packet """
|
||||
delay = self._delay()
|
||||
key = object()
|
||||
delayed_call = reactor.callLater(delay, self._write_and_remove, key, txData, address)
|
||||
self._call_later_list[key] = delayed_call
|
||||
|
||||
def _write_and_remove(self, key, txData, address):
|
||||
del self._call_later_list[key]
|
||||
if self.transport:
|
||||
laterCall = reactor.callLater(delay, self.transport.write, txData, address)
|
||||
for key in self._callLaterList.keys():
|
||||
if key <= ts:
|
||||
del self._callLaterList[key]
|
||||
self._callLaterList[self._next] = laterCall
|
||||
try:
|
||||
self.transport.write(txData, address)
|
||||
except socket.error as err:
|
||||
if err.errno == errno.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesnt seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
raise err
|
||||
|
||||
def _sendResponse(self, contact, rpcID, response):
|
||||
""" Send a RPC response to the specified contact
|
||||
@@ -231,6 +282,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
|
||||
def _handleRPC(self, senderContact, rpcID, method, args):
|
||||
""" Executes a local function in response to an RPC request """
|
||||
|
||||
# Set up the deferred callchain
|
||||
def handleError(f):
|
||||
self._sendError(senderContact, rpcID, f.type, f.getErrorMessage())
|
||||
@@ -246,60 +298,72 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
func = getattr(self._node, method, None)
|
||||
if callable(func) and hasattr(func, 'rpcmethod'):
|
||||
# Call the exposed Node method and return the result to the deferred callback chain
|
||||
log.debug("DHT RECV CALL %s with args %s", method, args)
|
||||
try:
|
||||
##try:
|
||||
## # Try to pass the sender's node id to the function...
|
||||
result = func(*args, **{'_rpcNodeID': senderContact.id, '_rpcNodeContact': senderContact})
|
||||
##except TypeError:
|
||||
## # ...or simply call it if that fails
|
||||
## result = func(*args)
|
||||
kwargs = {'_rpcNodeID': senderContact.id, '_rpcNodeContact': senderContact}
|
||||
result = func(*args, **kwargs)
|
||||
except Exception, e:
|
||||
df.errback(failure.Failure(e))
|
||||
else:
|
||||
df.callback(result)
|
||||
else:
|
||||
# No such exposed method
|
||||
df.errback( failure.Failure( AttributeError('Invalid method: %s' % method) ) )
|
||||
df.errback(failure.Failure(AttributeError('Invalid method: %s' % method)))
|
||||
|
||||
def _msgTimeout(self, messageID):
|
||||
""" Called when an RPC request message times out """
|
||||
# Find the message that timed out
|
||||
if self._sentMessages.has_key(messageID):
|
||||
remoteContactID, df = self._sentMessages[messageID][0:2]
|
||||
if self._partialMessages.has_key(messageID):
|
||||
# We are still receiving this message
|
||||
# See if any progress has been made; if not, kill the message
|
||||
if self._partialMessagesProgress.has_key(messageID):
|
||||
if len(self._partialMessagesProgress[messageID]) == len(self._partialMessages[messageID]):
|
||||
# No progress has been made
|
||||
del self._partialMessagesProgress[messageID]
|
||||
del self._partialMessages[messageID]
|
||||
df.errback(failure.Failure(TimeoutError(remoteContactID)))
|
||||
return
|
||||
# Reset the RPC timeout timer
|
||||
timeoutCall = reactor.callLater(constants.rpcTimeout, self._msgTimeout, messageID) #IGNORE:E1101
|
||||
self._sentMessages[messageID] = (remoteContactID, df, timeoutCall)
|
||||
return
|
||||
del self._sentMessages[messageID]
|
||||
# The message's destination node is now considered to be dead;
|
||||
# raise an (asynchronous) TimeoutError exception and update the host node
|
||||
self._node.removeContact(remoteContactID)
|
||||
df.errback(failure.Failure(TimeoutError(remoteContactID)))
|
||||
else:
|
||||
if not self._sentMessages.has_key(messageID):
|
||||
# This should never be reached
|
||||
print "ERROR: deferred timed out, but is not present in sent messages list!"
|
||||
log.error("deferred timed out, but is not present in sent messages list!")
|
||||
return
|
||||
remoteContactID, df = self._sentMessages[messageID][0:2]
|
||||
if self._partialMessages.has_key(messageID):
|
||||
# We are still receiving this message
|
||||
self._msgTimeoutInProgress(messageID, remoteContactID, df)
|
||||
return
|
||||
del self._sentMessages[messageID]
|
||||
# The message's destination node is now considered to be dead;
|
||||
# raise an (asynchronous) TimeoutError exception and update the host node
|
||||
self._node.removeContact(remoteContactID)
|
||||
df.errback(failure.Failure(TimeoutError(remoteContactID)))
|
||||
|
||||
def _msgTimeoutInProgress(self, messageID, remoteContactID, df):
|
||||
# See if any progress has been made; if not, kill the message
|
||||
if self._hasProgressBeenMade(messageID):
|
||||
# Reset the RPC timeout timer
|
||||
timeoutCall = reactor.callLater(constants.rpcTimeout, self._msgTimeout, messageID)
|
||||
self._sentMessages[messageID] = (remoteContactID, df, timeoutCall)
|
||||
else:
|
||||
# No progress has been made
|
||||
del self._partialMessagesProgress[messageID]
|
||||
del self._partialMessages[messageID]
|
||||
df.errback(failure.Failure(TimeoutError(remoteContactID)))
|
||||
|
||||
def _hasProgressBeenMade(self, messageID):
|
||||
return (
|
||||
self._partialMessagesProgress.has_key(messageID) and
|
||||
(
|
||||
len(self._partialMessagesProgress[messageID]) !=
|
||||
len(self._partialMessages[messageID])
|
||||
)
|
||||
)
|
||||
|
||||
def stopProtocol(self):
|
||||
""" Called when the transport is disconnected.
|
||||
|
||||
|
||||
Will only be called once, after all ports are disconnected.
|
||||
"""
|
||||
for key in self._callLaterList.keys():
|
||||
log.info('Stopping DHT')
|
||||
for delayed_call in self._call_later_list.values():
|
||||
try:
|
||||
if key > time.time():
|
||||
self._callLaterList[key].cancel()
|
||||
except Exception, e:
|
||||
print e
|
||||
del self._callLaterList[key]
|
||||
#TODO: test: do we really need the reactor.iterate() call?
|
||||
reactor.iterate()
|
||||
delayed_call.cancel()
|
||||
except (error.AlreadyCalled, error.AlreadyCancelled):
|
||||
log.debug('Attempted to cancel a DelayedCall that was not active')
|
||||
except Exception:
|
||||
log.exception('Failed to cancel a DelayedCall')
|
||||
# not sure why this is needed, but taking this out sometimes causes
|
||||
# exceptions.AttributeError: 'Port' object has no attribute 'socket'
|
||||
# to happen on shutdown
|
||||
# reactor.iterate()
|
||||
log.info('DHT stopped')
|
||||
+94
-83
@@ -5,24 +5,28 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import time, random
|
||||
|
||||
import time
|
||||
import random
|
||||
import constants
|
||||
import kbucket
|
||||
|
||||
from protocol import TimeoutError
|
||||
|
||||
|
||||
class RoutingTable(object):
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
|
||||
Classes inheriting from this should provide a suitable routing table for
|
||||
a parent Node object (i.e. the local entity in the Kademlia network)
|
||||
"""
|
||||
|
||||
def __init__(self, parentNodeID):
|
||||
"""
|
||||
@param parentNodeID: The n-bit node ID of the node to which this
|
||||
routing table belongs
|
||||
@type parentNodeID: str
|
||||
"""
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add the given contact to the correct k-bucket; if it already
|
||||
exists, its status will be updated
|
||||
@@ -30,21 +34,11 @@ class RoutingTable(object):
|
||||
@param contact: The contact to add to this node's k-buckets
|
||||
@type contact: kademlia.contact.Contact
|
||||
"""
|
||||
|
||||
def distance(self, keyOne, keyTwo):
|
||||
""" Calculate the XOR result between two string variables
|
||||
|
||||
@return: XOR result of two long variables
|
||||
@rtype: long
|
||||
"""
|
||||
valKeyOne = long(keyOne.encode('hex'), 16)
|
||||
valKeyTwo = long(keyTwo.encode('hex'), 16)
|
||||
return valKeyOne ^ valKeyTwo
|
||||
|
||||
def findCloseNodes(self, key, count, _rpcNodeID=None):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
@param count: the amount of contacts to return
|
||||
@@ -53,20 +47,22 @@ class RoutingTable(object):
|
||||
Whatever ID is passed in the paramater will get
|
||||
excluded from the list of returned contacts.
|
||||
@type _rpcNodeID: str
|
||||
|
||||
|
||||
@return: A list of node contacts (C{kademlia.contact.Contact instances})
|
||||
closest to the specified key.
|
||||
closest to the specified key.
|
||||
This method will return C{k} (or C{count}, if specified)
|
||||
contacts if at all possible; it will only return fewer if the
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
|
||||
def getContact(self, contactID):
|
||||
""" Returns the (known) contact with the specified node ID
|
||||
|
||||
|
||||
@raise ValueError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
|
||||
def getRefreshList(self, startIndex=0, force=False):
|
||||
""" Finds all k-buckets that need refreshing, starting at the
|
||||
k-bucket with the specified index, and returns IDs to be searched for
|
||||
@@ -83,22 +79,24 @@ class RoutingTable(object):
|
||||
will be refreshed, regardless of the time they were last
|
||||
accessed.
|
||||
@type force: bool
|
||||
|
||||
|
||||
@return: A list of node ID's that the parent node should search for
|
||||
in order to refresh the routing Table
|
||||
@rtype: list
|
||||
"""
|
||||
|
||||
def removeContact(self, contactID):
|
||||
""" Remove the contact with the specified node ID from the routing
|
||||
table
|
||||
|
||||
|
||||
@param contactID: The node ID of the contact to remove
|
||||
@type contactID: str
|
||||
"""
|
||||
|
||||
def touchKBucket(self, key):
|
||||
""" Update the "last accessed" timestamp of the k-bucket which covers
|
||||
the range containing the specified key in the key/ID space
|
||||
|
||||
|
||||
@param key: A key in the range of the target k-bucket
|
||||
@type key: str
|
||||
"""
|
||||
@@ -106,19 +104,20 @@ class RoutingTable(object):
|
||||
|
||||
class TreeRoutingTable(RoutingTable):
|
||||
""" This class implements a routing table used by a Node class.
|
||||
|
||||
|
||||
The Kademlia routing table is a binary tree whose leaves are k-buckets,
|
||||
where each k-bucket contains nodes with some common prefix of their IDs.
|
||||
This prefix is the k-bucket's position in the binary tree; it therefore
|
||||
covers some range of ID values, and together all of the k-buckets cover
|
||||
the entire n-bit ID (or key) space (with no overlap).
|
||||
|
||||
|
||||
@note: In this implementation, nodes in the tree (the k-buckets) are
|
||||
added dynamically, as needed; this technique is described in the 13-page
|
||||
version of the Kademlia paper, in section 2.4. It does, however, use the
|
||||
C{PING} RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
|
||||
def __init__(self, parentNodeID):
|
||||
"""
|
||||
@param parentNodeID: The n-bit node ID of the node to which this
|
||||
@@ -126,7 +125,7 @@ class TreeRoutingTable(RoutingTable):
|
||||
@type parentNodeID: str
|
||||
"""
|
||||
# Create the initial (single) k-bucket covering the range of the entire n-bit ID space
|
||||
self._buckets = [kbucket.KBucket(rangeMin=0, rangeMax=2**constants.key_bits)]
|
||||
self._buckets = [kbucket.KBucket(rangeMin=0, rangeMax=2 ** constants.key_bits)]
|
||||
self._parentNodeID = parentNodeID
|
||||
|
||||
def addContact(self, contact):
|
||||
@@ -143,7 +142,8 @@ class TreeRoutingTable(RoutingTable):
|
||||
try:
|
||||
self._buckets[bucketIndex].addContact(contact)
|
||||
except kbucket.BucketFull:
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's id)
|
||||
# The bucket is full; see if it can be split (by checking
|
||||
# if its range includes the host node's id)
|
||||
if self._buckets[bucketIndex].keyInRange(self._parentNodeID):
|
||||
self._splitBucket(bucketIndex)
|
||||
# Retry the insertion attempt
|
||||
@@ -151,18 +151,20 @@ class TreeRoutingTable(RoutingTable):
|
||||
else:
|
||||
# We can't split the k-bucket
|
||||
# NOTE:
|
||||
# In section 2.4 of the 13-page version of the Kademlia paper, it is specified that
|
||||
# in this case, the new contact should simply be dropped. However, in section 2.2,
|
||||
# it states that the head contact in the k-bucket (i.e. the least-recently seen node)
|
||||
# should be pinged - if it does not reply, it should be dropped, and the new contact
|
||||
# added to the tail of the k-bucket. This implementation follows section 2.2 regarding
|
||||
# this point.
|
||||
headContact = self._buckets[bucketIndex]._contacts[0]
|
||||
|
||||
# In section 2.4 of the 13-page version of the
|
||||
# Kademlia paper, it is specified that in this case,
|
||||
# the new contact should simply be dropped. However,
|
||||
# in section 2.2, it states that the head contact in
|
||||
# the k-bucket (i.e. the least-recently seen node)
|
||||
# should be pinged - if it does not reply, it should
|
||||
# be dropped, and the new contact added to the tail of
|
||||
# the k-bucket. This implementation follows section
|
||||
# 2.2 regarding this point.
|
||||
|
||||
def replaceContact(failure):
|
||||
""" Callback for the deferred PING RPC to see if the head
|
||||
node in the k-bucket is still responding
|
||||
|
||||
|
||||
@type failure: twisted.python.failure.Failure
|
||||
"""
|
||||
failure.trap(TimeoutError)
|
||||
@@ -176,17 +178,18 @@ class TreeRoutingTable(RoutingTable):
|
||||
pass
|
||||
# ...and add the new one at the tail of the bucket
|
||||
self.addContact(contact)
|
||||
|
||||
|
||||
# Ping the least-recently seen contact in this k-bucket
|
||||
headContact = self._buckets[bucketIndex]._contacts[0]
|
||||
df = headContact.ping()
|
||||
# If there's an error (i.e. timeout), remove the head contact, and append the new one
|
||||
head_contact = self._buckets[bucketIndex]._contacts[0]
|
||||
df = head_contact.ping()
|
||||
# If there's an error (i.e. timeout), remove the head
|
||||
# contact, and append the new one
|
||||
df.addErrback(replaceContact)
|
||||
|
||||
|
||||
def findCloseNodes(self, key, count, _rpcNodeID=None):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
@param count: the amount of contacts to return
|
||||
@@ -195,39 +198,41 @@ class TreeRoutingTable(RoutingTable):
|
||||
Whatever ID is passed in the paramater will get
|
||||
excluded from the list of returned contacts.
|
||||
@type _rpcNodeID: str
|
||||
|
||||
|
||||
@return: A list of node contacts (C{kademlia.contact.Contact instances})
|
||||
closest to the specified key.
|
||||
closest to the specified key.
|
||||
This method will return C{k} (or C{count}, if specified)
|
||||
contacts if at all possible; it will only return fewer if the
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
#if key == self.id:
|
||||
# bucketIndex = 0 #TODO: maybe not allow this to continue?
|
||||
#else:
|
||||
bucketIndex = self._kbucketIndex(key)
|
||||
closestNodes = self._buckets[bucketIndex].getContacts(constants.k, _rpcNodeID)
|
||||
# This method must return k contacts (even if we have the node with the specified key as node ID),
|
||||
# unless there is less than k remote nodes in the routing table
|
||||
# This method must return k contacts (even if we have the node
|
||||
# with the specified key as node ID), unless there is less
|
||||
# than k remote nodes in the routing table
|
||||
i = 1
|
||||
canGoLower = bucketIndex-i >= 0
|
||||
canGoHigher = bucketIndex+i < len(self._buckets)
|
||||
# Fill up the node list to k nodes, starting with the closest neighbouring nodes known
|
||||
canGoLower = bucketIndex - i >= 0
|
||||
canGoHigher = bucketIndex + i < len(self._buckets)
|
||||
# Fill up the node list to k nodes, starting with the closest neighbouring nodes known
|
||||
while len(closestNodes) < constants.k and (canGoLower or canGoHigher):
|
||||
#TODO: this may need to be optimized
|
||||
# TODO: this may need to be optimized
|
||||
if canGoLower:
|
||||
closestNodes.extend(self._buckets[bucketIndex-i].getContacts(constants.k - len(closestNodes), _rpcNodeID))
|
||||
canGoLower = bucketIndex-(i+1) >= 0
|
||||
closestNodes.extend(
|
||||
self._buckets[bucketIndex - i].getContacts(
|
||||
constants.k - len(closestNodes), _rpcNodeID))
|
||||
canGoLower = bucketIndex - (i + 1) >= 0
|
||||
if canGoHigher:
|
||||
closestNodes.extend(self._buckets[bucketIndex+i].getContacts(constants.k - len(closestNodes), _rpcNodeID))
|
||||
canGoHigher = bucketIndex+(i+1) < len(self._buckets)
|
||||
closestNodes.extend(
|
||||
self._buckets[bucketIndex + i].getContacts(
|
||||
constants.k - len(closestNodes), _rpcNodeID))
|
||||
canGoHigher = bucketIndex + (i + 1) < len(self._buckets)
|
||||
i += 1
|
||||
return closestNodes
|
||||
|
||||
def getContact(self, contactID):
|
||||
""" Returns the (known) contact with the specified node ID
|
||||
|
||||
|
||||
@raise ValueError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
@@ -255,7 +260,7 @@ class TreeRoutingTable(RoutingTable):
|
||||
will be refreshed, regardless of the time they were last
|
||||
accessed.
|
||||
@type force: bool
|
||||
|
||||
|
||||
@return: A list of node ID's that the parent node should search for
|
||||
in order to refresh the routing Table
|
||||
@rtype: list
|
||||
@@ -272,7 +277,7 @@ class TreeRoutingTable(RoutingTable):
|
||||
def removeContact(self, contactID):
|
||||
""" Remove the contact with the specified node ID from the routing
|
||||
table
|
||||
|
||||
|
||||
@param contactID: The node ID of the contact to remove
|
||||
@type contactID: str
|
||||
"""
|
||||
@@ -280,13 +285,12 @@ class TreeRoutingTable(RoutingTable):
|
||||
try:
|
||||
self._buckets[bucketIndex].removeContact(contactID)
|
||||
except ValueError:
|
||||
#print 'removeContact(): Contact not in routing table'
|
||||
return
|
||||
|
||||
def touchKBucket(self, key):
|
||||
""" Update the "last accessed" timestamp of the k-bucket which covers
|
||||
the range containing the specified key in the key/ID space
|
||||
|
||||
|
||||
@param key: A key in the range of the target k-bucket
|
||||
@type key: str
|
||||
"""
|
||||
@@ -296,17 +300,16 @@ class TreeRoutingTable(RoutingTable):
|
||||
def _kbucketIndex(self, key):
|
||||
""" Calculate the index of the k-bucket which is responsible for the
|
||||
specified key (or ID)
|
||||
|
||||
|
||||
@param key: The key for which to find the appropriate k-bucket index
|
||||
@type key: str
|
||||
|
||||
|
||||
@return: The index of the k-bucket responsible for the specified key
|
||||
@rtype: int
|
||||
"""
|
||||
valKey = long(key.encode('hex'), 16)
|
||||
i = 0
|
||||
for bucket in self._buckets:
|
||||
if bucket.keyInRange(valKey):
|
||||
if bucket.keyInRange(key):
|
||||
return i
|
||||
else:
|
||||
i += 1
|
||||
@@ -314,31 +317,32 @@ class TreeRoutingTable(RoutingTable):
|
||||
|
||||
def _randomIDInBucketRange(self, bucketIndex):
|
||||
""" Returns a random ID in the specified k-bucket's range
|
||||
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
idValue = random.randrange(self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax)
|
||||
idValue = random.randrange(
|
||||
self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax)
|
||||
randomID = hex(idValue)[2:]
|
||||
if randomID[-1] == 'L':
|
||||
randomID = randomID[:-1]
|
||||
if len(randomID) % 2 != 0:
|
||||
randomID = '0' + randomID
|
||||
randomID = randomID.decode('hex')
|
||||
randomID = (constants.key_bits/8 - len(randomID))*'\x00' + randomID
|
||||
randomID = (constants.key_bits / 8 - len(randomID)) * '\x00' + randomID
|
||||
return randomID
|
||||
|
||||
def _splitBucket(self, oldBucketIndex):
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
cover the same range in the key/ID space
|
||||
|
||||
|
||||
@param oldBucketIndex: The index of k-bucket to split (in this table's
|
||||
list of k-buckets)
|
||||
@type oldBucketIndex: int
|
||||
"""
|
||||
# Resize the range of the current (old) k-bucket
|
||||
oldBucket = self._buckets[oldBucketIndex]
|
||||
splitPoint = oldBucket.rangeMax - (oldBucket.rangeMax - oldBucket.rangeMin)/2
|
||||
splitPoint = oldBucket.rangeMax - (oldBucket.rangeMax - oldBucket.rangeMin) / 2
|
||||
# Create a new k-bucket to cover the range split off from the old bucket
|
||||
newBucket = kbucket.KBucket(splitPoint, oldBucket.rangeMax)
|
||||
oldBucket.rangeMax = splitPoint
|
||||
@@ -352,16 +356,18 @@ class TreeRoutingTable(RoutingTable):
|
||||
for contact in newBucket._contacts:
|
||||
oldBucket.removeContact(contact)
|
||||
|
||||
|
||||
class OptimizedTreeRoutingTable(TreeRoutingTable):
|
||||
""" A version of the "tree"-type routing table specified by Kademlia,
|
||||
along with contact accounting optimizations specified in section 4.1 of
|
||||
of the 13-page version of the Kademlia paper.
|
||||
"""
|
||||
|
||||
def __init__(self, parentNodeID):
|
||||
TreeRoutingTable.__init__(self, parentNodeID)
|
||||
# Cache containing nodes eligible to replace stale k-bucket entries
|
||||
self._replacementCache = {}
|
||||
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add the given contact to the correct k-bucket; if it already
|
||||
exists, its status will be updated
|
||||
@@ -379,7 +385,8 @@ class OptimizedTreeRoutingTable(TreeRoutingTable):
|
||||
try:
|
||||
self._buckets[bucketIndex].addContact(contact)
|
||||
except kbucket.BucketFull:
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's id)
|
||||
# The bucket is full; see if it can be split (by checking
|
||||
# if its range includes the host node's id)
|
||||
if self._buckets[bucketIndex].keyInRange(self._parentNodeID):
|
||||
self._splitBucket(bucketIndex)
|
||||
# Retry the insertion attempt
|
||||
@@ -388,22 +395,26 @@ class OptimizedTreeRoutingTable(TreeRoutingTable):
|
||||
# We can't split the k-bucket
|
||||
# NOTE: This implementation follows section 4.1 of the 13 page version
|
||||
# of the Kademlia paper (optimized contact accounting without PINGs
|
||||
#- results in much less network traffic, at the expense of some memory)
|
||||
# - results in much less network traffic, at the expense of some memory)
|
||||
|
||||
# Put the new contact in our replacement cache for the corresponding k-bucket (or update it's position if it exists already)
|
||||
if not self._replacementCache.has_key(bucketIndex):
|
||||
# Put the new contact in our replacement cache for the
|
||||
# corresponding k-bucket (or update it's position if
|
||||
# it exists already)
|
||||
if bucketIndex not in self._replacementCache:
|
||||
self._replacementCache[bucketIndex] = []
|
||||
if contact in self._replacementCache[bucketIndex]:
|
||||
self._replacementCache[bucketIndex].remove(contact)
|
||||
#TODO: Using k to limit the size of the contact replacement cache - maybe define a seperate value for this in constants.py?
|
||||
elif len(self._replacementCache) >= constants.k:
|
||||
self._replacementCache.pop(0)
|
||||
# TODO: Using k to limit the size of the contact
|
||||
# replacement cache - maybe define a separate value for
|
||||
# this in constants.py?
|
||||
elif len(self._replacementCache[bucketIndex]) >= constants.k:
|
||||
self._replacementCache[bucketIndex].pop(0)
|
||||
self._replacementCache[bucketIndex].append(contact)
|
||||
|
||||
|
||||
def removeContact(self, contactID):
|
||||
""" Remove the contact with the specified node ID from the routing
|
||||
table
|
||||
|
||||
|
||||
@param contactID: The node ID of the contact to remove
|
||||
@type contactID: str
|
||||
"""
|
||||
@@ -411,12 +422,12 @@ class OptimizedTreeRoutingTable(TreeRoutingTable):
|
||||
try:
|
||||
contact = self._buckets[bucketIndex].getContact(contactID)
|
||||
except ValueError:
|
||||
#print 'removeContact(): Contact not in routing table'
|
||||
return
|
||||
contact.failedRPCs += 1
|
||||
if contact.failedRPCs >= 5:
|
||||
if contact.failedRPCs >= 5:
|
||||
self._buckets[bucketIndex].removeContact(contactID)
|
||||
# Replace this stale contact with one from our replacemnent cache, if we have any
|
||||
if self._replacementCache.has_key(bucketIndex):
|
||||
# Replace this stale contact with one from our replacement cache, if we have any
|
||||
if bucketIndex in self._replacementCache:
|
||||
if len(self._replacementCache[bucketIndex]) > 0:
|
||||
self._buckets[bucketIndex].addContact( self._replacementCache[bucketIndex].pop() )
|
||||
self._buckets[bucketIndex].addContact(
|
||||
self._replacementCache[bucketIndex].pop())
|
||||
@@ -1,100 +0,0 @@
|
||||
from lbrynet.dht.node import Node
|
||||
import binascii
|
||||
from twisted.internet import reactor, task
|
||||
import logging
|
||||
import sys
|
||||
from lbrynet.core.utils import generate_id
|
||||
|
||||
|
||||
def print_usage():
|
||||
print "Usage:\n%s UDP_PORT KNOWN_NODE_IP KNOWN_NODE_PORT HASH"
|
||||
|
||||
|
||||
def join_network(udp_port, known_nodes):
|
||||
lbryid = generate_id()
|
||||
|
||||
logging.info('Creating Node...')
|
||||
node = Node(udpPort=udp_port, lbryid=lbryid)
|
||||
|
||||
logging.info('Joining network...')
|
||||
d = node.joinNetwork(known_nodes)
|
||||
|
||||
def log_network_size():
|
||||
logging.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
|
||||
logging.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
|
||||
|
||||
d.addCallback(lambda _: log_network_size())
|
||||
|
||||
d.addCallback(lambda _: node)
|
||||
|
||||
return d
|
||||
|
||||
|
||||
def get_hosts(node, h):
|
||||
|
||||
def print_hosts(hosts):
|
||||
print "Hosts returned from the DHT: "
|
||||
print hosts
|
||||
|
||||
logging.info("Looking up %s", h)
|
||||
d = node.getPeersForBlob(h)
|
||||
d.addCallback(print_hosts)
|
||||
return d
|
||||
|
||||
|
||||
def announce_hash(node, h):
|
||||
d = node.announceHaveBlob(h, 34567)
|
||||
|
||||
def log_results(results):
|
||||
for success, result in results:
|
||||
if success:
|
||||
logging.info("Succeeded: %s", str(result))
|
||||
else:
|
||||
logging.info("Failed: %s", str(result.getErrorMessage()))
|
||||
|
||||
d.addCallback(log_results)
|
||||
return d
|
||||
|
||||
|
||||
def get_args():
|
||||
if len(sys.argv) < 5:
|
||||
print_usage()
|
||||
sys.exit(1)
|
||||
udp_port = int(sys.argv[1])
|
||||
known_nodes = [(sys.argv[2], int(sys.argv[3]))]
|
||||
h = binascii.unhexlify(sys.argv[4])
|
||||
return udp_port, known_nodes, h
|
||||
|
||||
|
||||
def run_dht_script(dht_func):
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format)
|
||||
|
||||
udp_port, known_nodes, h = get_args()
|
||||
|
||||
d = task.deferLater(reactor, 0, join_network, udp_port, known_nodes)
|
||||
|
||||
def run_dht_func(node):
|
||||
return dht_func(node, h)
|
||||
|
||||
d.addCallback(run_dht_func)
|
||||
|
||||
def log_err(err):
|
||||
logging.error("An error occurred: %s", err.getTraceback())
|
||||
return err
|
||||
|
||||
def shut_down():
|
||||
logging.info("Shutting down")
|
||||
reactor.stop()
|
||||
|
||||
d.addErrback(log_err)
|
||||
d.addBoth(lambda _: shut_down())
|
||||
reactor.run()
|
||||
|
||||
|
||||
def get_hosts_for_hash_in_dht():
|
||||
run_dht_script(get_hosts)
|
||||
|
||||
|
||||
def announce_hash_to_dht():
|
||||
run_dht_script(announce_hash)
|
||||
+152
-63
@@ -36,11 +36,11 @@ class IRequestSender(Interface):
|
||||
"""
|
||||
|
||||
def add_blob_request(self, blob_request):
|
||||
"""
|
||||
Add a request for a blob to the next message that will be sent to the peer.
|
||||
"""Add a request for a blob to the next message that will be sent to the peer.
|
||||
|
||||
This will cause the protocol to call blob_request.write(data) for all incoming
|
||||
data, after the response message has been parsed out, until blob_request.finished_deferred fires.
|
||||
This will cause the protocol to call blob_request.write(data)
|
||||
for all incoming data, after the response message has been
|
||||
parsed out, until blob_request.finished_deferred fires.
|
||||
|
||||
@param blob_request: the request for the blob
|
||||
@type blob_request: ClientBlobRequest
|
||||
@@ -56,8 +56,7 @@ class IRequestCreator(Interface):
|
||||
"""
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
"""
|
||||
Create a Request object for the peer and then give the protocol that request.
|
||||
"""Create a Request object for the peer and then give the protocol that request.
|
||||
|
||||
@param peer: the Peer object which the request will be sent to.
|
||||
@type peer: Peer
|
||||
@@ -65,7 +64,8 @@ class IRequestCreator(Interface):
|
||||
@param protocol: the protocol to pass the request to.
|
||||
@type protocol: object which implements IRequestSender
|
||||
|
||||
@return: Deferred object which will callback with True or False depending on whether a Request was sent
|
||||
@return: Deferred object which will callback with True or
|
||||
False depending on whether a Request was sent
|
||||
@rtype: Deferred which fires with boolean
|
||||
"""
|
||||
|
||||
@@ -83,12 +83,12 @@ class IMetadataHandler(Interface):
|
||||
Get metadata for the IDownloadManager.
|
||||
"""
|
||||
def get_initial_blobs(self):
|
||||
"""
|
||||
Return metadata about blobs that are known to be associated with the stream at the time that the
|
||||
stream is set up.
|
||||
"""Return metadata about blobs that are known to be associated with
|
||||
the stream at the time that the stream is set up.
|
||||
|
||||
@return: Deferred object which will call back with a list of BlobInfo objects
|
||||
@rtype: Deferred which fires with [BlobInfo]
|
||||
|
||||
"""
|
||||
|
||||
def final_blob_num(self):
|
||||
@@ -156,11 +156,12 @@ class IDownloadManager(Interface):
|
||||
"""
|
||||
|
||||
def needed_blobs(self):
|
||||
"""
|
||||
Returns a list of BlobInfos representing all of the blobs that the stream still needs to download.
|
||||
"""Returns a list of BlobInfos representing all of the blobs that the
|
||||
stream still needs to download.
|
||||
|
||||
@return: the list of BlobInfos representing blobs that the stream still needs to download.
|
||||
@rtype: [BlobInfo]
|
||||
|
||||
"""
|
||||
|
||||
def final_blob_num(self):
|
||||
@@ -172,14 +173,15 @@ class IDownloadManager(Interface):
|
||||
"""
|
||||
|
||||
def handle_blob(self, blob_num):
|
||||
"""
|
||||
This function is called when the next blob in the stream is ready to be handled, whatever that may mean.
|
||||
"""This function is called when the next blob in the stream is ready
|
||||
to be handled, whatever that may mean.
|
||||
|
||||
@param blob_num: The blob_num of the blob that is ready to be handled.
|
||||
@type blob_num: integer
|
||||
|
||||
@return: A Deferred which fires when the blob has been 'handled'
|
||||
@rtype: Deferred which can fire with anything
|
||||
|
||||
"""
|
||||
|
||||
|
||||
@@ -188,8 +190,8 @@ class IConnectionManager(Interface):
|
||||
Connects to peers so that IRequestCreators can send their requests.
|
||||
"""
|
||||
def get_next_request(self, peer, protocol):
|
||||
"""
|
||||
Ask all IRequestCreators belonging to this object to create a Request for peer and give it to protocol
|
||||
"""Ask all IRequestCreators belonging to this object to create a
|
||||
Request for peer and give it to protocol
|
||||
|
||||
@param peer: the peer which the request will be sent to.
|
||||
@type peer: Peer
|
||||
@@ -197,9 +199,11 @@ class IConnectionManager(Interface):
|
||||
@param protocol: the protocol which the request should be sent to by the IRequestCreator.
|
||||
@type protocol: IRequestSender
|
||||
|
||||
@return: Deferred object which will callback with True or False depending on whether the IRequestSender
|
||||
should send the request or hang up
|
||||
@return: Deferred object which will callback with True or
|
||||
False depending on whether the IRequestSender should send
|
||||
the request or hang up
|
||||
@rtype: Deferred which fires with boolean
|
||||
|
||||
"""
|
||||
|
||||
def protocol_disconnected(self, peer, protocol):
|
||||
@@ -217,11 +221,12 @@ class IConnectionManager(Interface):
|
||||
|
||||
|
||||
class IProgressManager(Interface):
|
||||
"""
|
||||
Responsible for keeping track of the progress of the download.
|
||||
"""Responsible for keeping track of the progress of the download.
|
||||
|
||||
Specifically, it is their responsibility to decide which blobs
|
||||
need to be downloaded and keep track of the progress of the
|
||||
download
|
||||
|
||||
Specifically, it is their responsibility to decide which blobs need to be downloaded and keep track of
|
||||
the progress of the download
|
||||
"""
|
||||
def stream_position(self):
|
||||
"""
|
||||
@@ -235,11 +240,12 @@ class IProgressManager(Interface):
|
||||
"""
|
||||
|
||||
def needed_blobs(self):
|
||||
"""
|
||||
Returns a list of BlobInfos representing all of the blobs that the stream still needs to download.
|
||||
"""Returns a list of BlobInfos representing all of the blobs that the
|
||||
stream still needs to download.
|
||||
|
||||
@return: the list of BlobInfos representing blobs that the stream still needs to download.
|
||||
@rtype: [BlobInfo]
|
||||
|
||||
"""
|
||||
|
||||
def blob_downloaded(self, blob, blob_info):
|
||||
@@ -334,24 +340,26 @@ class IRateLimiter(Interface):
|
||||
"""
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
"""
|
||||
Register an IRateLimited object with the IRateLimiter so that the IRateLimiter can throttle it
|
||||
"""Register an IRateLimited object with the IRateLimiter so that the
|
||||
IRateLimiter can throttle it
|
||||
|
||||
@param protocol: An object implementing the interface IRateLimited
|
||||
@type protocol: Object implementing IRateLimited
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
|
||||
def unregister_protocol(self, protocol):
|
||||
"""
|
||||
Unregister an IRateLimited object so that it won't be throttled any more.
|
||||
"""Unregister an IRateLimited object so that it won't be throttled any more.
|
||||
|
||||
@param protocol: An object implementing the interface IRateLimited, which was previously registered with this
|
||||
@param protocol: An object implementing the interface
|
||||
IRateLimited, which was previously registered with this
|
||||
IRateLimiter via "register_protocol"
|
||||
@type protocol: Object implementing IRateLimited
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
|
||||
|
||||
@@ -360,11 +368,11 @@ class IRequestHandler(Interface):
|
||||
Pass client queries on to IQueryHandlers
|
||||
"""
|
||||
def register_query_handler(self, query_handler, query_identifiers):
|
||||
"""
|
||||
Register a query handler, which will be passed any queries that
|
||||
"""Register a query handler, which will be passed any queries that
|
||||
match any of the identifiers in query_identifiers
|
||||
|
||||
@param query_handler: the object which will handle queries matching the given query_identifiers
|
||||
@param query_handler: the object which will handle queries
|
||||
matching the given query_identifiers
|
||||
@type query_handler: Object implementing IQueryHandler
|
||||
|
||||
@param query_identifiers: A list of strings representing the query identifiers
|
||||
@@ -372,6 +380,7 @@ class IRequestHandler(Interface):
|
||||
@type query_identifiers: [string]
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
|
||||
def register_blob_sender(self, blob_sender):
|
||||
@@ -445,10 +454,7 @@ class IQueryHandlerFactory(Interface):
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""
|
||||
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
|
||||
"""
|
||||
class IStreamDownloaderOptions(Interface):
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
"""
|
||||
Return the list of options that can be used to modify IStreamDownloader behavior
|
||||
@@ -459,27 +465,50 @@ class IStreamDownloaderFactory(Interface):
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: [(option_description, default)]
|
||||
@rtype: [(string, string)]
|
||||
@return: [DownloadOption]
|
||||
@rtype: [DownloadOption]
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""Construct IStreamDownloaders and provide options that will be
|
||||
passed to those IStreamDownloaders.
|
||||
|
||||
"""
|
||||
|
||||
def can_download(self, sd_validator, payment_rate_manager):
|
||||
"""Decide whether the downloaders created by this factory can
|
||||
download the stream described by sd_validator
|
||||
|
||||
@param sd_validator: object containing stream metadata
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: True if the downloaders can download the stream, False otherwise
|
||||
@rtype: bool
|
||||
|
||||
"""
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
"""
|
||||
Create an object that implements the IStreamDownloader interface
|
||||
"""Create an object that implements the IStreamDownloader interface
|
||||
|
||||
@param sd_validator: object containing stream metadata which will be given to the IStreamDownloader
|
||||
@param sd_validator: object containing stream metadata which
|
||||
will be given to the IStreamDownloader
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param options: a list of strings that will be used by the IStreamDownloaderFactory to
|
||||
@param options: a list of values that will be used by the IStreamDownloaderFactory to
|
||||
construct the IStreamDownloader. the options are in the same order as they were given
|
||||
by get_downloader_options.
|
||||
@type options: [string]
|
||||
@type options: [Object]
|
||||
|
||||
@param payment_rate_manager: the PaymentRateManager which the IStreamDownloader should use.
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: a Deferred which fires with the downloader object
|
||||
@rtype: Deferred which fires with IStreamDownloader
|
||||
|
||||
"""
|
||||
|
||||
def get_description(self):
|
||||
@@ -496,15 +525,15 @@ class IStreamDownloader(Interface):
|
||||
Use metadata and data from the network for some useful purpose.
|
||||
"""
|
||||
def start(self):
|
||||
"""
|
||||
start downloading the stream
|
||||
"""start downloading the stream
|
||||
|
||||
@return: a Deferred which fires when the stream is finished downloading, or errbacks when the stream is
|
||||
cancelled.
|
||||
@return: a Deferred which fires when the stream is finished
|
||||
downloading, or errbacks when the stream is cancelled.
|
||||
@rtype: Deferred which fires with anything
|
||||
|
||||
"""
|
||||
|
||||
def insufficient_funds(self):
|
||||
def insufficient_funds(self, err):
|
||||
"""
|
||||
this function informs the stream downloader that funds are too low to finish downloading.
|
||||
|
||||
@@ -525,26 +554,28 @@ class IStreamDescriptorValidator(Interface):
|
||||
|
||||
def info_to_show(self):
|
||||
"""
|
||||
@return: A list of tuples representing metadata that should be presented to the user before starting the
|
||||
download
|
||||
|
||||
@return: A list of tuples representing metadata that should be
|
||||
presented to the user before starting the download
|
||||
@rtype: [(string, string)]
|
||||
"""
|
||||
|
||||
|
||||
class ILBRYWallet(Interface):
|
||||
"""
|
||||
Send and receive payments.
|
||||
class IWallet(Interface):
|
||||
"""Send and receive payments.
|
||||
|
||||
To send a payment, a payment reservation must be obtained first. This guarantees that a payment
|
||||
isn't promised if it can't be paid. When the service in question is rendered, the payment
|
||||
reservation must be given to the ILBRYWallet along with the final price. The reservation can also
|
||||
be canceled.
|
||||
To send a payment, a payment reservation must be obtained
|
||||
first. This guarantees that a payment isn't promised if it can't
|
||||
be paid. When the service in question is rendered, the payment
|
||||
reservation must be given to the IWallet along with the final
|
||||
price. The reservation can also be canceled.
|
||||
"""
|
||||
def stop(self):
|
||||
"""
|
||||
Send out any unsent payments, close any connections, and stop checking for incoming payments.
|
||||
"""Send out any unsent payments, close any connections, and stop
|
||||
checking for incoming payments.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
|
||||
def start(self):
|
||||
@@ -573,8 +604,8 @@ class ILBRYWallet(Interface):
|
||||
"""
|
||||
|
||||
def reserve_points(self, peer, amount):
|
||||
"""
|
||||
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
|
||||
"""Ensure a certain amount of points are available to be sent as
|
||||
payment, before the service is rendered
|
||||
|
||||
@param peer: The peer to which the payment will ultimately be sent
|
||||
@type peer: Peer
|
||||
@@ -582,8 +613,10 @@ class ILBRYWallet(Interface):
|
||||
@param amount: The amount of points to reserve
|
||||
@type amount: float
|
||||
|
||||
@return: A ReservedPoints object which is given to send_points once the service has been rendered
|
||||
@return: A ReservedPoints object which is given to send_points
|
||||
once the service has been rendered
|
||||
@rtype: ReservedPoints
|
||||
|
||||
"""
|
||||
|
||||
def cancel_point_reservation(self, reserved_points):
|
||||
@@ -630,4 +663,60 @@ class ILBRYWallet(Interface):
|
||||
@type amount: float
|
||||
|
||||
@return: None
|
||||
"""
|
||||
"""
|
||||
|
||||
|
||||
class IBlobPriceModel(Interface):
|
||||
"""
|
||||
A blob price model
|
||||
|
||||
Used by INegotiationStrategy classes
|
||||
"""
|
||||
|
||||
def calculate_price(self, blob):
|
||||
"""
|
||||
Calculate the price for a blob
|
||||
|
||||
@param blob: a blob hash
|
||||
@type blob: str
|
||||
|
||||
@return: blob price target
|
||||
@type: Decimal
|
||||
"""
|
||||
|
||||
|
||||
class INegotiationStrategy(Interface):
|
||||
"""
|
||||
Strategy to negotiate download payment rates
|
||||
"""
|
||||
|
||||
def make_offer(self, peer, blobs):
|
||||
"""
|
||||
Make a rate offer for the given peer and blobs
|
||||
|
||||
@param peer: peer to make an offer to
|
||||
@type: str
|
||||
|
||||
@param blobs: blob hashes to make an offer for
|
||||
@type: list
|
||||
|
||||
@return: rate offer
|
||||
@rtype: Offer
|
||||
"""
|
||||
|
||||
def respond_to_offer(self, offer, peer, blobs):
|
||||
"""
|
||||
Respond to a rate offer given by a peer
|
||||
|
||||
@param offer: offer to reply to
|
||||
@type: Offer
|
||||
|
||||
@param peer: peer to make an offer to
|
||||
@type: str
|
||||
|
||||
@param blobs: blob hashes to make an offer for
|
||||
@type: list
|
||||
|
||||
@return: accepted, rejected, or unset offer
|
||||
@rtype: Offer
|
||||
"""
|
||||
@@ -0,0 +1,370 @@
|
||||
import logging
|
||||
import sqlite3
|
||||
import os
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHash, NoSuchSDHash
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DBEncryptedFileMetadataManager(object):
|
||||
"""Store and provide access to LBRY file metadata using sqlite"""
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
|
||||
def setup(self):
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return self._get_all_streams()
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
d = self._store_stream(stream_hash, file_name, key, suggested_file_name)
|
||||
d.addCallback(lambda _: self.add_blobs_to_stream(stream_hash, blobs))
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return self._get_stream_info(stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return self._check_if_stream_exists(stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return self._delete_stream(stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None,
|
||||
end_blob=None, count=None, reverse=False):
|
||||
log.debug("Getting blobs for stream %s. Count is %s", stream_hash, count)
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
d1 = defer.succeed(None)
|
||||
if end_blob is not None:
|
||||
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
d2 = defer.succeed(None)
|
||||
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
|
||||
def get_positions(results):
|
||||
start_num = None
|
||||
end_num = None
|
||||
if results[0][0] is True:
|
||||
start_num = results[0][1]
|
||||
if results[1][0] is True:
|
||||
end_num = results[1][1]
|
||||
return start_num, end_num
|
||||
|
||||
dl.addCallback(get_positions)
|
||||
return dl
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = get_positions_of_start_and_end()
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return self._get_stream_of_blobhash(blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return self._get_sd_blob_hashes_for_stream(stream_hash)
|
||||
|
||||
def get_stream_hash_for_sd_hash(self, sd_hash):
|
||||
return self._get_stream_hash_for_sd_blob_hash(sd_hash)
|
||||
|
||||
def _open_db(self):
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool(
|
||||
"sqlite3",
|
||||
(os.path.join(self.db_dir, "lbryfile_info.db")),
|
||||
check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists lbry_files (" +
|
||||
" stream_hash text primary key, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" suggested_file_name text" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_stream(self, stream_hash):
|
||||
d = self.db_conn.runQuery(
|
||||
"select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(
|
||||
lambda result: result[0][0] if result else Failure(NoSuchStreamHash(stream_hash)))
|
||||
|
||||
def do_delete(transaction, s_h):
|
||||
transaction.execute("delete from lbry_files where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_blobs where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_descriptors where stream_hash = ?", (s_h,))
|
||||
|
||||
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _store_stream(self, stream_hash, name, key, suggested_file_name):
|
||||
d = self.db_conn.runQuery("insert into lbry_files values (?, ?, ?, ?)",
|
||||
(stream_hash, key, name, suggested_file_name))
|
||||
|
||||
def check_duplicate(err):
|
||||
if err.check(sqlite3.IntegrityError):
|
||||
raise DuplicateStreamHashError(stream_hash)
|
||||
return err
|
||||
|
||||
d.addErrback(check_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_streams(self):
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files")
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_info(self, stream_hash):
|
||||
def get_result(res):
|
||||
if res:
|
||||
return res[0]
|
||||
else:
|
||||
raise NoSuchStreamHash(stream_hash)
|
||||
|
||||
d = self.db_conn.runQuery(
|
||||
"select key, stream_name, suggested_file_name from lbry_files where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(get_result)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
d = self.db_conn.runQuery(
|
||||
"select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda r: True if len(r) else False)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
d = self.db_conn.runQuery(
|
||||
"select position from lbry_file_blobs where stream_hash = ? and blob_hash = ?",
|
||||
(stream_hash, blob_hash))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
params = []
|
||||
q_string = "select * from ("
|
||||
q_string += " select blob_hash, position, iv, length from lbry_file_blobs "
|
||||
q_string += " where stream_hash = ? "
|
||||
params.append(stream_hash)
|
||||
if start_num is not None:
|
||||
q_string += " and position > ? "
|
||||
params.append(start_num)
|
||||
if end_num is not None:
|
||||
q_string += " and position < ? "
|
||||
params.append(end_num)
|
||||
q_string += " order by position "
|
||||
if reverse is True:
|
||||
q_string += " DESC "
|
||||
if count is not None:
|
||||
q_string += " limit ? "
|
||||
params.append(count)
|
||||
q_string += ") order by position"
|
||||
# Order by position is done twice so that it always returns them from lowest position to
|
||||
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
|
||||
return self.db_conn.runQuery(q_string, tuple(params))
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
|
||||
def add_blobs(transaction):
|
||||
for blob_info in blob_infos:
|
||||
try:
|
||||
transaction.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
|
||||
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
|
||||
blob_info.iv, blob_info.length))
|
||||
except sqlite3.IntegrityError:
|
||||
if ignore_duplicate_error is False:
|
||||
raise
|
||||
|
||||
return self.db_conn.runInteraction(add_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_file_blobs where blob_hash = ?",
|
||||
(blob_hash,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
log.info("Saving sd blob hash %s to stream hash %s", str(sd_blob_hash), str(stream_hash))
|
||||
d = self.db_conn.runQuery("insert into lbry_file_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
|
||||
def ignore_duplicate(err):
|
||||
err.trap(sqlite3.IntegrityError)
|
||||
log.info("sd blob hash already known")
|
||||
|
||||
d.addErrback(ignore_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
log.debug("Looking up sd blob hashes for stream hash %s", str(stream_hash))
|
||||
d = self.db_conn.runQuery(
|
||||
"select sd_blob_hash from lbry_file_descriptors where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_hash_for_sd_blob_hash(self, sd_blob_hash):
|
||||
def _handle_result(result):
|
||||
if not result:
|
||||
raise NoSuchSDHash(sd_blob_hash)
|
||||
return result[0][0]
|
||||
|
||||
log.debug("Looking up sd blob hashes for sd blob hash %s", str(sd_blob_hash))
|
||||
d = self.db_conn.runQuery(
|
||||
"select stream_hash from lbry_file_descriptors where sd_blob_hash = ?",
|
||||
(sd_blob_hash,))
|
||||
d.addCallback(_handle_result)
|
||||
return d
|
||||
|
||||
|
||||
class TempEncryptedFileMetadataManager(object):
|
||||
def __init__(self):
|
||||
self.streams = {}
|
||||
self.stream_blobs = {}
|
||||
self.sd_files = {}
|
||||
|
||||
def setup(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return defer.succeed(self.streams.keys())
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
self.streams[stream_hash] = {'suggested_file_name': suggested_file_name,
|
||||
'stream_name': file_name,
|
||||
'key': key}
|
||||
d = self.add_blobs_to_stream(stream_hash, blobs)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
stream_info = self.streams[stream_hash]
|
||||
return defer.succeed([stream_info['key'], stream_info['stream_name'],
|
||||
stream_info['suggested_file_name']])
|
||||
return defer.succeed(None)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
del self.streams[stream_hash]
|
||||
for (s_h, b_h) in self.stream_blobs.keys():
|
||||
if s_h == stream_hash:
|
||||
del self.stream_blobs[(s_h, b_h)]
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
assert stream_hash in self.streams, "Can't add blobs to a stream that isn't known"
|
||||
for blob in blobs:
|
||||
info = {}
|
||||
info['blob_num'] = blob.blob_num
|
||||
info['length'] = blob.length
|
||||
info['iv'] = blob.iv
|
||||
self.stream_blobs[(stream_hash, blob.blob_hash)] = info
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None,
|
||||
end_blob=None, count=None, reverse=False):
|
||||
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
end_num = None
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num, count, reverse)
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
for (s_h, b_h) in self.stream_blobs.iterkeys():
|
||||
if b_h == blob_hash:
|
||||
return defer.succeed(s_h)
|
||||
return defer.succeed(None)
|
||||
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for (s_h, b_h), info in self.stream_blobs.iteritems():
|
||||
if stream_hash == s_h:
|
||||
position = info['blob_num']
|
||||
length = info['length']
|
||||
iv = info['iv']
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return defer.succeed(blob_infos)
|
||||
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
if (stream_hash, blob_hash) in self.stream_blobs:
|
||||
return defer.succeed(self.stream_blobs[(stream_hash, blob_hash)]['blob_num'])
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.sd_files[sd_blob_hash] = stream_hash
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return defer.succeed(
|
||||
[sd_hash for sd_hash, s_h in self.sd_files.iteritems() if stream_hash == s_h])
|
||||
@@ -1,268 +0,0 @@
|
||||
import logging
|
||||
import leveldb
|
||||
import json
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
|
||||
|
||||
class DBLBRYFileMetadataManager(object):
|
||||
"""Store and provide access to LBRY file metadata using leveldb files"""
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
|
||||
def setup(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
|
||||
def stop(self):
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return threads.deferToThread(self._get_all_streams)
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
d = threads.deferToThread(self._store_stream, stream_hash, file_name, key, suggested_file_name)
|
||||
d.addCallback(lambda _: self.add_blobs_to_stream(stream_hash, blobs))
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return threads.deferToThread(self._get_stream_info, stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_stream, stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
|
||||
def add_blobs():
|
||||
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
return threads.deferToThread(add_blobs)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
logging.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
end_num = None
|
||||
return start_num, end_num
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = threads.deferToThread(get_positions_of_start_and_end)
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
|
||||
|
||||
def _open_db(self):
|
||||
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_info.db"))
|
||||
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_blob.db"))
|
||||
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_desc.db"))
|
||||
|
||||
def _delete_stream(self, stream_hash):
|
||||
desc_batch = leveldb.WriteBatch()
|
||||
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
desc_batch.Delete(sd_blob_hash)
|
||||
self.stream_desc_db.Write(desc_batch, sync=True)
|
||||
|
||||
blob_batch = leveldb.WriteBatch()
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
blob_batch.Delete(blob_hash_stream_hash)
|
||||
self.stream_blob_db.Write(blob_batch, sync=True)
|
||||
|
||||
stream_batch = leveldb.WriteBatch()
|
||||
for s_h, stream_info in self.stream_info_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
stream_batch.Delete(s_h)
|
||||
self.stream_info_db.Write(stream_batch, sync=True)
|
||||
|
||||
def _store_stream(self, stream_hash, name, key, suggested_file_name):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
|
||||
except KeyError:
|
||||
pass
|
||||
self.stream_info_db.Put(stream_hash, json.dumps((key, name, suggested_file_name)), sync=True)
|
||||
|
||||
def _get_all_streams(self):
|
||||
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
|
||||
|
||||
def _get_stream_info(self, stream_hash):
|
||||
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
|
||||
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
|
||||
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
|
||||
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
position, iv, length = json.loads(blob_info)
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return blob_infos
|
||||
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_info in blob_infos:
|
||||
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
|
||||
try:
|
||||
self.stream_blob_db.Get(blob_hash_stream_hash)
|
||||
if ignore_duplicate_error is False:
|
||||
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
batch.Put(blob_hash_stream_hash,
|
||||
json.dumps((blob_info.blob_num,
|
||||
blob_info.iv,
|
||||
blob_info.length)))
|
||||
self.stream_blob_db.Write(batch, sync=True)
|
||||
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if blob_hash == b_h:
|
||||
return s_h
|
||||
return None
|
||||
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
|
||||
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
|
||||
|
||||
|
||||
class TempLBRYFileMetadataManager(object):
|
||||
def __init__(self):
|
||||
self.streams = {}
|
||||
self.stream_blobs = {}
|
||||
self.sd_files = {}
|
||||
|
||||
def setup(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return defer.succeed(self.streams.keys())
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
self.streams[stream_hash] = {'suggested_file_name': suggested_file_name,
|
||||
'stream_name': file_name,
|
||||
'key': key}
|
||||
d = self.add_blobs_to_stream(stream_hash, blobs)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
stream_info = self.streams[stream_hash]
|
||||
return defer.succeed([stream_info['key'], stream_info['stream_name'],
|
||||
stream_info['suggested_file_name']])
|
||||
return defer.succeed(None)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
del self.streams[stream_hash]
|
||||
for (s_h, b_h) in self.stream_blobs.keys():
|
||||
if s_h == stream_hash:
|
||||
del self.stream_blobs[(s_h, b_h)]
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
assert stream_hash in self.streams, "Can't add blobs to a stream that isn't known"
|
||||
for blob in blobs:
|
||||
info = {}
|
||||
info['blob_num'] = blob.blob_num
|
||||
info['length'] = blob.length
|
||||
info['iv'] = blob.iv
|
||||
self.stream_blobs[(stream_hash, blob.blob_hash)] = info
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
end_num = None
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num, count, reverse)
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
for (s_h, b_h) in self.stream_blobs.iterkeys():
|
||||
if b_h == blob_hash:
|
||||
return defer.succeed(s_h)
|
||||
return defer.succeed(None)
|
||||
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for (s_h, b_h), info in self.stream_blobs.iteritems():
|
||||
if stream_hash == s_h:
|
||||
position = info['blob_num']
|
||||
length = info['length']
|
||||
iv = info['iv']
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return defer.succeed(blob_infos)
|
||||
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
if (stream_hash, blob_hash) in self.stream_blobs:
|
||||
return defer.succeed(self.stream_blobs[(stream_hash, blob_hash)]['blob_num'])
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.sd_files[sd_blob_hash] = stream_hash
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return defer.succeed([sd_hash for sd_hash, s_h in self.sd_files.iteritems() if stream_hash == s_h])
|
||||
@@ -2,15 +2,20 @@ import binascii
|
||||
import logging
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from twisted.internet import defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from twisted.internet import defer, threads
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
|
||||
from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
|
||||
import os
|
||||
|
||||
|
||||
LBRYFileStreamType = "lbryfile"
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
EncryptedFileStreamType = "lbryfile"
|
||||
|
||||
|
||||
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
logging.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
log.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
hex_stream_name = sd_info['stream_name']
|
||||
key = sd_info['key']
|
||||
stream_hash = sd_info['stream_hash']
|
||||
@@ -26,7 +31,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
blob_num = blob['blob_num']
|
||||
iv = blob['iv']
|
||||
crypt_blobs.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
log.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
d = stream_info_manager.save_stream(stream_hash, hex_stream_name, key,
|
||||
suggested_file_name, crypt_blobs)
|
||||
|
||||
@@ -45,7 +50,7 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
|
||||
|
||||
def format_info(stream_info):
|
||||
fields = {}
|
||||
fields['stream_type'] = LBRYFileStreamType
|
||||
fields['stream_type'] = EncryptedFileStreamType
|
||||
fields['stream_name'] = stream_info[1]
|
||||
fields['key'] = stream_info[0]
|
||||
fields['suggested_file_name'] = stream_info[2]
|
||||
@@ -75,12 +80,47 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
|
||||
return d
|
||||
|
||||
|
||||
class LBRYFileStreamDescriptorValidator(object):
|
||||
@defer.inlineCallbacks
|
||||
def publish_sd_blob(stream_info_manager, blob_manager, stream_hash):
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
sd_info = yield get_sd_info(stream_info_manager, stream_hash, True)
|
||||
sd_blob_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
yield stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
defer.returnValue(sd_blob_hash)
|
||||
|
||||
|
||||
def create_plain_sd(stream_info_manager, stream_hash, file_name, overwrite_existing=False):
|
||||
|
||||
def _get_file_name():
|
||||
actual_file_name = file_name
|
||||
if os.path.exists(actual_file_name):
|
||||
ext_num = 1
|
||||
while os.path.exists(actual_file_name + "_" + str(ext_num)):
|
||||
ext_num += 1
|
||||
actual_file_name = actual_file_name + "_" + str(ext_num)
|
||||
return actual_file_name
|
||||
|
||||
if overwrite_existing is False:
|
||||
d = threads.deferToThread(_get_file_name())
|
||||
else:
|
||||
d = defer.succeed(file_name)
|
||||
|
||||
def do_create(file_name):
|
||||
descriptor_writer = PlainStreamDescriptorWriter(file_name)
|
||||
d = get_sd_info(stream_info_manager, stream_hash, True)
|
||||
d.addCallback(descriptor_writer.create_descriptor)
|
||||
return d
|
||||
|
||||
d.addCallback(do_create)
|
||||
return d
|
||||
|
||||
|
||||
class EncryptedFileStreamDescriptorValidator(object):
|
||||
def __init__(self, raw_info):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
try:
|
||||
hex_stream_name = self.raw_info['stream_name']
|
||||
key = self.raw_info['key']
|
||||
@@ -88,11 +128,11 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
stream_hash = self.raw_info['stream_hash']
|
||||
blobs = self.raw_info['blobs']
|
||||
except KeyError as e:
|
||||
raise ValueError("Invalid stream descriptor. Missing '%s'" % (e.args[0]))
|
||||
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
|
||||
for c in hex_suggested_file_name:
|
||||
if c not in '0123456789abcdef':
|
||||
raise ValueError("Invalid stream descriptor: "
|
||||
"suggested file name is not a hex-encoded string")
|
||||
raise InvalidStreamDescriptorError(
|
||||
"Suggested file name is not a hex-encoded string")
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(hex_stream_name)
|
||||
h.update(key)
|
||||
@@ -118,10 +158,11 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
for blob in blobs:
|
||||
blobs_hashsum.update(get_blob_hashsum(blob))
|
||||
if blobs[-1]['length'] != 0:
|
||||
raise ValueError("Improperly formed stream descriptor. Must end with a zero-length blob.")
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
h.update(blobs_hashsum.digest())
|
||||
if h.hexdigest() != stream_hash:
|
||||
raise ValueError("Stream hash does not match stream metadata")
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
log.debug("It is validated")
|
||||
return defer.succeed(True)
|
||||
|
||||
def info_to_show(self):
|
||||
@@ -130,9 +171,15 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
info.append(("stream_size", str(size_so_far)))
|
||||
info.append(("stream_size", str(self.get_length_of_stream())))
|
||||
suggested_file_name = self.raw_info.get("suggested_file_name", None)
|
||||
if suggested_file_name is not None:
|
||||
suggested_file_name = binascii.unhexlify(suggested_file_name)
|
||||
info.append(("suggested_file_name", suggested_file_name))
|
||||
return info
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
return size_so_far
|
||||
@@ -0,0 +1,2 @@
|
||||
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
|
||||
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
|
||||
+155
-68
@@ -3,23 +3,29 @@ import binascii
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbryfile.client.LBRYFileMetadataHandler import LBRYFileMetadataHandler
|
||||
from lbrynet.lbryfile.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
import os
|
||||
from twisted.internet import defer, threads, reactor
|
||||
from twisted.python.procutils import which
|
||||
import logging
|
||||
import traceback
|
||||
|
||||
|
||||
class LBRYFileDownloader(CryptStreamDownloader):
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed):
|
||||
stream_info_manager, payment_rate_manager, wallet):
|
||||
CryptStreamDownloader.__init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, upload_allowed)
|
||||
payment_rate_manager, wallet)
|
||||
self.stream_hash = stream_hash
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.suggested_file_name = None
|
||||
@@ -40,13 +46,39 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
def delete_data(self):
|
||||
d1 = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def get_blob_hashes(blob_infos):
|
||||
return [b[0] for b in blob_infos if b[0] is not None]
|
||||
|
||||
d1.addCallback(get_blob_hashes)
|
||||
d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash)
|
||||
|
||||
def combine_blob_hashes(results):
|
||||
blob_hashes = []
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
blob_hashes.extend(result)
|
||||
return blob_hashes
|
||||
|
||||
def delete_blobs(blob_hashes):
|
||||
self.blob_manager.delete_blobs(blob_hashes)
|
||||
return True
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
dl.addCallback(combine_blob_hashes)
|
||||
dl.addCallback(delete_blobs)
|
||||
return dl
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self))
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _start(self):
|
||||
d = self._setup_output()
|
||||
@@ -60,6 +92,15 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b[3] for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
if self.download_manager is None:
|
||||
return 0
|
||||
@@ -80,10 +121,11 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
return 0
|
||||
|
||||
def _get_metadata_handler(self, download_manager):
|
||||
return LBRYFileMetadataHandler(self.stream_hash, self.stream_info_manager, download_manager)
|
||||
return EncryptedFileMetadataHandler(self.stream_hash,
|
||||
self.stream_info_manager, download_manager)
|
||||
|
||||
|
||||
class LBRYFileDownloaderFactory(object):
|
||||
class EncryptedFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
@@ -94,55 +136,59 @@ class LBRYFileDownloaderFactory(object):
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.wallet = wallet
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager, **kwargs):
|
||||
if options[0] is not None:
|
||||
payment_rate_manager.float(options[0])
|
||||
upload_allowed = options[1]
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
|
||||
assert len(options) == 1
|
||||
data_rate = options[0]
|
||||
payment_rate_manager.min_blob_data_payment_rate = data_rate
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.stream_info_manager.save_sd_blob_hash_to_stream(
|
||||
stream_hash, metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager, sd_validator.raw_info,
|
||||
upload_allowed)
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager,
|
||||
metadata.validator.raw_info)
|
||||
d = downloader.set_stream_info()
|
||||
d.addCallback(lambda _: downloader)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
pass
|
||||
|
||||
|
||||
class LBRYFileSaver(LBRYFileDownloader):
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet, download_directory, upload_allowed, file_name=None):
|
||||
LBRYFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
|
||||
payment_rate_manager, wallet, download_directory, file_name=None):
|
||||
EncryptedFileDownloader.__init__(self, stream_hash,
|
||||
peer_finder, rate_limiter,
|
||||
blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet)
|
||||
self.download_directory = download_directory
|
||||
self.file_name = file_name
|
||||
self.file_written_to = None
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
if self.file_written_to is not None:
|
||||
return str(self.file_written_to)
|
||||
else:
|
||||
return str(self.file_name)
|
||||
|
||||
def set_stream_info(self):
|
||||
d = LBRYFileDownloader.set_stream_info(self)
|
||||
d = EncryptedFileDownloader.set_stream_info(self)
|
||||
|
||||
def set_file_name():
|
||||
if self.file_name is None:
|
||||
@@ -154,14 +200,15 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
d.addCallback(lambda _: set_file_name())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
download_manager)
|
||||
|
||||
def _setup_output(self):
|
||||
def open_file():
|
||||
@@ -171,11 +218,29 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
file_name = "_"
|
||||
if os.path.exists(os.path.join(self.download_directory, file_name)):
|
||||
ext_num = 1
|
||||
|
||||
def _get_file_name(ext):
|
||||
if len(file_name.split(".")):
|
||||
fn = ''.join(file_name.split(".")[:-1])
|
||||
file_ext = ''.join(file_name.split(".")[-1])
|
||||
return fn + "-" + str(ext) + "." + file_ext
|
||||
else:
|
||||
return file_name + "_" + str(ext)
|
||||
|
||||
while os.path.exists(os.path.join(self.download_directory,
|
||||
file_name + "_" + str(ext_num))):
|
||||
_get_file_name(ext_num))):
|
||||
ext_num += 1
|
||||
file_name = file_name + "_" + str(ext_num)
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
|
||||
file_name = _get_file_name(ext_num)
|
||||
try:
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
self.file_written_to = os.path.join(self.download_directory, file_name)
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise ValueError(
|
||||
"Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." %
|
||||
os.path.join(self.download_directory, file_name))
|
||||
return threads.deferToThread(open_file)
|
||||
|
||||
def _close_output(self):
|
||||
@@ -200,38 +265,45 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
return self.stream_info_manager.delete_stream(self.stream_hash)
|
||||
|
||||
|
||||
class LBRYFileSaverFactory(LBRYFileDownloaderFactory):
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
wallet, download_directory):
|
||||
LBRYFileDownloaderFactory.__init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
EncryptedFileDownloaderFactory.__init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, wallet)
|
||||
self.download_directory = download_directory
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
|
||||
return LBRYFileSaver(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet,
|
||||
self.download_directory, upload_allowed)
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
return EncryptedFileSaver(stream_hash, self.peer_finder,
|
||||
self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager,
|
||||
payment_rate_manager, self.wallet,
|
||||
self.download_directory)
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save"
|
||||
|
||||
|
||||
class LBRYFileOpener(LBRYFileDownloader):
|
||||
class EncryptedFileOpener(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet, upload_allowed):
|
||||
LBRYFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
|
||||
payment_rate_manager, wallet):
|
||||
EncryptedFileDownloader.__init__(self, stream_hash,
|
||||
peer_finder, rate_limiter,
|
||||
blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet,
|
||||
)
|
||||
self.process = None
|
||||
self.process_log = None
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
download_manager)
|
||||
|
||||
def _setup_output(self):
|
||||
def start_process():
|
||||
@@ -275,10 +347,25 @@ class LBRYFileOpener(LBRYFileDownloader):
|
||||
return self.stream_info_manager.delete_stream(self.stream_hash)
|
||||
|
||||
|
||||
class LBRYFileOpenerFactory(LBRYFileDownloaderFactory):
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
|
||||
return LBRYFileOpener(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet, upload_allowed)
|
||||
class EncryptedFileOpenerFactory(EncryptedFileDownloaderFactory):
|
||||
def can_download(self, sd_validator):
|
||||
if which('vlc'):
|
||||
return True
|
||||
elif os.name == "nt":
|
||||
paths = [r'C:\Program Files\VideoLAN\VLC\vlc.exe',
|
||||
r'C:\Program Files (x86)\VideoLAN\VLC\vlc.exe']
|
||||
for p in paths:
|
||||
if os.path.exists(p):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_description(self):
|
||||
return "Stream"
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
return EncryptedFileOpener(stream_hash, self.peer_finder,
|
||||
self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager,
|
||||
payment_rate_manager, self.wallet,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Stream"
|
||||
+6
-3
@@ -4,7 +4,10 @@ from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from lbrynet.interfaces import IMetadataHandler
|
||||
|
||||
|
||||
class LBRYFileMetadataHandler(object):
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileMetadataHandler(object):
|
||||
implements(IMetadataHandler)
|
||||
|
||||
def __init__(self, stream_hash, stream_info_manager, download_manager):
|
||||
@@ -31,6 +34,6 @@ class LBRYFileMetadataHandler(object):
|
||||
if blob_hash is not None:
|
||||
infos.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
else:
|
||||
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
self._final_blob_num = blob_num - 1
|
||||
return infos
|
||||
return infos
|
||||
Loaded 100 of 260 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user