Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7932e29fb0 | ||
|
|
127f828852 | ||
|
|
8f918bab87 | ||
|
|
79e0a823e9 | ||
|
|
0005ba0715 | ||
|
|
13d74d1f76 | ||
|
|
ddf678af8a | ||
|
|
2c6d6d1602 | ||
|
|
7d0e9f6ab9 | ||
|
|
31a90e7f60 | ||
|
|
2bf098b5a3 | ||
|
|
458561fe6f | ||
|
|
157d6dca93 | ||
|
|
7b1b7fbf13 | ||
|
|
876cdce51f | ||
|
|
205a10bc8e | ||
|
|
ef8d1cfdc2 | ||
|
|
879c01ba00 | ||
|
|
8bb7517f32 | ||
|
|
70d9b3ed74 | ||
|
|
f39a21cf7b | ||
|
|
67b169c5a1 | ||
|
|
ff10546152 | ||
|
|
bd6fe4f986 | ||
|
|
360f4e50a0 | ||
|
|
f11f486f5a | ||
|
|
fe53cc97d9 | ||
|
|
baf6ff3f50 | ||
|
|
051e63d76b | ||
|
|
8cdda7d845 | ||
|
|
6a8789050c | ||
|
|
9140c95a0c | ||
|
|
4b52c88d45 | ||
|
|
b7dd9c176a | ||
|
|
6f2e2e727e | ||
|
|
48b137fe4c | ||
|
|
a6326ee7a1 | ||
|
|
c91034ee4c | ||
|
|
7c505f2e23 | ||
|
|
bfcd84a568 | ||
|
|
a314c530bc | ||
|
|
4cb2a8f2aa | ||
|
|
9d272befcd | ||
|
|
1695b26416 | ||
|
|
91fb78b077 | ||
|
|
0bbc8f5b7e | ||
|
|
464bd11c3a | ||
|
|
b88e771eb2 | ||
|
|
7a76763610 | ||
|
|
02d2bad03f | ||
|
|
f65a50a6a3 | ||
|
|
0a9ef07787 | ||
|
|
0e9ad16a2e | ||
|
|
cd0a9ba9e8 | ||
|
|
e08d0eb8f9 | ||
|
|
82f9cc57fd | ||
|
|
ae53129b25 | ||
|
|
543f65fc24 | ||
|
|
896a16c15d | ||
|
|
3d7dee1208 | ||
|
|
e03db79f79 | ||
|
|
0b0150ad73 | ||
|
|
0490250da4 | ||
|
|
dd952ddbe3 | ||
|
|
776fe41e03 | ||
|
|
8b1ea0a6a7 | ||
|
|
183617c85b | ||
|
|
386c67fa70 | ||
|
|
ba7f326e72 | ||
|
|
b9caf2c4a9 | ||
|
|
3188165514 | ||
|
|
1c036cce78 | ||
|
|
a2e648fcfc | ||
|
|
b75a89e876 | ||
|
|
82a3b5e9c5 | ||
|
|
7ecc8103d8 | ||
|
|
6b72b4de11 | ||
|
|
8e1ad0aa8b | ||
|
|
5b7f80487f | ||
|
|
ebc400804f | ||
|
|
9940327b9a | ||
|
|
b300afb504 | ||
|
|
0941618048 | ||
|
|
abc7b11e26 | ||
|
|
85dee27c92 | ||
|
|
5ae3485411 | ||
|
|
8f98469969 | ||
|
|
e1cdbdc91d | ||
|
|
908c9a3bfa | ||
|
|
f45a2cba32 | ||
|
|
763fefb33f | ||
|
|
ce7463f684 | ||
|
|
cd65b6fd13 | ||
|
|
472bb6af1b | ||
|
|
7f96e35b0e | ||
|
|
f9fef3d586 | ||
|
|
ec4f9011b9 | ||
|
|
d634c009d3 | ||
|
|
1872256d7a | ||
|
|
81d107d99c | ||
|
|
a7589914d0 | ||
|
|
0736fa506b | ||
|
|
dfec3c1a15 | ||
|
|
edc89be794 | ||
|
|
d41881596d | ||
|
|
3b34bddf0a | ||
|
|
23c012f417 | ||
|
|
2c8d643f4b | ||
|
|
0abc081edc | ||
|
|
11b36ff4c8 | ||
|
|
c407d32f5d | ||
|
|
87cc487051 | ||
|
|
5ea945ca0b | ||
|
|
1f06339d5a | ||
|
|
09d248b3cc | ||
|
|
28bf5215e4 | ||
|
|
f3a71245e8 | ||
|
|
37fee71898 | ||
|
|
61a354fa60 | ||
|
|
1d4b2c3527 | ||
|
|
694b5b9ee5 | ||
|
|
838508f342 | ||
|
|
694db88ac1 | ||
|
|
cb7c849273 | ||
|
|
9d82be6664 | ||
|
|
66745b0820 | ||
|
|
f845729adb | ||
|
|
6a587d03cf | ||
|
|
affb7d837a | ||
|
|
4ad6405f35 | ||
|
|
906e6b238e | ||
|
|
297865902b | ||
|
|
ab15074ae4 | ||
|
|
b68a1838f2 | ||
|
|
43b2cd1433 | ||
|
|
0036685ece | ||
|
|
d06c670a12 | ||
|
|
6d6cfbc7cc | ||
|
|
3c977f961f | ||
|
|
59c4593bf4 | ||
|
|
5bccfdb244 | ||
|
|
10cb20d08f | ||
|
|
693fef1964 | ||
|
|
fe7700d726 | ||
|
|
fee6de1c1f | ||
|
|
7d04ae2237 | ||
|
|
c748572bf6 | ||
|
|
660bf74d42 | ||
|
|
4db3be345b | ||
|
|
dcea8a8c78 | ||
|
|
815696485f | ||
|
|
b316b56885 | ||
|
|
269dc47b69 | ||
|
|
a7e3246a82 | ||
|
|
96ef5089fc | ||
|
|
c0d96a5733 | ||
|
|
7de4340c38 | ||
|
|
7dd5e7ceeb | ||
|
|
9de7354ea0 | ||
|
|
cfbdda411a | ||
|
|
96c4837cb8 | ||
|
|
2294e6bf36 | ||
|
|
4cffc32328 | ||
|
|
d1b23e25d9 | ||
|
|
e5ba70c3ba | ||
|
|
c88e4aecf5 | ||
|
|
469df84990 | ||
|
|
a4ec706b06 | ||
|
|
30533a7986 | ||
|
|
39b47bc569 | ||
|
|
96104e65ed | ||
|
|
e9bab33e2a | ||
|
|
3b6006d759 | ||
|
|
9535a24706 | ||
|
|
8f2be46536 | ||
|
|
9ca80a7cf7 | ||
|
|
72078f4af3 | ||
|
|
ff9ea37e0c | ||
|
|
9baf261cb3 | ||
|
|
f03a15d503 | ||
|
|
1a70a20c8b | ||
|
|
359cb83fab | ||
|
|
0893858385 | ||
|
|
e1997a6852 | ||
|
|
0f4fee475f | ||
|
|
864e963dea | ||
|
|
3d0debd775 | ||
|
|
6cc2b48a06 | ||
|
|
dee0917642 | ||
|
|
395a68b261 | ||
|
|
0d066b2d11 | ||
|
|
353ede26b8 | ||
|
|
46cfa20f65 | ||
|
|
30b1dd3ce6 | ||
|
|
da601818a3 | ||
|
|
6fa624ec8a | ||
|
|
779ba8e214 | ||
|
|
80f548fdfd | ||
|
|
6f9ec2fa66 | ||
|
|
40437e4a82 | ||
|
|
2ee6fac014 | ||
|
|
8fe5eb514a | ||
|
|
c4312cc387 | ||
|
|
110a9b1e0f | ||
|
|
035a1cf758 | ||
|
|
d4785849e1 | ||
|
|
b41756e38a | ||
|
|
60ee278519 | ||
|
|
b49e44926a | ||
|
|
81187da5b7 | ||
|
|
e70ecc8ac9 | ||
|
|
862055a6de | ||
|
|
7076c0c870 | ||
|
|
2cd6c644a0 | ||
|
|
0edacbe4c8 | ||
|
|
4b330d0e9f | ||
|
|
8ce08f7da1 | ||
|
|
127987f3ba | ||
|
|
3c7eec9456 | ||
|
|
5d45345398 | ||
|
|
b360eb6181 | ||
|
|
369cd516c0 | ||
|
|
24320d6444 | ||
|
|
1951ea09cd | ||
|
|
4d391e7429 | ||
|
|
fb88cdb445 | ||
|
|
572f1d1f6e | ||
|
|
27a8114b6f | ||
|
|
6f3873d99b | ||
|
|
e62e2cc440 | ||
|
|
a5797cdeb3 | ||
|
|
3dd99fdc92 | ||
|
|
0c2265a8da | ||
|
|
01811621a6 | ||
|
|
229ed0d7dc | ||
|
|
7f800ce5bb | ||
|
|
aa291ca79a | ||
|
|
813267ae18 | ||
|
|
fbf7928412 | ||
|
|
7167d47631 | ||
|
|
8b1bb673c1 | ||
|
|
417a8f719f | ||
|
|
ddb88eb1c9 | ||
|
|
d5f0001950 | ||
|
|
4a07b56e8a | ||
|
|
9da1b3c5b7 | ||
|
|
f9f07c47e2 | ||
|
|
f5213fb77a | ||
|
|
48a61605de | ||
|
|
76fb7697ff | ||
|
|
b71a3fee43 | ||
|
|
3a91896d8a | ||
|
|
36ae0d5f20 | ||
|
|
3f727d892c | ||
|
|
c104ed3f8b | ||
|
|
9416376a34 | ||
|
|
66e139c566 | ||
|
|
d0a82c68df | ||
|
|
93e2b3e20a | ||
|
|
2e59e5e3b8 | ||
|
|
727ea12dbd | ||
|
|
ea3225c897 | ||
|
|
bfcfa9dc3c | ||
|
|
ea21821558 | ||
|
|
a9ae1a1016 | ||
|
|
3f5efb1fa3 | ||
|
|
9ba38c84cc | ||
|
|
a5d3c0bbe1 | ||
|
|
d4aa109b73 | ||
|
|
bdeccb49b8 | ||
|
|
35ce7bbd61 | ||
|
|
293933163f | ||
|
|
22fc26797d | ||
|
|
8a95c8b7af | ||
|
|
33674f8cc3 | ||
|
|
6799de645b | ||
|
|
06a88599df | ||
|
|
03877f7ebd | ||
|
|
9f2dbd4291 | ||
|
|
6b7d575e95 | ||
|
|
640db41555 | ||
|
|
d860c2a2b4 | ||
|
|
733bb3bccf | ||
|
|
307751277d | ||
|
|
62c2539475 | ||
|
|
5b15da2c55 | ||
|
|
830f5ff6e3 | ||
|
|
ea1f260ed9 | ||
|
|
3e86254324 | ||
|
|
b5311dd1c3 | ||
|
|
a8a581d35c | ||
|
|
5f129faf32 | ||
|
|
018d78be6f | ||
|
|
b5a3929bb5 | ||
|
|
b67b4b2ef6 | ||
|
|
bed3f551fd | ||
|
|
5b0b8a19ff | ||
|
|
988c9595c7 | ||
|
|
92700808eb | ||
|
|
0dd9aa0d67 | ||
|
|
9cf1d6b710 | ||
|
|
d8d462f43c | ||
|
|
3f22f39ce1 | ||
|
|
e647663c34 | ||
|
|
b5987c75da | ||
|
|
32a973943f | ||
|
|
99c060719a | ||
|
|
d9cd215a60 | ||
|
|
2d3b1fd4eb | ||
|
|
19c2264429 | ||
|
|
ae4b984f0b | ||
|
|
897c113db1 | ||
|
|
50e5a59420 | ||
|
|
8735112f70 | ||
|
|
5a0b5dc680 | ||
|
|
d91e9aa910 | ||
|
|
be1803d558 | ||
|
|
84bbf2f76e | ||
|
|
fe7ea7c679 | ||
|
|
782e197e66 | ||
|
|
88be425ab2 | ||
|
|
7025c5102a | ||
|
|
1ee9b3c7a6 | ||
|
|
4f60a98eb2 | ||
|
|
a71e87a398 | ||
|
|
0ecfad0ae9 | ||
|
|
e909ae566b | ||
|
|
dc2f0adb3e | ||
|
|
180d981ed8 | ||
|
|
fd9bc6214a | ||
|
|
b3f3ed8026 | ||
|
|
967700dc28 | ||
|
|
3ad4ad50ec | ||
|
|
7f980446db | ||
|
|
04ced2e975 | ||
|
|
2253943eba | ||
|
|
a6fcd5d1f2 | ||
|
|
f245822814 | ||
|
|
aa3aff91d0 | ||
|
|
5537dd878f | ||
|
|
5e4e619708 | ||
|
|
d741c705cd | ||
|
|
d83abd81f8 | ||
|
|
640e000b9f | ||
|
|
8d961d6ce6 | ||
|
|
0a612ba36c | ||
|
|
eb7cc8463b | ||
|
|
50f0430d04 | ||
|
|
6e5ea935f6 | ||
|
|
c4eb3de5b2 | ||
|
|
f79b651b05 | ||
|
|
c130879ef7 | ||
|
|
5c391f4bb4 | ||
|
|
b853656aa6 | ||
|
|
8222f940b4 | ||
|
|
71b989b29b | ||
|
|
7e2f7e6f68 | ||
|
|
aefb6c5dc8 | ||
|
|
b892b81dad | ||
|
|
4d63629919 | ||
|
|
4727ee0403 | ||
|
|
fc47e8074f | ||
|
|
ff7829a078 | ||
|
|
d05bbea131 | ||
|
|
e08a681c8f | ||
|
|
6a72b3683e | ||
|
|
b201fc331c | ||
|
|
27337a9243 | ||
|
|
b671a3c2ba | ||
|
|
1cc6b7658c | ||
|
|
12e2827286 | ||
|
|
548d37929e | ||
|
|
eb783b6b6f | ||
|
|
d1b06c3688 | ||
|
|
d16b3429cd | ||
|
|
1e4b5cb450 | ||
|
|
a9da6e4bdf | ||
|
|
a099e2db6a | ||
|
|
7dbeb1f2cf | ||
|
|
550fd185a3 | ||
|
|
1720cce3b0 | ||
|
|
078b254d1d | ||
|
|
6057dc0804 | ||
|
|
5ea7c4f016 | ||
|
|
4334cdace1 | ||
|
|
6803bfae99 | ||
|
|
c49b0fdd49 | ||
|
|
bf3c56f0ec | ||
|
|
bbcecea28d | ||
|
|
a936782ddc | ||
|
|
7ac0fc5a24 | ||
|
|
e115158f86 | ||
|
|
bf34fa39b4 | ||
|
|
5dd29da84f | ||
|
|
2107ffdd18 | ||
|
|
492fc510fa | ||
|
|
c75d240a65 | ||
|
|
dbeb6bc3bc | ||
|
|
4d79a7828c | ||
|
|
aecafbbebd | ||
|
|
130f9cfc4d | ||
|
|
98d02b3fce | ||
|
|
4d52a33ee5 | ||
|
|
7ad916f116 | ||
|
|
05b47a9180 | ||
|
|
9a412bbc76 | ||
|
|
b3a16f95f7 | ||
|
|
871b3be09a | ||
|
|
7085b1e3c3 | ||
|
|
a9e4343775 | ||
|
|
3d4d895d9b | ||
|
|
3d89e3fd93 | ||
|
|
a8e2dd3f2a | ||
|
|
3ae0664fa9 | ||
|
|
98b9f2cf73 | ||
|
|
6fa65ebb9f | ||
|
|
add163a092 | ||
|
|
fcfed3f441 | ||
|
|
7b84d9b10a | ||
|
|
e9a1dbca4b | ||
|
|
145a2654d2 | ||
|
|
9f14cdb1d7 | ||
|
|
5cefa2e995 | ||
|
|
13cbe0fd42 | ||
|
|
c8bb0c0bc8 | ||
|
|
4cc941fbec | ||
|
|
b010192f8e | ||
|
|
2b1af8eea2 | ||
|
|
6ea1f6f78c | ||
|
|
1215470cc1 | ||
|
|
94195b7018 | ||
|
|
3c9c178bdc | ||
|
|
cc64547920 | ||
|
|
a23f2ca956 | ||
|
|
4dacc52ecb | ||
|
|
ccaeecfd46 | ||
|
|
3e45449e8c | ||
|
|
f352a3c597 | ||
|
|
d9a08f0c03 | ||
|
|
c77f24dde3 | ||
|
|
48c31a2119 | ||
|
|
5a578d6d79 | ||
|
|
8cdbc847d1 | ||
|
|
54dbf66cb7 | ||
|
|
177d384126 | ||
|
|
561fe757f3 | ||
|
|
c670ab638f | ||
|
|
eea5cbbafc | ||
|
|
e4c61a4a13 | ||
|
|
0f0afadc94 | ||
|
|
659563f27a | ||
|
|
cbd8a68d0f | ||
|
|
54229282a4 | ||
|
|
514575d0f1 | ||
|
|
756427560c | ||
|
|
900eeb177a | ||
|
|
e67fa3cadc | ||
|
|
c7c32fd4dc | ||
|
|
9471748f4c | ||
|
|
920b1868bb | ||
|
|
eed7c54aaf | ||
|
|
36e58a33ad | ||
|
|
93e8ff749c | ||
|
|
6cb22444ac | ||
|
|
78df101473 | ||
|
|
840508b8ba | ||
|
|
2761254790 | ||
|
|
f47d174f0b | ||
|
|
c50fbc7935 | ||
|
|
75f23a1723 | ||
|
|
f0aa31f9e8 | ||
|
|
aaf59218da | ||
|
|
0203ef66ea | ||
|
|
9c66108dc5 | ||
|
|
558da0a027 | ||
|
|
1d39ec0317 | ||
|
|
281b6468a0 | ||
|
|
9ba6da1f4d | ||
|
|
bde7a7fed6 | ||
|
|
7e021746e4 | ||
|
|
a61bb5f528 | ||
|
|
6a5289b2a2 | ||
|
|
081a5df82c | ||
|
|
3e77ec8008 | ||
|
|
b0b222afa8 | ||
|
|
d85d6f627d | ||
|
|
9505f93da6 | ||
|
|
1671034330 | ||
|
|
3f0ca563e4 | ||
|
|
ffa2a5e8fd | ||
|
|
c3633895d0 | ||
|
|
c15af0a63a | ||
|
|
56e2d77039 | ||
|
|
2cf6fa14a1 | ||
|
|
c374725255 | ||
|
|
babfb6b8d6 | ||
|
|
2ffa8ec7d0 | ||
|
|
b516119224 | ||
|
|
c0e6bb97d8 | ||
|
|
cc30223ff1 | ||
|
|
53c8748942 | ||
|
|
b3a8966e71 | ||
|
|
74800a0767 | ||
|
|
cadd1f993d | ||
|
|
d3807cb86c | ||
|
|
735e9195b5 | ||
|
|
a6619a628e | ||
|
|
456572f87c | ||
|
|
f024b96934 | ||
|
|
25c71bcd98 | ||
|
|
319e7ecc61 | ||
|
|
d9ece5e96a | ||
|
|
4bfe4450b1 | ||
|
|
f9f12ffb61 | ||
|
|
63f81d1033 | ||
|
|
8895d21abf | ||
|
|
2ac1ebeaa0 | ||
|
|
25437602d7 | ||
|
|
8159c71950 | ||
|
|
aa66af7ba6 | ||
|
|
be3633622c | ||
|
|
31655a641c | ||
|
|
b235f6fc6b | ||
|
|
76b297b173 | ||
|
|
1e5cff1d8b | ||
|
|
4e2296d5f6 | ||
|
|
18310632eb | ||
|
|
80ed8d9869 | ||
|
|
2a2658d440 | ||
|
|
710e549c03 | ||
|
|
47fae977dd | ||
|
|
0111919975 | ||
|
|
80837cb040 | ||
|
|
46153a8991 | ||
|
|
a9fccfd4ce | ||
|
|
dfb57e9438 | ||
|
|
eda2731702 | ||
|
|
f47eac5935 | ||
|
|
3bb4dece5a | ||
|
|
0504e231a1 | ||
|
|
6097e14617 | ||
|
|
0b149ce5f1 | ||
|
|
79384e7bf8 | ||
|
|
01eb79cac2 | ||
|
|
128b32c62c | ||
|
|
c1233bd26b | ||
|
|
e23f67def6 | ||
|
|
95f4b29be5 | ||
|
|
5727c708cf | ||
|
|
e49f0f99a1 | ||
|
|
84d340382b | ||
|
|
5bfb067d0f | ||
|
|
d8cb62a92c | ||
|
|
aab8cc40f9 | ||
|
|
9c82689f65 | ||
|
|
ce2755467d | ||
|
|
02b19402ae | ||
|
|
80af66e852 | ||
|
|
15cfa45646 | ||
|
|
ac63849b49 | ||
|
|
02142e352b | ||
|
|
3d92413372 | ||
|
|
5022ed2acc | ||
|
|
31b9d22649 | ||
|
|
7a54894015 | ||
|
|
7100345a09 | ||
|
|
929db15214 | ||
|
|
d9455926c5 | ||
|
|
ba5cd741e1 | ||
|
|
8c94b6243e | ||
|
|
f6473e3383 | ||
|
|
3cd9d6a604 | ||
|
|
142281777e | ||
|
|
8e7b8af180 | ||
|
|
c046cd02d5 | ||
|
|
f5508fcdf1 | ||
|
|
adc2eab6da | ||
|
|
702698d8aa | ||
|
|
73dbe1de37 | ||
|
|
c5706fa28b | ||
|
|
c906a9571e | ||
|
|
038712f431 | ||
|
|
ed2d2c6f86 | ||
|
|
e7158e5a4b | ||
|
|
c66df259a6 | ||
|
|
0b8437fa12 | ||
|
|
a32b6ebf82 | ||
|
|
edbda1696e | ||
|
|
ece218785c | ||
|
|
cc1b3609a1 | ||
|
|
0841aad07a | ||
|
|
30b04ed0c2 | ||
|
|
90b28b3dbd | ||
|
|
0f7ca8f24e | ||
|
|
726d85ebd0 | ||
|
|
c524de9198 | ||
|
|
2fbf03d8ca | ||
|
|
68b8112af0 | ||
|
|
975e83a015 | ||
|
|
1ddb02dbb7 | ||
|
|
1159204ed8 | ||
|
|
a5212e7741 | ||
|
|
1932fd72e3 | ||
|
|
c061fe091b | ||
|
|
3621983a4d | ||
|
|
b6656690ca | ||
|
|
0c9e82ee6b | ||
|
|
b8ebb1faf1 | ||
|
|
690575193f | ||
|
|
9e9cf63c90 | ||
|
|
f5dc213e53 | ||
|
|
138e9c1486 | ||
|
|
7053c0dc93 | ||
|
|
ee8a653b13 | ||
|
|
71fad7e97f | ||
|
|
04740eb928 | ||
|
|
8c63d58c96 | ||
|
|
991672d423 | ||
|
|
55ce38cc87 | ||
|
|
96f3390407 | ||
|
|
aa2426d4e4 | ||
|
|
d0e3e13852 | ||
|
|
a2a1e30d68 | ||
|
|
a6f08b9e0c | ||
|
|
e70842a38c | ||
|
|
2af98d7d42 | ||
|
|
f28eee34e8 | ||
|
|
208610b645 | ||
|
|
e38f8076bc | ||
|
|
6180e1a48b | ||
|
|
e8fe1cd14d | ||
|
|
75d1fa585c | ||
|
|
072ff4b02e | ||
|
|
d8c4e2a72e | ||
|
|
66396de6a3 | ||
|
|
1ca9f575ef | ||
|
|
272e074a15 | ||
|
|
9a738c3783 | ||
|
|
6b4519e959 | ||
|
|
9b1d5213a6 | ||
|
|
f9ed9a4505 | ||
|
|
3ed7ce5630 | ||
|
|
63cd8343c4 | ||
|
|
2d5f8aed24 | ||
|
|
b8d23d0965 | ||
|
|
f4ef92e653 | ||
|
|
c7430f4ae9 | ||
|
|
623fda3087 | ||
|
|
80f27f2b29 | ||
|
|
defbd73b4f | ||
|
|
5285982515 | ||
|
|
f83daa5a22 | ||
|
|
0a07c8f131 | ||
|
|
35481a92f5 | ||
|
|
0995d864e8 | ||
|
|
1a42e43573 | ||
|
|
132aa569fe | ||
|
|
e93a787bd6 | ||
|
|
d6f902653f | ||
|
|
d172d43ddf | ||
|
|
67909724e7 | ||
|
|
1c88cbc5f6 | ||
|
|
3fb4831804 | ||
|
|
b65570fb16 | ||
|
|
871e6e6f64 | ||
|
|
0cd92a96c0 | ||
|
|
ff31ddd2d5 | ||
|
|
e9f5b43785 | ||
|
|
36b41081ab | ||
|
|
1092313892 | ||
|
|
ce605cb8f3 | ||
|
|
0733d885fb | ||
|
|
ef6fe3d1d9 | ||
|
|
82a4fea81a | ||
|
|
14a0252819 | ||
|
|
849aeb65f0 | ||
|
|
a99dc6e93d | ||
|
|
1a5fcc1348 | ||
|
|
c9d88f5628 | ||
|
|
0910711aca | ||
|
|
6a1a9ef6bc | ||
|
|
ef1c059079 | ||
|
|
a05ab37373 | ||
|
|
7ea88cfd22 | ||
|
|
8d14fe6a90 | ||
|
|
49a05b42f9 | ||
|
|
b6da1d7769 | ||
|
|
7e2ad58edd | ||
|
|
7ad0d98404 | ||
|
|
b7052bba95 | ||
|
|
4a5618bcc4 | ||
|
|
c1850fe604 | ||
|
|
fccdf32c9f | ||
|
|
c474411e1b | ||
|
|
9033eac5b1 | ||
|
|
0ba0426591 | ||
|
|
9a9b540701 | ||
|
|
b8ddbb9a26 | ||
|
|
bc3c2b96ed | ||
|
|
622dc87d84 | ||
|
|
18edbdf404 | ||
|
|
02a3eab693 | ||
|
|
dded868a80 | ||
|
|
6fe878bdb9 | ||
|
|
505ece9e36 | ||
|
|
72acfa1e0c | ||
|
|
02cd8b9a44 | ||
|
|
f32250f431 | ||
|
|
1523cad044 | ||
|
|
ed55353700 | ||
|
|
8d3bd45102 | ||
|
|
f69753f6a0 | ||
|
|
b7e2e87ac1 | ||
|
|
3b98508986 | ||
|
|
36ebfc42d9 | ||
|
|
ae5f55ff65 | ||
|
|
01131da5b7 | ||
|
|
66ae865de0 | ||
|
|
8c50f08fec | ||
|
|
e69de47daf | ||
|
|
acef10e848 | ||
|
|
edb9d50d9e | ||
|
|
b989d4f6be | ||
|
|
84dd5c0eca | ||
|
|
db8daf20b7 | ||
|
|
cb985b5a03 | ||
|
|
28c4e38272 | ||
|
|
b29377dc2b | ||
|
|
ca09592a72 | ||
|
|
b0e14fae43 | ||
|
|
dfa68f126b | ||
|
|
4570f68727 | ||
|
|
4705acba55 | ||
|
|
1dea20a358 | ||
|
|
191f661b35 | ||
|
|
f3ab38ba8f | ||
|
|
2d0a892e70 | ||
|
|
00eefe87d7 | ||
|
|
f68fd96555 | ||
|
|
fc50a3242f | ||
|
|
64e9ce2bfa | ||
|
|
e8d04b1921 | ||
|
|
9fedf14e55 | ||
|
|
7d535ef7d1 | ||
|
|
85d610bcdf | ||
|
|
3af8b7bd0c | ||
|
|
6215d85581 | ||
|
|
308d9f5abd | ||
|
|
bcfabba038 | ||
|
|
69139b1987 | ||
|
|
3f8f9d9d73 | ||
|
|
2283579109 | ||
|
|
1e0af94ba2 | ||
|
|
dd631328e0 | ||
|
|
cca2a57c69 | ||
|
|
bbfbcacc6a | ||
|
|
3f95c1a9d5 | ||
|
|
7bd05e9b16 | ||
|
|
10742d7c51 | ||
|
|
bbf7940e45 | ||
|
|
f5af71f4e6 | ||
|
|
1ffcd67c9d | ||
|
|
cf8592ba12 | ||
|
|
1e1d036590 | ||
|
|
64afb6d305 | ||
|
|
20a9e018f7 | ||
|
|
a574c33ac6 | ||
|
|
d21e458fe1 | ||
|
|
bb3ba53c6e | ||
|
|
ba2e4df76b | ||
|
|
33e7e4bee3 | ||
|
|
b83dce2cb3 | ||
|
|
6821676853 | ||
|
|
52024986ee | ||
|
|
2e3342d20d | ||
|
|
419aa1d78f | ||
|
|
de4ddd578f | ||
|
|
5dd68a7ed4 | ||
|
|
881168dc7a | ||
|
|
295e427abe | ||
|
|
67c5d15f51 | ||
|
|
39a7f37f35 | ||
|
|
15d276b0bc | ||
|
|
4387025c48 | ||
|
|
52c20f02c7 | ||
|
|
413dab8094 | ||
|
|
6b1d11a04d | ||
|
|
5441379f38 | ||
|
|
52d954c64e | ||
|
|
415495fc16 | ||
|
|
4223298634 | ||
|
|
e058c71019 | ||
|
|
12b2bbfa26 | ||
|
|
7d22bfffa5 | ||
|
|
4ade80c8a9 | ||
|
|
9674381fc2 | ||
|
|
f6d91280d7 | ||
|
|
e4ebbd8f69 | ||
|
|
7f5560de6d | ||
|
|
0925171e07 | ||
|
|
900d3ad01d | ||
|
|
6a1efedc1f | ||
|
|
0f602e0e62 | ||
|
|
3a425fc47d | ||
|
|
36722cbbd8 | ||
|
|
9572336e96 | ||
|
|
4895cde474 | ||
|
|
21ca1a73cf | ||
|
|
457c1c6ae6 | ||
|
|
c1ff127101 | ||
|
|
5e815a5832 | ||
|
|
4604e9cd54 | ||
|
|
ef580a7680 | ||
|
|
2102776be8 | ||
|
|
f3af7a6b77 | ||
|
|
5abeceac1f | ||
|
|
22f73a0817 | ||
|
|
29b5aef73a | ||
|
|
3e602bce52 | ||
|
|
fe39901885 | ||
|
|
ee9e8b365c | ||
|
|
bd6fa35d4a | ||
|
|
5e1fdf66b0 | ||
|
|
d86e709b90 | ||
|
|
f82cffe882 | ||
|
|
3d88eb92c1 | ||
|
|
0811ebb52d | ||
|
|
491d431ea5 | ||
|
|
ceeebebcad | ||
|
|
04d766aba7 | ||
|
|
3b431e180e | ||
|
|
a2eab1577d | ||
|
|
f8dd3d05dc | ||
|
|
93993e62d6 | ||
|
|
483c88b958 | ||
|
|
4a1e846958 | ||
|
|
927b9afe7c | ||
|
|
9e8827f015 | ||
|
|
ca45f461ae | ||
|
|
7e5af11c2d | ||
|
|
102436274f | ||
|
|
49c26a76b4 | ||
|
|
d469ef7bf2 | ||
|
|
a5374cb876 | ||
|
|
a53e911a0c | ||
|
|
c4a78a149b | ||
|
|
3814912adf | ||
|
|
75ef652cb3 | ||
|
|
1eae3fe5dc | ||
|
|
d0bc383a79 | ||
|
|
ea5d31b530 | ||
|
|
e56ad19e81 | ||
|
|
ed393eb75c | ||
|
|
2e2d309d38 | ||
|
|
6bfb33adb2 | ||
|
|
05138e1db1 | ||
|
|
b65402db41 | ||
|
|
fbef187400 | ||
|
|
25d1b8b9e5 | ||
|
|
ec91594083 | ||
|
|
e67f55bd74 | ||
|
|
276a7f9d2a | ||
|
|
4888d7a567 | ||
|
|
767f78315b | ||
|
|
4311b218fb | ||
|
|
89892042dd | ||
|
|
a22dfb3864 | ||
|
|
f40c42dcf0 | ||
|
|
1d31aa9da3 | ||
|
|
f999073fb4 | ||
|
|
5eb7c64ec3 | ||
|
|
7e79818555 | ||
|
|
a51576b250 | ||
|
|
0bee2689d1 | ||
|
|
c232743963 | ||
|
|
c2d5f092b3 | ||
|
|
8e8e186de9 | ||
|
|
55f47a2b1b | ||
|
|
5c27b9eb90 | ||
|
|
1d018533d1 | ||
|
|
34b8841f5c | ||
|
|
a54166a27c | ||
|
|
4adec39dce | ||
|
|
e63ae08638 | ||
|
|
f1b4c6a1ec | ||
|
|
78b4b3ffa8 | ||
|
|
f51fd48cdb | ||
|
|
9127a816e3 | ||
|
|
1a5cd9b062 | ||
|
|
a86b4c4e58 | ||
|
|
a384960a7b | ||
|
|
5817b80c94 | ||
|
|
97efd63c99 | ||
|
|
5ebcc01bcc | ||
|
|
ddcc008ea6 | ||
|
|
8ec10e2304 | ||
|
|
172f275bc7 | ||
|
|
0dac5943bb | ||
|
|
2dcd2b13be | ||
|
|
8743decd17 | ||
|
|
a15d7ca543 | ||
|
|
a90029ec50 | ||
|
|
563896b126 | ||
|
|
5aecd02668 | ||
|
|
eef6312a17 | ||
|
|
1e48f5c8f4 | ||
|
|
7908fa2814 | ||
|
|
7ffb3ea042 | ||
|
|
842dd90780 | ||
|
|
ee71a03f96 | ||
|
|
5221631b40 | ||
|
|
ade2b4ac0d | ||
|
|
643b95c0e4 | ||
|
|
ed0f9d4fb5 | ||
|
|
6f353c31cb | ||
|
|
404412a950 | ||
|
|
0e1778d8da | ||
|
|
dfc0ca9525 | ||
|
|
53bf1fe4fd | ||
|
|
5c05daa007 | ||
|
|
f74f075b4e | ||
|
|
1b53e8e98c | ||
|
|
4caee284bc | ||
|
|
5368c91399 | ||
|
|
00b1881f20 | ||
|
|
b727bd20c8 | ||
|
|
5a703c116f | ||
|
|
57f728f40e | ||
|
|
74bbde0c34 | ||
|
|
c58fc0781c | ||
|
|
7a266d49c1 | ||
|
|
344eeeb8c0 | ||
|
|
08d50ddfc5 | ||
|
|
f83c892b13 | ||
|
|
7dbec9d395 | ||
|
|
5171c801d9 | ||
|
|
72b4a778a6 | ||
|
|
0254df63f9 | ||
|
|
03c02ff74f | ||
|
|
9277c3c674 | ||
|
|
34b1259dc6 | ||
|
|
257e38de12 | ||
|
|
1984132eae | ||
|
|
8b1d75ea98 | ||
|
|
a79af11371 | ||
|
|
087a9b70b4 | ||
|
|
8b37952128 | ||
|
|
8336930091 | ||
|
|
d875325cee | ||
|
|
f1c5d1cb23 | ||
|
|
b55752e715 | ||
|
|
36a99e6ec8 | ||
|
|
df3c5b9a77 | ||
|
|
4c2ab23c1f | ||
|
|
0a3f81e364 | ||
|
|
69c5631b25 | ||
|
|
cca4331ac4 | ||
|
|
5972dbb6a7 | ||
|
|
794f2d8576 | ||
|
|
e35dce530a | ||
|
|
6034bd3cb2 | ||
|
|
42c77bbb67 | ||
|
|
d504c8f4a3 | ||
|
|
976c8b3ce0 | ||
|
|
905e7e41a6 | ||
|
|
16f48d0400 | ||
|
|
6d7608d851 | ||
|
|
7cd30efbe9 | ||
|
|
8c3652e1ac | ||
|
|
7b9e3aace9 | ||
|
|
568737e123 | ||
|
|
808e3ea6e3 | ||
|
|
e4f14d871f | ||
|
|
1cedd8ee5b | ||
|
|
b80fe24179 | ||
|
|
3cea41a854 | ||
|
|
62b80c13cc | ||
|
|
08b612575c | ||
|
|
b3b804bf2d | ||
|
|
09d8a86e30 | ||
|
|
cc5d917c72 | ||
|
|
7cc3e9d5ef | ||
|
|
87aa9fc6c4 | ||
|
|
8d5c315817 | ||
|
|
f89486f5fe | ||
|
|
cf4d78a24c | ||
|
|
ee88970075 | ||
|
|
f5fa40feb7 | ||
|
|
da13192cfe | ||
|
|
8ff65a65db | ||
|
|
2d882a11a5 | ||
|
|
79adc677a1 | ||
|
|
a8e2dad6d0 | ||
|
|
16954c5626 | ||
|
|
56305c4ede | ||
|
|
30713cc21d | ||
|
|
2bac0a25e0 | ||
|
|
c3aa5b0515 | ||
|
|
30afa43ed4 | ||
|
|
f2e66abe67 | ||
|
|
cdb311a501 | ||
|
|
4c2b3e157c | ||
|
|
52625b710b | ||
|
|
b411bb4e85 | ||
|
|
f16111369f | ||
|
|
628d052749 | ||
|
|
2e26bb0e57 | ||
|
|
652c7d72f6 | ||
|
|
0b046ea8cf | ||
|
|
7133f05fea | ||
|
|
da1c23d7dd | ||
|
|
a329dcd844 | ||
|
|
b57212d5f7 | ||
|
|
12c9e16f37 | ||
|
|
422381ac01 | ||
|
|
4b817cef65 | ||
|
|
4255a9916d | ||
|
|
b1566e904b | ||
|
|
8629076814 | ||
|
|
761e4f2718 | ||
|
|
3441dd4d2d | ||
|
|
9a8139a8b2 | ||
|
|
a0f823910c | ||
|
|
755338ded0 | ||
|
|
ecd85a53a1 | ||
|
|
8faa0d9e01 | ||
|
|
20ca2eac37 | ||
|
|
85b56247a3 | ||
|
|
febc66fb02 | ||
|
|
3f929aee3c | ||
|
|
8c46dd194d | ||
|
|
a9f99d05b9 | ||
|
|
e224c33817 | ||
|
|
4e6afbc8ff | ||
|
|
80b912ea14 | ||
|
|
85a571521b | ||
|
|
98efa9f466 | ||
|
|
a30a9ad763 | ||
|
|
36787eeeeb | ||
|
|
104cafedb0 | ||
|
|
284d307c0f | ||
|
|
f50c98e10d | ||
|
|
a4a5d63da7 | ||
|
|
9f0f9e0928 | ||
|
|
ef62fd7e34 | ||
|
|
0151dd8875 | ||
|
|
2026024c8a | ||
|
|
3d56baddd7 | ||
|
|
c10e7b2b41 | ||
|
|
ba605e985a | ||
|
|
4cd1cdf495 | ||
|
|
f9c644b964 | ||
|
|
da68bcf952 | ||
|
|
2c3b625b6b | ||
|
|
16ef259ae1 | ||
|
|
c56b3e75ea | ||
|
|
772866389f | ||
|
|
41f8b5aee2 | ||
|
|
75052fc773 | ||
|
|
e7a580fd3a | ||
|
|
c967230969 | ||
|
|
f3e6b2ab57 | ||
|
|
7385cb0c89 | ||
|
|
d1be9e7dc8 | ||
|
|
a22052e85b | ||
|
|
ab401541f2 | ||
|
|
dab4427c18 | ||
|
|
ad7d508a3c | ||
|
|
cb30514cbc | ||
|
|
53af9db285 | ||
|
|
b557a14134 | ||
|
|
8922fd6dde | ||
|
|
6b39e549f7 | ||
|
|
eb0dd827b1 | ||
|
|
a48902d7d3 | ||
|
|
7fca1f865d | ||
|
|
c5dc06962d | ||
|
|
45ecdafd52 | ||
|
|
f9e8bf92bd | ||
|
|
de614b293c | ||
|
|
8382075147 | ||
|
|
b605630a9c | ||
|
|
0a89fb1928 | ||
|
|
4c81906b50 | ||
|
|
b6220e985a | ||
|
|
7a97a28e00 | ||
|
|
dda8f281a1 | ||
|
|
28b561a810 | ||
|
|
44aba93d37 | ||
|
|
efb5157027 | ||
|
|
f48c1562f3 | ||
|
|
098b5b28a8 | ||
|
|
998e8e748c | ||
|
|
be96c77467 | ||
|
|
b5e8aec548 | ||
|
|
278c7dc836 | ||
|
|
1425ad87ab | ||
|
|
f985a2a8ad | ||
|
|
20f423a2fc | ||
|
|
210c1215d6 | ||
|
|
1c507393a9 | ||
|
|
b04cfdc091 | ||
|
|
42a3006f6e | ||
|
|
7d5c618246 | ||
|
|
74250982d1 | ||
|
|
8bd6fb5b4e | ||
|
|
bb2fb92b0e | ||
|
|
10b016312d | ||
|
|
b4500a2914 | ||
|
|
da9b1b3f58 | ||
|
|
da31520465 | ||
|
|
f3d3a0e57f | ||
|
|
9a655988dd | ||
|
|
43b541bb7d | ||
|
|
e32853744a | ||
|
|
b017b63745 | ||
|
|
29d6d824fc | ||
|
|
d6983a25b0 | ||
|
|
4cab6726b0 | ||
|
|
509b8f3a29 | ||
|
|
461c2f9055 | ||
|
|
c1d0f9cf1b | ||
|
|
019de08c64 | ||
|
|
30f88f99b7 | ||
|
|
04ee9894c9 | ||
|
|
46368f52f5 | ||
|
|
226e9084c9 | ||
|
|
caf5d2f788 | ||
|
|
591634f175 | ||
|
|
dfaf51a432 | ||
|
|
c2ec066c85 | ||
|
|
3e7c09bb44 | ||
|
|
28f66e30dd | ||
|
|
bac7f097a9 | ||
|
|
fecd29a967 | ||
|
|
c22eed9109 | ||
|
|
2c1ba623a4 | ||
|
|
561cdba7c9 | ||
|
|
e5e25012ff | ||
|
|
c5d653a51a | ||
|
|
c4620a0b0c | ||
|
|
dc969f80f3 | ||
|
|
a82f85f058 | ||
|
|
3ef85e711d | ||
|
|
29f7307f9d | ||
|
|
24eed71959 | ||
|
|
9341ccb643 | ||
|
|
4385b90cca | ||
|
|
9c10ee12fd | ||
|
|
7a622986d6 | ||
|
|
99b3cdd45d | ||
|
|
7b9d84f528 | ||
|
|
5f49827bd2 | ||
|
|
e1b4fe02d6 | ||
|
|
cdab127b98 | ||
|
|
bac7ea5dc0 | ||
|
|
b3b5581f2d | ||
|
|
9a26d869fe | ||
|
|
57138d9629 | ||
|
|
47d897b48d | ||
|
|
6b9f1d519e | ||
|
|
8b966c61d8 | ||
|
|
87e09efff5 | ||
|
|
65a0583c85 | ||
|
|
2000a1b334 | ||
|
|
cdd44af043 | ||
|
|
df99d06d11 | ||
|
|
b32d096be2 | ||
|
|
b9d5d1041c | ||
|
|
994dc9ff33 | ||
|
|
2ac97736ce | ||
|
|
925e106c43 | ||
|
|
59002413f8 | ||
|
|
32293ee3bb | ||
|
|
e04f35af4f | ||
|
|
c50f83400a | ||
|
|
31fa974379 | ||
|
|
aa8c362cb8 | ||
|
|
e48cc40610 | ||
|
|
a931f59dda | ||
|
|
4140fc4d28 | ||
|
|
f0d39e6081 | ||
|
|
c11872a229 | ||
|
|
f4fe730dc9 | ||
|
|
a4c83a08c7 | ||
|
|
c3277d24c3 | ||
|
|
28a23590ef | ||
|
|
162275992b | ||
|
|
08e25f4622 | ||
|
|
9a300d773f | ||
|
|
a5362f3170 | ||
|
|
7b653b90a8 | ||
|
|
f464b3ae7c | ||
|
|
d4b7064d2f | ||
|
|
2540b9969f | ||
|
|
4355b799cb | ||
|
|
a7fb434f9f | ||
|
|
b3465f3bbf | ||
|
|
4dfc3ff94c | ||
|
|
2670279b0d | ||
|
|
f4fd4245be | ||
|
|
200d239e0b | ||
|
|
41a0590182 | ||
|
|
63b2c49354 | ||
|
|
c99d7ce89d | ||
|
|
79e382e064 | ||
|
|
fa1d4dae0f | ||
|
|
35f84b4414 | ||
|
|
a882bdb08c | ||
|
|
b5c2e1c95b | ||
|
|
cfd6ea5a34 | ||
|
|
b3cf5e8698 | ||
|
|
2d07223995 | ||
|
|
32bb9fd4ec | ||
|
|
37b5f22c1a | ||
|
|
e0b00c09ce | ||
|
|
71c4dba34d | ||
|
|
9a3187e36c | ||
|
|
c714a3f91c | ||
|
|
5d4846cf60 | ||
|
|
543b327d09 | ||
|
|
1ccd2a3cc9 | ||
|
|
94398071eb | ||
|
|
143476c76c | ||
|
|
e85683e40f | ||
|
|
141162e5ae | ||
|
|
13acbfb64f | ||
|
|
a55e3d8578 | ||
|
|
2486ae6d3f | ||
|
|
ba991b47ea | ||
|
|
a096d0db37 | ||
|
|
e4a30c052e | ||
|
|
ccfe80ecfa | ||
|
|
df5b889f7d | ||
|
|
55f25c2613 | ||
|
|
34eaa822a0 | ||
|
|
5072f503c4 | ||
|
|
8e6b5a5e55 | ||
|
|
246647641a | ||
|
|
06a1bdbcba | ||
|
|
a0b75d7ada | ||
|
|
d36036a9fe | ||
|
|
a68d891770 | ||
|
|
f272a9ead2 | ||
|
|
8db2e10696 | ||
|
|
bea4f7da98 | ||
|
|
3cd74c06d6 | ||
|
|
053681e4a5 | ||
|
|
e3225f7123 | ||
|
|
ef62b0e746 | ||
|
|
7756d12993 | ||
|
|
3b92d12a44 | ||
|
|
4449bf3c75 | ||
|
|
0200ed1341 | ||
|
|
02bde900d6 | ||
|
|
36e299b4cf | ||
|
|
4e34f861c9 | ||
|
|
42df2fe242 | ||
|
|
7f11da8650 | ||
|
|
dff0fcb1fd | ||
|
|
c31d22f996 | ||
|
|
14ab2b52e9 | ||
|
|
e91f34f09b | ||
|
|
6c733f84e9 | ||
|
|
71c2cd5065 | ||
|
|
45f9a10bfb | ||
|
|
6ad203f767 | ||
|
|
5bf895eefe | ||
|
|
5a8f5b14c1 | ||
|
|
27ccb37646 | ||
|
|
b4c528f6b5 | ||
|
|
517a255d5a | ||
|
|
d76feeb320 | ||
|
|
e4f23b3fce | ||
|
|
02d8dc22e9 | ||
|
|
2fc3ec7224 | ||
|
|
a243ac7a01 | ||
|
|
6f1ee78623 | ||
|
|
6633370a34 | ||
|
|
a22e29532c | ||
|
|
96c8d1f68b | ||
|
|
4aa3bd56c4 | ||
|
|
7d1cf7975a | ||
|
|
59d08a92d0 | ||
|
|
33c1a002da | ||
|
|
8f04d3418a | ||
|
|
f5b556c6cb | ||
|
|
a18f3bf08a | ||
|
|
41a0a0c64c | ||
|
|
b4c8a317d0 | ||
|
|
30ac075555 | ||
|
|
fcb317ca40 | ||
|
|
9991f79326 | ||
|
|
7cf3f8c7cd | ||
|
|
eeef183cc6 | ||
|
|
96bc02c8cb | ||
|
|
af276a24ed | ||
|
|
a92dbae459 | ||
|
|
b0bfb5e443 | ||
|
|
671e6da104 | ||
|
|
485be533c9 | ||
|
|
53ff4cd651 | ||
|
|
a98c760d93 | ||
|
|
9b0c95e37d | ||
|
|
b419b8f642 | ||
|
|
c32f8d0998 | ||
|
|
c6d253cc25 | ||
|
|
c05cb62cdb | ||
|
|
ddd213dda3 | ||
|
|
8aab844a48 | ||
|
|
8fc1cbfc54 | ||
|
|
82ea330dda | ||
|
|
c63aa3033a | ||
|
|
7837738c17 | ||
|
|
bdb0ad4836 | ||
|
|
0dbf810cfe | ||
|
|
58aa4887ad | ||
|
|
3ff0916f49 | ||
|
|
cc9fb3ff41 | ||
|
|
2c51928b9d | ||
|
|
4451978af6 | ||
|
|
d4193c55ae | ||
|
|
ec3e365b3c | ||
|
|
c245c69389 | ||
|
|
9a9a7c9d7a | ||
|
|
20b516b5fa | ||
|
|
4dcfeca092 | ||
|
|
8ca8c73868 | ||
|
|
1a1c101b9a | ||
|
|
a0a0e2d324 | ||
|
|
d544199eff | ||
|
|
407ebeb699 | ||
|
|
9ba12b3e3f | ||
|
|
97976cbee9 | ||
|
|
cb5c772233 | ||
|
|
2bfbb824bf | ||
|
|
05189651ee | ||
|
|
0c80134523 | ||
|
|
6c686520c4 | ||
|
|
49ae029572 | ||
|
|
10e43203ed | ||
|
|
e25be5a522 | ||
|
|
47ff64a9e9 | ||
|
|
35f3887d79 | ||
|
|
dacfc8b00a | ||
|
|
0840ac7d06 | ||
|
|
f34b96f565 | ||
|
|
808fbed4f3 | ||
|
|
b48800049b | ||
|
|
7e5d4166d8 | ||
|
|
6b724ba8db | ||
|
|
25e1427a84 | ||
|
|
82944bc103 | ||
|
|
1e73b2437d | ||
|
|
574cd304e3 | ||
|
|
2edcd1f5b2 | ||
|
|
e144b26619 | ||
|
|
9ef9220689 | ||
|
|
23b6a8c495 | ||
|
|
4b9552fdf5 | ||
|
|
a249097170 | ||
|
|
6f4b2d7967 | ||
|
|
888d77c96c | ||
|
|
e2ae1ca866 | ||
|
|
7ef9a0f0bf | ||
|
|
885a0882f2 | ||
|
|
2ddeca2976 | ||
|
|
2085012874 | ||
|
|
ebb495ab8b | ||
|
|
905274d6d4 | ||
|
|
120d7edf12 | ||
|
|
12cd06cae5 | ||
|
|
42c26102d8 | ||
|
|
e906564f70 | ||
|
|
b8b2dc3b6d | ||
|
|
d69f76e68a | ||
|
|
ec1ba02cca | ||
|
|
49869d071a | ||
|
|
39275682d9 | ||
|
|
1dbc3a049b | ||
|
|
0a7b2683ff | ||
|
|
9714985855 | ||
|
|
d809b8eee0 | ||
|
|
52e2ed2aa7 | ||
|
|
ab3cc3d2b6 | ||
|
|
0a33324321 | ||
|
|
a948335cb9 | ||
|
|
6d2e5af6bc | ||
|
|
9a7c71394e | ||
|
|
1ebf184e9f | ||
|
|
48b013fa85 | ||
|
|
e96dee478d | ||
|
|
25d2a12a92 | ||
|
|
b0af31a71c | ||
|
|
9fdd766429 | ||
|
|
28ce23350d | ||
|
|
1abe890067 | ||
|
|
75fbedeeac | ||
|
|
a1861a2f84 | ||
|
|
0eecfb52ed | ||
|
|
ce1c28ffb9 | ||
|
|
9667b81c8d | ||
|
|
afac596a7b | ||
|
|
636692ba6a | ||
|
|
203f7e7ea6 | ||
|
|
8aee19b0ef | ||
|
|
49ed31c895 | ||
|
|
526075cd5e | ||
|
|
71340805ec | ||
|
|
654585aa68 | ||
|
|
441d3c1220 | ||
|
|
fc2d7258fb | ||
|
|
8195cedcf4 | ||
|
|
3e98477707 | ||
|
|
7b9479eca9 | ||
|
|
f15b67d386 | ||
|
|
f3141727d7 | ||
|
|
c7e4638736 | ||
|
|
d04981969f | ||
|
|
a148d4f446 | ||
|
|
a9e105f9eb | ||
|
|
c8afce2a3a | ||
|
|
ebf9d11f4d | ||
|
|
7a3bdf93e9 | ||
|
|
35d5c585ec | ||
|
|
45eb4f7201 | ||
|
|
e4c2585e23 | ||
|
|
a05f2fe3fb | ||
|
|
94cdd6829c | ||
|
|
34aa51e98a | ||
|
|
6e93c68a77 | ||
|
|
f16eec2ad2 | ||
|
|
657c1a4626 | ||
|
|
6bf6e943cc | ||
|
|
6252f903ee | ||
|
|
a1e827fb3b | ||
|
|
40218e02e1 | ||
|
|
0891b0d294 | ||
|
|
756c43c087 | ||
|
|
6488e66088 | ||
|
|
31ee3fb12b | ||
|
|
b80d45ad45 | ||
|
|
ea2a3c3b70 | ||
|
|
c0312159eb | ||
|
|
e124f582aa | ||
|
|
54150a8f2a | ||
|
|
385c221ef8 | ||
|
|
7df6e99e28 | ||
|
|
76a5f2f516 | ||
|
|
936c1f2ada | ||
|
|
5787802077 | ||
|
|
c902d25524 | ||
|
|
d152336f42 | ||
|
|
7ab0ae18be | ||
|
|
6189eddecf | ||
|
|
cc7471a9af | ||
|
|
eabaca361c | ||
|
|
141ca2498c | ||
|
|
964db2a779 | ||
|
|
f2867542b1 | ||
|
|
8f6e886821 | ||
|
|
dfca7bc2c4 | ||
|
|
f94b04bb19 | ||
|
|
be1e97c616 | ||
|
|
9790e9f843 | ||
|
|
e339d3857f | ||
|
|
0e4f21842c | ||
|
|
d4ea9a1ba3 | ||
|
|
5300fa929b | ||
|
|
8a5a66a06a | ||
|
|
43ed4427b4 | ||
|
|
22b0d92e6b | ||
|
|
7eb528232e | ||
|
|
33b6b85e72 | ||
|
|
8177119671 | ||
|
|
4b407a9dd2 | ||
|
|
9b904a7b4a | ||
|
|
e0424a1e38 | ||
|
|
00c08e05f5 | ||
|
|
0836d9c5ee | ||
|
|
ff5b6c4eab | ||
|
|
bb5cd49327 | ||
|
|
6f40ef6015 | ||
|
|
c7758506ac | ||
|
|
192ac6959a | ||
|
|
465a4643a8 | ||
|
|
1bf0065d9d | ||
|
|
c14efc843c | ||
|
|
0506dfba10 | ||
|
|
7e4f36e8b4 | ||
|
|
55dc482a61 | ||
|
|
05b9f9cae6 | ||
|
|
c79868de3e | ||
|
|
7eeac060ab | ||
|
|
c8b2b7b279 | ||
|
|
f4b3187494 |
No files matched your search
@@ -0,0 +1,65 @@
|
||||
version: 1.0.{build}
|
||||
|
||||
environment:
|
||||
key_pass:
|
||||
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
|
||||
pfx_key:
|
||||
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
|
||||
|
||||
|
||||
notifications:
|
||||
- provider: Slack
|
||||
incoming_webhook:
|
||||
secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
|
||||
|
||||
|
||||
clone_folder: c:\projects\lbry
|
||||
|
||||
|
||||
|
||||
install:
|
||||
- cmd: dir C:\projects\lbry\packaging\windows\
|
||||
- ps: .\packaging\windows\init.ps1
|
||||
- nuget install secure-file -ExcludeVersion
|
||||
- secure-file\tools\secure-file -decrypt packaging\windows\certs\lbry2.pfx.enc -secret %pfx_key%
|
||||
- ps: .\packaging\windows\install.ps1
|
||||
|
||||
|
||||
build_script:
|
||||
- ps: .\packaging\windows\build.ps1
|
||||
|
||||
|
||||
test_script:
|
||||
- ps: .\packaging\windows\test.ps1
|
||||
|
||||
|
||||
artifacts:
|
||||
- path: dist/*.msi
|
||||
name: msi
|
||||
- path: build/exe.win32-2.7/
|
||||
name: lbry-portable
|
||||
- path: packaging/windows/lbry-win32-app/LBRY-URI.reg
|
||||
name: LBRY-URI
|
||||
|
||||
|
||||
deploy:
|
||||
- provider: GitHub
|
||||
release: $(APPVEYOR_REPO_TAG_NAME)
|
||||
description: 'Release'
|
||||
auth_token:
|
||||
secure: 28gMVxQkXr2iXP4F+5kVwefUtKCfS1ePZ97PVfaSR8UDupjAvKhSJx862TnEjukb
|
||||
artifact: /.*\.msi/
|
||||
draft: false
|
||||
prerelease: true
|
||||
on:
|
||||
appveyor_repo_tag: true # deploy on tag push only
|
||||
- provider: S3
|
||||
access_key_id:
|
||||
secure: E25iHvmHiJP56GWFTe14L3pS8QKb5ZcyWZLY0zqh6BA=
|
||||
secret_access_key:
|
||||
secure: YwZe6fCv29akFYjMCCBJBF7LtI6mxPrxbq7SSoAbn1BPdqjATFegeteGAsqHur/C
|
||||
bucket: lbrynet-master
|
||||
region: us-west-2
|
||||
artifact: /.*\.msi/
|
||||
on:
|
||||
branch: master
|
||||
@@ -0,0 +1,19 @@
|
||||
[bumpversion]
|
||||
current_version = 0.7.9
|
||||
commit = True
|
||||
tag = True
|
||||
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+)(?P<candidate>\d+))?
|
||||
serialize =
|
||||
{major}.{minor}.{patch}{release}{candidate}
|
||||
{major}.{minor}.{patch}
|
||||
|
||||
[bumpversion:file:lbrynet/__init__.py]
|
||||
|
||||
[bumpversion:file:packaging/ubuntu/lbry.desktop]
|
||||
|
||||
[bumpversion:part:release]
|
||||
optional_value = production
|
||||
values =
|
||||
rc
|
||||
production
|
||||
|
||||
+31
@@ -1 +1,32 @@
|
||||
*.pyc
|
||||
|
||||
*.xml
|
||||
|
||||
.idea/.name
|
||||
|
||||
*.iml
|
||||
|
||||
*.log
|
||||
|
||||
*.txt
|
||||
|
||||
lbrynet.egg-info/PKG-INFO
|
||||
|
||||
*.egg
|
||||
|
||||
/build
|
||||
|
||||
/dist
|
||||
|
||||
*.so
|
||||
|
||||
*.pem
|
||||
|
||||
*.decTest
|
||||
|
||||
.coverage
|
||||
|
||||
# temporary files from the twisted.trial test runner
|
||||
_trial_temp/
|
||||
|
||||
.DS_Store
|
||||
Whitespace-only changes.
@@ -0,0 +1,380 @@
|
||||
[MASTER]
|
||||
|
||||
# Specify a configuration file.
|
||||
#rcfile=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=yes
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=1
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
# be used to obtain the result of joining multiple strings with the addition
|
||||
# operator. Joining a lot of strings can lead to a maximum recursion error in
|
||||
# Pylint and this flag can prevent that. It has one side effect, the resulting
|
||||
# AST will be different than the one from reality.
|
||||
optimize-ast=no
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||
confidence=
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
# --disable=W"
|
||||
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html. You can also give a reporter class, eg
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
output-format=text
|
||||
|
||||
# Put messages in a separate file for each module / package specified on the
|
||||
# command line instead of printing them on stdout. Reports (if any) will be
|
||||
# written in a file name "pylint_global.[txt|html]".
|
||||
files-output=no
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=yes
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details
|
||||
#msg-template=
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||
# not used).
|
||||
dummy-variables-rgx=_$|dummy
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,_cb
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format
|
||||
logging-modules=logging
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# List of builtins function names that should not be used, separated by a comma
|
||||
bad-functions=map,filter,input
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
good-names=i,j,k,ex,Run,_
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=foo,bar,baz,toto,tutu,tata
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name
|
||||
include-naming-hint=no
|
||||
|
||||
# Regular expression matching correct function names
|
||||
function-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for function names
|
||||
function-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct variable names
|
||||
variable-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for variable names
|
||||
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct constant names
|
||||
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Naming hint for constant names
|
||||
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Regular expression matching correct attribute names
|
||||
attr-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for attribute names
|
||||
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct argument names
|
||||
argument-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for argument names
|
||||
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct class attribute names
|
||||
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Naming hint for class attribute names
|
||||
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Regular expression matching correct inline iteration names
|
||||
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Naming hint for inline iteration names
|
||||
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class names
|
||||
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Naming hint for class names
|
||||
class-name-hint=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Regular expression matching correct module names
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Naming hint for module names
|
||||
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Regular expression matching correct method names
|
||||
method-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for method names
|
||||
method-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=^_
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=-1
|
||||
|
||||
|
||||
[ELIF]
|
||||
|
||||
# Maximum number of nested blocks for function / method body
|
||||
max-nested-blocks=5
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||
# install python-enchant package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to indicated private dictionary in
|
||||
# --spelling-private-dict-file option instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=100
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=no
|
||||
|
||||
# List of optional constructs for which whitespace checking is disabled. `dict-
|
||||
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
|
||||
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
|
||||
# `empty-line` allows space-only lines.
|
||||
no-space-check=trailing-comma,dict-separator
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=1000
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,XXX,TODO
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Ignore imports when computing similarities.
|
||||
ignore-imports=no
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||
ignore-mixin-members=yes
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet.reactor,RequestMessage
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,TERMIOS,Bastion,rexec
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
ignored-argument-names=_.*
|
||||
|
||||
# Maximum number of locals for function / method body
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of return / yield for function / method body
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of branch for function / method body
|
||||
max-branches=12
|
||||
|
||||
# Maximum number of statements in function / method body
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
# Maximum number of boolean expressions in a if statement
|
||||
max-bool-expr=5
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,__new__,setUp
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,_fields,_replace,_source,_make
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when being caught. Defaults to
|
||||
# "Exception"
|
||||
overgeneral-exceptions=Exception
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
sudo: required
|
||||
dist: trusty
|
||||
# dh-virtualenv requires that we use the same python interpreter
|
||||
# that comes with the system, so we don't want to use anything that
|
||||
# travis would try to set-up for us in python
|
||||
language: generic
|
||||
- os: osx
|
||||
# Use generic language for osx
|
||||
# python 2.7 is broken on osx on travis, so follow we have to specify the installation ourselves
|
||||
# https://github.com/travis-ci/travis-ci/issues/2312#issuecomment-195620855
|
||||
language: generic
|
||||
osx_image: xcode7.3
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi
|
||||
- mkdir -p lbrynet/resources/ui
|
||||
- ./packaging/travis/setup_build.sh
|
||||
|
||||
install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./packaging/travis/install_dependencies_and_run_tests.sh; fi
|
||||
|
||||
before_script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.cer.enc -d -a -out packaging/osx/certs/dist.cer; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.p12.enc -d -a -out packaging/osx/certs/dist.p12; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/osx/add-key.sh; fi
|
||||
|
||||
script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash packaging/ubuntu/ubuntu_package_setup.sh -t; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then cd packaging/osx/lbry-osx-app && ./setup_app.sh && cd $TRAVIS_BUILD_DIR; fi
|
||||
# fail the build if this is a build for a tag and we don't have the versions matching; allow tags that start with 'test' to pass
|
||||
- if [[ -n "${TRAVIS_TAG}" ]]; then if [[ "${TRAVIS_TAG}" == test* ]] || [[ "v`python setup.py -V`" = "${TRAVIS_TAG}" ]]; then true; else false; fi; fi
|
||||
|
||||
before_deploy:
|
||||
# s3 release can only upload a folder so move the package into an upload folder
|
||||
- mkdir "${TRAVIS_BUILD_DIR}/upload"
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then mv "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg" "${TRAVIS_BUILD_DIR}/upload"; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mv "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb" "${TRAVIS_BUILD_DIR}/upload"; fi
|
||||
|
||||
deploy:
|
||||
- provider: releases
|
||||
file: "${TRAVIS_BUILD_DIR}/upload/`python setup.py --name`_`python setup.py -V`_amd64.deb"
|
||||
skip_cleanup: true
|
||||
prerelease: true
|
||||
on:
|
||||
tags: true
|
||||
condition: "$TRAVIS_OS_NAME = linux"
|
||||
# this is the oauth token for the lbry-ci user
|
||||
api_key:
|
||||
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
|
||||
- provider: releases
|
||||
file: "${TRAVIS_BUILD_DIR}/upload/`python setup.py --name`.`python setup.py -V`.dmg"
|
||||
skip_cleanup: true
|
||||
prerelease: true
|
||||
on:
|
||||
tags: true
|
||||
condition: "$TRAVIS_OS_NAME = osx"
|
||||
# this is the oauth token for the lbry-ci user
|
||||
api_key:
|
||||
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
|
||||
- provider: s3
|
||||
access_key_id:
|
||||
secure: "gmJNW8bda2snpA2F+0gucjgO/orvZL0a348QmiffYdtXleIseyY+C4ZI9llWm+s8n2TxiqBYpc/A3Bv7JM0yIccjF0CWQb8pu4HkVT2xLT/1p28EdfxKhR0H5sLtkxOCvLIuf3ppZWac8IzRWvh9TDER1TVxbIGvbOJCtKJ+sUKGCWVkxJY6lzAy7+YHwY9sLV6GvarbkfcnbwNh0qPEROcc6JXUQRKCjWOoZLSZHx4dwxlDiXaG1GyUSm2bwtB11VbMZqaP5eO2zypcUr24J0WkzuAOFIUMDERQnY6eSR62T88BjVjL+07kQufSTtnHC49uHsylyQLwkphkwi2Ei3c/fJ5XI1gY0Med/WOHuy99LAaaHQvHxVa9Zxvrgivvu1oa2QdLir05asMCB0G6Yjgz38Fl8jzdKN/PLHs/lhgBGOJfMN+0UR7hOrLeXdJcMriVfwzYrCl6KkFgTe50rcMx8vv6SH15AqZ5sceksy+maA4O8Mxq1hEe2qJABgRdRZK3FnFKNnQOxfrlU6N9zMNsicVcRwH6WNBtoL4BK3KeMproWnorh9rXmdoSG2Fh5X636cudYhLrAbO5yaoZAXFxHqOV1n63v3tgv4MKrB999OF9V9HyYXv3ro9WZTTsH7LskLA7YmTvXtpMVgBK8SfnvqCjLim5qpVvIVPQIOY="
|
||||
secret_access_key:
|
||||
secure: "pS28jMSNV8HUSa8P8pReXgBtHW0OSWDZMAlZhBrtUI05goUnK6buXadvzCynZWgDiXPHuEuLZOLDVj5yL8N3eCa+sG1tEDcCjPxrBFxHisng8atgn3FaL15MBOdXn0qOBUr+vQ4jbUozsBYGqmoZQV0/Af53tU55KK5HJrotkd++SNTEQrFbWZavM+Wvhw4yJN2hmRer9II72DCZkayUjpvBq+irnGeU0g9SrX8VULCg+sqcnbqdLWd+VALHzQr+O74MaWE8mu2PA8A1GKWZZ/a+LoTh1T2bArpX94q0ueea1eEcUA154vC1azkKK5vy9hzENt5qiiEleBDE31DhbGaeEoakEXvHWrdI3DYiakD6prj0necD8aAK6VhPtqwiTtTf7E4BE9hxY4nJRDuZckvFI4LYFMvHlbdqBieeZPflStkws7VgsfrGV7EberYIbSEohUGGUBg3rdaCIPsn8W4nQAixq39Dcfm0orGo4BgargT6FFtAeDzsfYdbKBk/F8BOPMDJz9P1utAZ/ZeBexic16aeIbk7+wtaQUd0Hswm+2QUHddshvp9IzNniFb1BLvRR/lOVl0HdUkJj/haR2M8XhwcRFIKpL2oMZHwRmLW4c1kkEx+KSfAB77arjgrCzq3y5MlKUSliL5qZZMIJX0ReRQ2dcy3DCY0Fxdj014="
|
||||
bucket: lbrynet-master
|
||||
skip_cleanup: true
|
||||
local_dir: "${TRAVIS_BUILD_DIR}/upload"
|
||||
region: us-west-2
|
||||
on:
|
||||
branch: master
|
||||
|
||||
|
||||
env:
|
||||
global:
|
||||
# needed to unlock the identity file when its on the keychain
|
||||
- secure: "aqgVNHfeh6JUqTTWG5+W+tTsIJkePS6HyLkcZlq6ODYrfdGKa90EeV4q4wdGfP5IkrBbf+WBFACDGVp574E7vfMLMNKUDuUtgnGVPwqOIhjG82ud8Xa7qF7lsw50QOnRYYVd2GLlCIzk8sffT5ncjSPN2ClNVM5iTwCkC8TNSAVnEJxu6bG6hcaT4uCWWjs0m+39O1xJwxJ4vTjpE/gy+j2FTSaUR5cNavOVCyJqpeKlga9aoBVsQHWvxlYurdWbwRLwVIV7bDE+sYPnwn1nMFQpx5RZ1AX1Z2UxNFKcYzLJgcWe85OjxLyT4udX+XZ9SLsdOjm1n201OLEKsmTxmHS8yqpbu6+pKQ1rLMEVpgPGfS8DdtqZyb3z0u6q4jztpm+uBe8hnFgwQFGXO63nOQsI0n1PMR3evAnVx7jYt7y/UALs2A9yjosMwDqgql1FMhyd0OJGo8Ky8YpDsr2J5zGVvIqt2/N+lP+SOe2D1J/5EhGlY0o4tqIAFskh3q8/GR6UGm70KT2l4LJqrU1WGrGxPJ+HoOEmvG6eqLesk03fAX3v5+DgXZWErnzXMIOGDPaFVpmx+G9VZTIlmf+3Wbu9TnZE5PRwFTdP1rqjGpjUhHeF0VTc2qgNq1OfL98CBO1wLgA195+em58cELalnwDMWUTmY1Jt1kUuAtCc11U="
|
||||
# used to decypt the aes decrypt the certificate and identity files
|
||||
- secure: "FzftiEBFMngQIci5KZ+tpKs/BubalhWXJ9f8yogGNVa3XplLheUXYgcsNU8sYT2MJaKAHN9fSjHWb67UtKT7yXddXxeOPnW6wJ3ua/FXnpynsmeGKTfh3stvgjpzdXous67uHmCWMMklfb6z7UuDohjUMAe5n5HKw5tq1RzTKpc2kJacOC6qUT5laUOvULyCaO9E9HmbHeR5ZeXAC6pnzX2ccsSrcXvPozHzBIZ9RyothKs+CZw8PEuJo07RRL8meboegqYOUrOYuz6A2gS2mZJoy59ivZKOFxS5shEuv2Jt80RyfyxBoUpKFq8OG3Am5nAEzDiTIGzmIoKDEGKeTagk/sEtSZXiMDkzDT4GX/j1rUNLCBU87bXEFS2zfRsrfg8c1XZPIzDYBT1PY2QtLBdddF5zzDoKdPLJ3sjN+fZFE5RlnwfwnMHriVRZZlzjcdk0Z06gKTBCxUg5BZamnOOK+K8qunMJXVS+Vmi5u4RoTZZiCosUlYKnSKJ8suO9C0+znxoViusPqP4ONprNHgDoZw+UKio84QW3PrZv9h4zH/D+msDgJRZ0ceqDD+6Wz1J8Mm5ptW1GOLh/IU12TPXjxteqh0Um2vv5eIPmjK9uEK666kK7PqtPDkYhAfWvF+nmOOyPMJfbP4MW/i9WHNF4ghsIMbPKfqNhgSmfrYw="
|
||||
@@ -0,0 +1,85 @@
|
||||
#### Getting LBRY for development
|
||||
|
||||
Q: How do I get lbry for command line?
|
||||
|
||||
A: In order to run lbry from command line, you need more than the packaged app/deb.
|
||||
|
||||
######On OS X
|
||||
|
||||
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app.
|
||||
|
||||
######On Linux
|
||||
|
||||
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
sudo python setup.py install
|
||||
|
||||
#### Using LBRY
|
||||
|
||||
Q: How do I run lbry from command line?
|
||||
|
||||
A: The command is `lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I stop lbry from the command line?
|
||||
|
||||
A: You can ctrl-c or run `stop-lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I run lbry with lbrycrdd (the blockchain node application)?
|
||||
|
||||
A: Start lbry with the --wallet flag set: `lbrynet-daemon --wallet=lbrycrd`
|
||||
|
||||
Note: when you change the wallet it is persistant until you specify you want to use another wallet - lbryum - with the --wallet flag again.
|
||||
|
||||
***********
|
||||
|
||||
Q: Where are all the behind the scenes files?
|
||||
|
||||
A: On linux, the relevant directories are `~/.lbrynet`, `~/.lbrycrd`, and `~/.lbryum`, depending on which wallets you've used. On OS X, the folders of interest are `~/Library/Application Support/LBRY`, `~/.lbrycrd` and `~/.lbryum`, also depending on which wallets you've used.
|
||||
|
||||
***********
|
||||
|
||||
Q: How can I see the log in the console?
|
||||
|
||||
A: Run lbry with the --log-to-console flag set: `lbrynet-daemon --log-to-console`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I specify a web-UI to use?
|
||||
|
||||
A: If the files for the UI you'd like to use are storred locally on your computer, start lbry with the --ui flag: `lbrynet-daemon --ui=/full/path/to/ui/files/root/folder`
|
||||
|
||||
Note, once set with the UI flag the given UI will be cached by lbry and used as the default going forward. Also, it will only successfully load a UI if it contains a conforming requirements.txt file to specify required lbrynet and lbryum versions. [Here](https://github.com/lbryio/lbry-web-ui/blob/master/dist/requirements.txt) is an example requirements.txt file.
|
||||
|
||||
To reset your ui to pull from lbryio, or to try a UI still in development, run lbry with the --branch flag: `lbrynet=daemon --branch=master`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I see the list of API functions I can call, and how do I call them?
|
||||
|
||||
A: Here is an example script to get the documentation for the various API calls. To use any of the functions displayed, just provide any specified arguments in a dictionary.
|
||||
|
||||
Note: the lbry api can only be used while either the app or lbrynet-daemon command line are running
|
||||
|
||||
import sys
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
sys.exit(0)
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
if not api.is_running():
|
||||
print api.daemon_status()
|
||||
else:
|
||||
for func in api.help():
|
||||
print "%s:\n%s" % (func, api.help({'function': func}))
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
To use the LBRYWallet, which enables spending and accepting LBRYcrds in exchange for data, the
|
||||
LBRYcrd application (insert link to LBRYcrd website here) must be installed and running. If
|
||||
this is not desired, the testing client can be used to simulate trading points, which is
|
||||
built into LBRYnet.
|
||||
|
||||
on Ubuntu:
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
|
||||
Getting the source
|
||||
------------------
|
||||
|
||||
Don't you already have it?
|
||||
|
||||
Setting up the environment
|
||||
--------------------------
|
||||
|
||||
It's recommended that you use a virtualenv
|
||||
|
||||
sudo apt-get install python-virtualenv
|
||||
cd <source base directory>
|
||||
virtualenv .
|
||||
source bin/activate
|
||||
|
||||
(to deactivate the virtualenv, enter 'deactivate')
|
||||
|
||||
python setup.py install
|
||||
|
||||
this will install all of the libraries and a few applications
|
||||
|
||||
For running the file sharing application, see RUNNING
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
#### Installing the LBRY app
|
||||
--------------------------
|
||||
|
||||
Installing LBRY is simple. You can get a dmg installer for OS X or a .deb for linux [here](https://github.com/lbryio/lbry/releases/latest).
|
||||
|
||||
##### OS X
|
||||
Drag and drop LBRY.app into your applications folder (replacing any older versions). When it's running you'll have a LBRY icon in your status bar and the browser will open to the UI.
|
||||
|
||||
##### Linux
|
||||
Double click the .deb file and follow the prompts. The app can be started by searching "LBRY", and it can be turned off by clicking the red 'x' in the browser interface.
|
||||
|
||||
On both systems you can also open the UI while the app is running by going to lbry://lbry in Firefox or Safari, or localhost:5279 in Chrome.
|
||||
|
||||
|
||||
#### Installing LBRY from source
|
||||
--------------------------
|
||||
|
||||
Installing LBRY from source is not the recommended way to install
|
||||
LBRY. Only do so if you're interested in doing development work.
|
||||
|
||||
##### OS X Prerequisites
|
||||
|
||||
In order to install LBRY from source you'll need to first install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/). The dependencies can then be installed by running
|
||||
|
||||
```
|
||||
brew install mpfr libmpc
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
```
|
||||
|
||||
##### Linux Prerequisites
|
||||
On Ubuntu or Mint you can install the prerequisites by running
|
||||
|
||||
```
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev \
|
||||
python-pip git python-virtualenv libssl-dev libffi-dev
|
||||
```
|
||||
|
||||
##### OSX and Linux Installation
|
||||
|
||||
It is strongly recommended to create a new virtualenv for LBRY
|
||||
|
||||
```
|
||||
virtualenv lbry-venv
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
and install the package in the new virtualenv:
|
||||
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
pip install -r requirements.txt
|
||||
pip install .
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
|
||||
|
||||
##### On windows
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- Yapsy
|
||||
- cx_Freeze
|
||||
- requests
|
||||
- gmpy
|
||||
|
||||
Install the each of the preceding binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Install the rest of the required modules with the standard `pip install module` command
|
||||
|
||||
The one module which requires additional work is [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9).
|
||||
Download the source and compile with MinGW by running `mingw32make.bat`
|
||||
Then install the module by running `python setupmingw32.py install`
|
||||
@@ -1,22 +1,15 @@
|
||||
Copyright (c) 2015, LBRY, Inc.
|
||||
All rights reserved.
|
||||
The MIT License (MIT)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
Copyright (c) 2015-2016 LBRY Inc
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -1,57 +0,0 @@
|
||||
LBRYnet
|
||||
=======
|
||||
|
||||
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
|
||||
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
|
||||
table, used by peers to discover other peers.
|
||||
|
||||
Overview
|
||||
--------
|
||||
|
||||
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
|
||||
guarantees that peers can verify the correctness of each chunk without having to know anything
|
||||
about its contents, and can confidently re-transmit the chunk to other peers. Peers wishing to
|
||||
transmit chunks to other peers announce to the distributed hash table that they are associated
|
||||
with the sha384 hash sum in question. When a peer wants to download that chunk from the network,
|
||||
it asks the distributed hash table which peers are associated with that sha384 hash sum. The
|
||||
distributed hash table can also be used more generally. It simply stores IP addresses and
|
||||
ports which are associated with 384-bit numbers, and can be used by any type of application to
|
||||
help peers find each other. For example, an application for which clients don't know all of the
|
||||
necessary chunks may use some identifier, chosen by the application, to find clients which do
|
||||
know all of the necessary chunks.
|
||||
|
||||
Running
|
||||
-------
|
||||
|
||||
LBRYnet comes with an file sharing application, called 'lbrynet-console', which breaks
|
||||
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
|
||||
a special file called a 'stream descriptor' containing the hash sums and some other file metadata,
|
||||
and makes the chunks available for download by other peers. A peer wishing to download the file
|
||||
must first obtain the 'stream descriptor' and then may open it with his 'lbrynet-console' client,
|
||||
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
|
||||
chunks into the original file, according to the metadata included in the 'stream descriptor'.
|
||||
|
||||
To install and use this client, see INSTALL and RUNNING
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
See INSTALL
|
||||
|
||||
Developers
|
||||
----------
|
||||
|
||||
Documentation: doc.lbry.io
|
||||
Source code: trac.lbry.io/browser
|
||||
|
||||
To contribute to the development of LBRYnet or lbrynet-console, contact jimmy@lbry.io
|
||||
|
||||
Support
|
||||
-------
|
||||
|
||||
Send all support requests to jimmy@lbry.io
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
See LICENSE
|
||||
@@ -0,0 +1,71 @@
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
[](https://coveralls.io/github/lbryio/lbry)
|
||||
|
||||
# LBRYnet
|
||||
|
||||
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
|
||||
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
|
||||
table, used by peers to discover other peers.
|
||||
|
||||
## Installation
|
||||
|
||||
Download the [latest release](https://github.com/lbryio/lbry/releases/latest) or see [INSTALL.md](INSTALL.md) for manual installation.
|
||||
|
||||
## Overview
|
||||
|
||||
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
|
||||
guarantees that peers can verify the correctness of each chunk without having to know anything
|
||||
about its contents, and can confidently re-transmit the chunk to other peers. Peers wishing to
|
||||
transmit chunks to other peers announce to the distributed hash table that they are associated
|
||||
with the sha384 hash sum in question. When a peer wants to download that chunk from the network,
|
||||
it asks the distributed hash table which peers are associated with that sha384 hash sum. The
|
||||
distributed hash table can also be used more generally. It simply stores IP addresses and
|
||||
ports which are associated with 384-bit numbers, and can be used by any type of application to
|
||||
help peers find each other. For example, an application for which clients don't know all of the
|
||||
necessary chunks may use some identifier, chosen by the application, to find clients which do
|
||||
know all of the necessary chunks.
|
||||
|
||||
## For Developers
|
||||
|
||||
The bundled LBRY application uses the lbrynet JSONRPC api found in `lbrynet.lbrynet_daemon.LBRYDaemon`. This api allows for applications and web services like the lbry browser UI to interact with lbrynet. If you've installed lbrynet, you can run `lbrynet-daemon` without running the app. While the app or `lbrynet-daemon` is running, you can use the following to show the help for all the available commands:
|
||||
|
||||
```
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
API_CONNECTION_STRING = "http://localhost:5279/lbryapi"
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
if not api.is_running():
|
||||
print api.daemon_status()
|
||||
else:
|
||||
for func in api.help():
|
||||
print "%s:\n%s" % (func, api.help({'function': func}))
|
||||
```
|
||||
|
||||
If you've installed lbrynet, it comes with a file sharing application, called `lbrynet-daemon`, which breaks
|
||||
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
|
||||
a special file called a 'stream descriptor' containing the hash sums and some other file metadata,
|
||||
and makes the chunks available for download by other peers. A peer wishing to download the file
|
||||
must first obtain the 'stream descriptor' and then may open it with his `lbrynet-daemon` client,
|
||||
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
|
||||
chunks into the original file, according to the metadata included in the 'stream descriptor'.
|
||||
|
||||
For instructions on installing from source, see [INSTALL.md](INSTALL.md).
|
||||
|
||||
Source code: https://github.com/lbryio/lbry
|
||||
|
||||
To contribute, [join us on Slack](https://lbry-slackin.herokuapp.com/) or contact josh@lbry.io. Pull requests are also welcome.
|
||||
|
||||
## Support
|
||||
|
||||
Please open an issue and describe your situation in detail. We will respond as soon as we can.
|
||||
|
||||
For private issues, contact josh@lbry.io.
|
||||
|
||||
## License
|
||||
|
||||
See [LICENSE](LICENSE)
|
||||
@@ -1,52 +0,0 @@
|
||||
To install LBRYnet and lbrynet-console, see INSTALL
|
||||
|
||||
lbrynet-console is a console application which makes use of the LBRYnet to share files.
|
||||
|
||||
In particular, lbrynet-console splits files into encrypted chunks of data compatible with
|
||||
LBRYnet, groups all metadata into a 'stream descriptor file' which can be sent directly to
|
||||
others wishing to obtain the file, or can be itself turned into a chunk compatible with
|
||||
LBRYnet and downloaded via LBRYnet by anyone knowing its sha384 hashsum. lbrynet-console
|
||||
also acts as a client whichreads a stream descriptor file, downloads the chunks of data
|
||||
specified by the hash sums found in the stream descriptor file, decrypts them according to
|
||||
metadata found in the stream, and reconstructs the original file. lbrynet-console features
|
||||
a server so that clients can connect to it and download the chunks and other data gotten
|
||||
from files created locally and files that have been downloaded from LBRYnet.
|
||||
|
||||
lbrynet-console also has a plugin system. There are two plugins: a live stream proof of
|
||||
concept which is currently far behind the development of the rest of the application and
|
||||
therefore will not run, and a plugin which attempts to determine which chunks on the
|
||||
network should be downloaded in order for the application to turn a profit. It will run,
|
||||
but its usefulness is extremely limited.
|
||||
|
||||
Passing '--help' to lbrynet-console will cause it to print out a quick help message
|
||||
describing other command line options to the application.
|
||||
|
||||
Once the application has been started, the user is presented with a numbered list of
|
||||
actions which looks something like this:
|
||||
|
||||
...
|
||||
[2] Toggle whether an LBRY File is running
|
||||
[3] Create an LBRY File from file
|
||||
[4] Publish a stream descriptor file to the DHT for an LBRY File
|
||||
...
|
||||
|
||||
To perform an action, type the desired number and then hit enter. For example, if you wish
|
||||
to create an LBRY file from a file as described in the beginning of this document, type 3 and
|
||||
hit enter.
|
||||
|
||||
If the application needs more input in order to for the action to be taken, the application
|
||||
will continue to print prompts for input until it has received what it needs.
|
||||
|
||||
For example, when creating an LBRY file from a file, the application needs to know which file
|
||||
it's supposed to use to create the LBRY file, so the user will be prompted for it:
|
||||
|
||||
File name:
|
||||
|
||||
The user should input the desired file name and hit enter, at which point the application
|
||||
will go about splitting the file and making it available on the network.
|
||||
|
||||
Some actions will produce sub-menus of actions, which work the same way.
|
||||
|
||||
A more detailed user guide is available at doc.lbry.io
|
||||
|
||||
Any issues may be reported to jimmy@lbry.io
|
||||
@@ -0,0 +1,6 @@
|
||||
import logging
|
||||
|
||||
__version__ = "0.7.9"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -0,0 +1,6 @@
|
||||
from constants import *
|
||||
from events import *
|
||||
from api import Api
|
||||
from track import Track
|
||||
from manager import Manager
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
|
||||
from requests import auth
|
||||
from requests_futures import sessions
|
||||
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.analytics import utils
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def log_response(fn):
|
||||
def _log(future):
|
||||
if future.cancelled():
|
||||
log.warning('Request was unexpectedly cancelled')
|
||||
elif future.exception():
|
||||
exc, traceback = future.exception_info()
|
||||
log.warning('Failed to send an analytics event', exc_info=(type(exc), exc, traceback))
|
||||
else:
|
||||
response = future.result()
|
||||
log.debug('Response (%s): %s', response.status_code, response.content)
|
||||
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
future = fn(*args, **kwargs)
|
||||
future.add_done_callback(_log)
|
||||
return future
|
||||
return wrapper
|
||||
|
||||
|
||||
class Api(object):
|
||||
def __init__(self, session, url, write_key):
|
||||
self.session = session
|
||||
self.url = url
|
||||
self.write_key = write_key
|
||||
|
||||
def post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
assert endpoint[0] == '/'
|
||||
headers = {"Connection": "close"}
|
||||
return self.session.post(
|
||||
self.url + endpoint, json=data, auth=self.auth, headers=headers)
|
||||
|
||||
@property
|
||||
def auth(self):
|
||||
return auth.HTTPBasicAuth(self.write_key, '')
|
||||
|
||||
@log_response
|
||||
def batch(self, events):
|
||||
"""Send multiple events in one request.
|
||||
|
||||
Each event needs to have its type specified.
|
||||
"""
|
||||
data = json.dumps({
|
||||
'batch': events,
|
||||
'sentAt': utils.now(),
|
||||
})
|
||||
log.debug('sending %s events', len(events))
|
||||
log.debug('Data: %s', data)
|
||||
return self.post('/batch', data)
|
||||
|
||||
@log_response
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
log.debug('Sending track event: %s', event)
|
||||
return self.post('/track', event)
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, session=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if not session:
|
||||
session = sessions.FuturesSession()
|
||||
return cls(
|
||||
session,
|
||||
settings.ANALYTICS_ENDPOINT,
|
||||
utils.deobfuscate(settings.ANALYTICS_TOKEN)
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Constants for metrics"""
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
BLOB_BYTES_AVAILABLE = 'Blob Bytes Available'
|
||||
@@ -0,0 +1,104 @@
|
||||
import logging
|
||||
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.conf import LBRYUM_WALLET
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_sd_hash(stream_info):
|
||||
if not stream_info:
|
||||
return None
|
||||
try:
|
||||
return stream_info['sources']['lbry_sd_hash']
|
||||
except (KeyError, TypeError, ValueError):
|
||||
log.debug('Failed to get sd_hash from %s', stream_info, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
class Events(object):
|
||||
def __init__(self, context, lbry_id, session_id):
|
||||
self.context = context
|
||||
self.lbry_id = lbry_id
|
||||
self.session_id = session_id
|
||||
|
||||
def update_context(self, context):
|
||||
self.context = context
|
||||
|
||||
def server_startup(self):
|
||||
return self._event('Server Startup')
|
||||
|
||||
def server_startup_success(self):
|
||||
return self._event('Server Startup Success')
|
||||
|
||||
def server_startup_error(self, message):
|
||||
return self._event('Server Startup Error', {
|
||||
'message': message,
|
||||
})
|
||||
|
||||
def heartbeat(self):
|
||||
return self._event('Heartbeat')
|
||||
|
||||
def download_started(self, name, stream_info=None):
|
||||
properties = {
|
||||
'name': name,
|
||||
'stream_info': get_sd_hash(stream_info)
|
||||
}
|
||||
return self._event('Download Started', properties)
|
||||
|
||||
def error(self, message, sd_hash=None):
|
||||
properties = {
|
||||
'message': message,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
return self._event('Error', properties)
|
||||
|
||||
def metric_observed(self, metric_name, value):
|
||||
properties = {
|
||||
'value': value,
|
||||
}
|
||||
return self._event(metric_name, properties)
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _properties(self, event_properties=None):
|
||||
event_properties = event_properties or {}
|
||||
properties = {
|
||||
'lbry_id': self.lbry_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties)
|
||||
return properties
|
||||
|
||||
|
||||
def make_context(platform, wallet):
|
||||
return {
|
||||
'app': {
|
||||
'name': 'lbrynet',
|
||||
'version': platform['lbrynet_version'],
|
||||
'ui_version': platform['ui_version'],
|
||||
'python_version': platform['python_version'],
|
||||
'build': platform['build'],
|
||||
'wallet': {
|
||||
'name': wallet,
|
||||
# TODO: add in version info for lbrycrdd
|
||||
'version': platform['lbryum_version'] if wallet == LBRYUM_WALLET else None
|
||||
},
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
'library': {
|
||||
'name': 'lbrynet-analytics',
|
||||
'version': '1.0.0'
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
from lbrynet.core import looping_call_manager
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import task
|
||||
|
||||
from lbrynet.core.Platform import get_platform
|
||||
from lbrynet.conf import settings
|
||||
|
||||
import constants
|
||||
from api import Api
|
||||
from events import Events, make_context
|
||||
from track import Track
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, analytics_api, events_generator, track):
|
||||
self.analytics_api = analytics_api
|
||||
self.events_generator = events_generator
|
||||
self.track = track
|
||||
self.looping_call_manager = self.setup_looping_calls()
|
||||
self.is_started = False
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, api=None, events=None):
|
||||
if api is None:
|
||||
api = Api.new_instance()
|
||||
if events is None:
|
||||
events = Events(
|
||||
make_context(get_platform(), settings.wallet),
|
||||
'not loaded', 'not loaded'
|
||||
)
|
||||
return cls(api, events, Track())
|
||||
|
||||
def update_events_generator(self, events_generator):
|
||||
self.events_generator = events_generator
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 60),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 300),
|
||||
]
|
||||
|
||||
def setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def send_server_startup(self):
|
||||
event = self.events_generator.server_startup()
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_server_startup_success(self):
|
||||
event = self.events_generator.server_startup_success()
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
event = self.events_generator.server_startup_error(message)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_download_started(self, name, stream_info=None):
|
||||
event = self.events_generator.download_started(name, stream_info)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def send_error(self, message, sd_hash=None):
|
||||
event = self.events_generator.error(message, sd_hash)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _send_heartbeat(self):
|
||||
heartbeat = self.events_generator.heartbeat()
|
||||
self.analytics_api.track(heartbeat)
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
should_send, value = self.track.summarize_and_reset(constants.BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
event = self.events_generator.metric_observed(constants.BLOB_BYTES_UPLOADED, value)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
event = self.events_generator.metric_observed(event_name, value)
|
||||
self.analytics_api.track(event)
|
||||
|
||||
|
||||
def if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
@@ -0,0 +1,24 @@
|
||||
import collections
|
||||
|
||||
|
||||
class Track(object):
|
||||
"""Track and summarize observations of metrics."""
|
||||
def __init__(self):
|
||||
self.data = collections.defaultdict(list)
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self.data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self.data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
@@ -0,0 +1,2 @@
|
||||
# dont touch this. Travis changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
+352
-20
@@ -1,25 +1,357 @@
|
||||
"""
|
||||
Some network wide and also application specific parameters
|
||||
"""
|
||||
|
||||
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
from appdirs import user_data_dir
|
||||
import envparse
|
||||
|
||||
LBRYCRD_WALLET = 'lbrycrd'
|
||||
LBRYUM_WALLET = 'lbryum'
|
||||
PTC_WALLET = 'ptc'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
MAX_HANDSHAKE_SIZE = 2**16
|
||||
MAX_REQUEST_SIZE = 2**16
|
||||
MAX_BLOB_REQUEST_SIZE = 2**16
|
||||
MAX_RESPONSE_INFO_SIZE = 2**16
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
BLOBFILES_DIR = ".blobfiles"
|
||||
BLOB_SIZE = 2**21
|
||||
MIN_BLOB_DATA_PAYMENT_RATE = .5 # points/megabyte
|
||||
MIN_BLOB_INFO_PAYMENT_RATE = 2.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MAX_CONNECTIONS_PER_STREAM = 5
|
||||
LINUX = 1
|
||||
DARWIN = 2
|
||||
WINDOWS = 3
|
||||
KB = 2**10
|
||||
MB = 2**20
|
||||
|
||||
POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
|
||||
#POINTTRADER_SERVER = 'http://127.0.0.1:2424'
|
||||
|
||||
CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
if sys.platform.startswith("darwin"):
|
||||
platform = DARWIN
|
||||
default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
|
||||
default_data_dir = user_data_dir("LBRY")
|
||||
default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum")
|
||||
elif sys.platform.startswith("win"):
|
||||
platform = WINDOWS
|
||||
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
|
||||
default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
default_data_dir = os.path.join(
|
||||
get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrynet")
|
||||
default_lbryum_dir = os.path.join(
|
||||
get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum")
|
||||
else:
|
||||
platform = LINUX
|
||||
default_download_directory = os.path.join(os.path.expanduser("~"), 'Downloads')
|
||||
default_data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
default_lbryum_dir = os.path.join(os.path.expanduser("~"), ".lbryum")
|
||||
|
||||
|
||||
class Settings(object):
|
||||
"""A collection of configuration settings"""
|
||||
__fixed = []
|
||||
_excluded = ['get_dict', 'update']
|
||||
|
||||
def __iter__(self):
|
||||
for k in self.__dict__.iterkeys():
|
||||
if k.startswith('_') or k in self._excluded:
|
||||
continue
|
||||
yield k
|
||||
|
||||
def __getitem__(self, item):
|
||||
assert item in self, IndexError
|
||||
return self.__dict__[item]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
assert key in self and key not in self.__fixed, KeyError(key)
|
||||
self.__dict__[key] = value
|
||||
|
||||
def __contains__(self, item):
|
||||
return item in iter(self)
|
||||
|
||||
def get_dict(self):
|
||||
return {k: self[k] for k in self}
|
||||
|
||||
def update(self, updated_settings):
|
||||
for k, v in updated_settings.iteritems():
|
||||
try:
|
||||
self.__setitem__(k, v)
|
||||
except (KeyError, AssertionError):
|
||||
pass
|
||||
|
||||
|
||||
class Env(envparse.Env):
|
||||
"""An Env parser that automatically namespaces the variables with LBRY"""
|
||||
NAMESPACE = 'LBRY_'
|
||||
def __init__(self, **schema):
|
||||
self.original_schema = schema
|
||||
my_schema = {
|
||||
self._convert_key(key): self._convert_value(value)
|
||||
for key, value in schema.items()
|
||||
}
|
||||
envparse.Env.__init__(self, **my_schema)
|
||||
|
||||
def __call__(self, key, *args, **kwargs):
|
||||
my_key = self._convert_key(key)
|
||||
return super(Env, self).__call__(my_key, *args, **kwargs)
|
||||
|
||||
def _convert_key(self, key):
|
||||
return Env.NAMESPACE + key.upper()
|
||||
|
||||
def _convert_value(self, value):
|
||||
"""Allow value to be specified as an object, tuple or dict
|
||||
|
||||
if object or dict, follow default envparse rules, if tuple
|
||||
it needs to be of the form (cast, default) or (cast, default, subcast)
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, (tuple, list)):
|
||||
new_value = {'cast': value[0], 'default': value[1]}
|
||||
if len(value) == 3:
|
||||
new_value['subcast'] = value[2]
|
||||
return new_value
|
||||
return value
|
||||
|
||||
|
||||
def server_port(server_port):
|
||||
server, port = server_port.split(':')
|
||||
return server, port
|
||||
|
||||
|
||||
DEFAULT_DHT_NODES = [
|
||||
('lbrynet1.lbry.io', 4444),
|
||||
('lbrynet2.lbry.io', 4444),
|
||||
('lbrynet3.lbry.io', 4444)
|
||||
]
|
||||
|
||||
|
||||
ENVIRONMENT = Env(
|
||||
is_generous_host=(bool, True),
|
||||
run_on_startup=(bool, False),
|
||||
download_directory=(str, default_download_directory),
|
||||
max_upload=(float, 0.0),
|
||||
max_download=(float, 0.0),
|
||||
upload_log=(bool, True),
|
||||
delete_blobs_on_remove=(bool, True),
|
||||
use_upnp=(bool, True),
|
||||
start_lbrycrdd=(bool, True),
|
||||
run_reflector_server=(bool, False),
|
||||
startup_scripts=(list, []),
|
||||
# TODO: this doesn't seem like the kind of thing that should
|
||||
# be configured; move it elsewhere.
|
||||
last_version=(dict, {'lbrynet': '0.0.1', 'lbryum': '0.0.1'}),
|
||||
peer_port=(int, 3333),
|
||||
dht_node_port=(int, 4444),
|
||||
reflector_port=(int, 5566),
|
||||
download_timeout=(int, 30),
|
||||
max_search_results=(int, 25),
|
||||
search_timeout=(float, 3.0),
|
||||
cache_time=(int, 150),
|
||||
host_ui=(bool, True),
|
||||
check_ui_requirements=(bool, True),
|
||||
local_ui_path=(bool, False),
|
||||
api_port=(int, 5279),
|
||||
search_servers=(list, ['lighthouse1.lbry.io:50005']),
|
||||
data_rate=(float, .0001), # points/megabyte
|
||||
min_info_rate=(float, .02), # points/1000 infos
|
||||
min_valuable_info_rate=(float, .05), # points/1000 infos
|
||||
min_valuable_hash_rate=(float, .05), # points/1000 infos
|
||||
max_connections_per_stream=(int, 5),
|
||||
known_dht_nodes=(list, DEFAULT_DHT_NODES, server_port),
|
||||
pointtrader_server=(str, 'http://127.0.0.1:2424'),
|
||||
reflector_servers=(list, [("reflector.lbry.io", 5566)], server_port),
|
||||
wallet=(str, LBRYUM_WALLET),
|
||||
ui_branch=(str, "master"),
|
||||
default_ui_branch=(str, 'master'),
|
||||
data_dir=(str, default_data_dir),
|
||||
lbryum_wallet_dir=(str, default_lbryum_dir),
|
||||
use_auth_http=(bool, False),
|
||||
sd_download_timeout=(int, 3),
|
||||
# TODO: this field is more complicated than it needs to be because
|
||||
# it goes through a Fee validator when loaded by the exchange rate
|
||||
# manager. Look into refactoring the exchange rate conversion to
|
||||
# take in a simpler form.
|
||||
#
|
||||
# TODO: writing json on the cmd line is a pain, come up with a nicer
|
||||
# parser for this data structure. (maybe MAX_KEY_FEE=USD:25
|
||||
max_key_fee=(json.loads, {'USD': {'amount': 25.0, 'address': ''}}),
|
||||
# Changing this value is not-advised as it could potentially
|
||||
# expose the lbrynet daemon to the outside world which would
|
||||
# give an attacker access to your wallet and you could lose
|
||||
# all of your credits.
|
||||
API_INTERFACE=(str, "localhost"),
|
||||
)
|
||||
|
||||
|
||||
class AdjustableSettings(Settings):
|
||||
_excluded = ['get_dict', 'update', 'environ']
|
||||
|
||||
"""Settings that are allowed to be overriden by the user"""
|
||||
def __init__(self, environ=None):
|
||||
self.environ = environ or ENVIRONMENT
|
||||
|
||||
for opt in self.environ.original_schema:
|
||||
self.__dict__[opt] = self.environ(opt)
|
||||
|
||||
Settings.__init__(self)
|
||||
|
||||
def __getattr__(self, attr):
|
||||
if attr in self.environ.original_schema:
|
||||
return self.environ(attr)
|
||||
raise AttributeError
|
||||
|
||||
class ApplicationSettings(Settings):
|
||||
"""Settings that are constants and shouldn't be overriden"""
|
||||
def __init__(self):
|
||||
self.MAX_HANDSHAKE_SIZE = 64*KB
|
||||
self.MAX_REQUEST_SIZE = 64*KB
|
||||
self.MAX_BLOB_REQUEST_SIZE = 64*KB
|
||||
self.MAX_RESPONSE_INFO_SIZE = 64*KB
|
||||
self.MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
self.BLOBFILES_DIR = "blobfiles"
|
||||
self.BLOB_SIZE = 2*MB
|
||||
self.LOG_FILE_NAME = "lbrynet.log"
|
||||
self.LOG_POST_URL = "https://lbry.io/log-upload"
|
||||
self.CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
self.API_ADDRESS = "lbryapi"
|
||||
self.ICON_PATH = "icons" if platform is WINDOWS else "app.icns"
|
||||
self.APP_NAME = "LBRY"
|
||||
self.PROTOCOL_PREFIX = "lbry"
|
||||
self.WALLET_TYPES = [LBRYUM_WALLET, LBRYCRD_WALLET]
|
||||
self.SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
|
||||
self.CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
self.LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv'
|
||||
self.ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
self.ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
self.DB_REVISION_FILE_NAME = 'db_revision'
|
||||
Settings.__init__(self)
|
||||
|
||||
|
||||
APPLICATION_SETTINGS = AdjustableSettings()
|
||||
ADJUSTABLE_SETTINGS = AdjustableSettings()
|
||||
|
||||
|
||||
class DefaultSettings(ApplicationSettings, AdjustableSettings):
|
||||
__fixed = APPLICATION_SETTINGS.get_dict().keys()
|
||||
|
||||
def __init__(self):
|
||||
ApplicationSettings.__init__(self)
|
||||
AdjustableSettings.__init__(self)
|
||||
|
||||
def get_dict(self):
|
||||
d = ApplicationSettings.get_dict(self)
|
||||
d.update(AdjustableSettings.get_dict(self))
|
||||
return d
|
||||
|
||||
|
||||
DEFAULT_SETTINGS = DefaultSettings()
|
||||
|
||||
|
||||
class Config(DefaultSettings):
|
||||
__shared_state = copy.deepcopy(DEFAULT_SETTINGS.get_dict())
|
||||
|
||||
@property
|
||||
def ORIGIN(self):
|
||||
return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port)
|
||||
|
||||
@property
|
||||
def REFERER(self):
|
||||
return "http://%s:%i/" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port)
|
||||
|
||||
@property
|
||||
def API_CONNECTION_STRING(self):
|
||||
return "http://%s:%i/%s" % (
|
||||
DEFAULT_SETTINGS.API_INTERFACE, self.api_port, DEFAULT_SETTINGS.API_ADDRESS)
|
||||
|
||||
@property
|
||||
def UI_ADDRESS(self):
|
||||
return "http://%s:%i" % (DEFAULT_SETTINGS.API_INTERFACE, self.api_port)
|
||||
|
||||
def get_dict(self):
|
||||
return {k: self[k] for k in self}
|
||||
|
||||
def get_adjustable_settings_dict(self):
|
||||
return {opt: val for opt, val in self.get_dict().iteritems() if opt in self.environ.original_schema}
|
||||
|
||||
def ensure_data_dir(self):
|
||||
# although there is a risk of a race condition here we don't
|
||||
# expect there to be multiple processes accessing this
|
||||
# directory so the risk can be ignored
|
||||
if not os.path.isdir(self.data_dir):
|
||||
os.makedirs(self.data_dir)
|
||||
return self.data_dir
|
||||
|
||||
def get_log_filename(self):
|
||||
"""Return the log file for this platform.
|
||||
|
||||
Also ensure the containing directory exists
|
||||
"""
|
||||
return os.path.join(self.ensure_data_dir(), self.LOG_FILE_NAME)
|
||||
|
||||
def get_db_revision_filename(self):
|
||||
return os.path.join(self.ensure_data_dir(), self.DB_REVISION_FILE_NAME)
|
||||
|
||||
def get_conf_filename(self):
|
||||
return get_settings_file_ext(self.ensure_data_dir())
|
||||
|
||||
|
||||
def update_settings_from_file(filename=None):
|
||||
filename = filename or settings.get_conf_filename()
|
||||
try:
|
||||
updates = load_settings(filename)
|
||||
log.info("Loaded settings file: %s", updates)
|
||||
settings.update(updates)
|
||||
except (IOError, OSError) as ex:
|
||||
log.info('%s: Failed to update settings from %s', ex, filename)
|
||||
|
||||
|
||||
def get_settings_file_ext(data_dir):
|
||||
yml_path = os.path.join(data_dir, "daemon_settings.yml")
|
||||
json_path = os.path.join(data_dir, "daemon_settings.json")
|
||||
if os.path.isfile(yml_path):
|
||||
return yml_path
|
||||
elif os.path.isfile(json_path):
|
||||
return json_path
|
||||
else:
|
||||
return yml_path
|
||||
|
||||
|
||||
settings_decoders = {
|
||||
'.json': json.loads,
|
||||
'.yml': yaml.load
|
||||
}
|
||||
|
||||
settings_encoders = {
|
||||
'.json': json.dumps,
|
||||
'.yml': yaml.safe_dump
|
||||
}
|
||||
|
||||
|
||||
def load_settings(path):
|
||||
ext = os.path.splitext(path)[1]
|
||||
with open(path, 'r') as settings_file:
|
||||
data = settings_file.read()
|
||||
decoder = settings_decoders.get(ext, False)
|
||||
assert decoder is not False, "Unknown settings format .%s" % ext
|
||||
return decoder(data)
|
||||
|
||||
|
||||
# TODO: be careful with this. If a setting is overriden by an environment variable
|
||||
# or command line flag we don't want to persist it for future settings.
|
||||
def save_settings(path=None):
|
||||
path = path or settings.get_conf_filename()
|
||||
to_save = settings.get_adjustable_settings_dict()
|
||||
|
||||
ext = os.path.splitext(path)[1]
|
||||
encoder = settings_encoders.get(ext, False)
|
||||
assert encoder is not False, "Unknown settings format .%s" % ext
|
||||
with open(path, 'w') as settings_file:
|
||||
settings_file.write(encoder(to_save))
|
||||
|
||||
|
||||
# TODO: don't load the configuration automatically. The configuration
|
||||
# should be loaded at runtime, not at module import time. Module
|
||||
# import should have no side-effects. This is also bad because
|
||||
# it means that settings are read from the environment even for
|
||||
# tests, which is rarely what you want to happen.
|
||||
settings = Config()
|
||||
@@ -0,0 +1,95 @@
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
from decimal import Decimal
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityTracker(object):
|
||||
"""
|
||||
Class to track peer counts for known blobs, and to discover new popular blobs
|
||||
|
||||
Attributes:
|
||||
availability (dict): dictionary of peers for known blobs
|
||||
"""
|
||||
|
||||
def __init__(self, blob_manager, peer_finder, dht_node):
|
||||
self.availability = {}
|
||||
self._last_mean_availability = Decimal(0.0)
|
||||
self._blob_manager = blob_manager
|
||||
self._peer_finder = peer_finder
|
||||
self._dht_node = dht_node
|
||||
self._check_popular = LoopingCall(self._update_most_popular)
|
||||
self._check_mine = LoopingCall(self._update_mine)
|
||||
|
||||
def start(self):
|
||||
log.info("Starting %s", self)
|
||||
self._check_popular.start(30)
|
||||
self._check_mine.start(120)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self._check_popular.running:
|
||||
self._check_popular.stop()
|
||||
if self._check_mine.running:
|
||||
self._check_mine.stop()
|
||||
|
||||
def get_blob_availability(self, blob):
|
||||
def _get_peer_count(peers):
|
||||
have_blob = sum(1 for peer in peers if peer.is_available())
|
||||
return {blob: have_blob}
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob)
|
||||
d.addCallback(_get_peer_count)
|
||||
return d
|
||||
|
||||
def get_availability_for_blobs(self, blobs):
|
||||
dl = [self.get_blob_availability(blob) for blob in blobs if blob]
|
||||
d = defer.DeferredList(dl)
|
||||
d.addCallback(lambda results: [val for success, val in results if success])
|
||||
return d
|
||||
|
||||
@property
|
||||
def last_mean_availability(self):
|
||||
return max(Decimal(0.01), self._last_mean_availability)
|
||||
|
||||
|
||||
def _update_peers_for_blob(self, blob):
|
||||
def _save_peer_info(blob_hash, peers):
|
||||
v = {blob_hash: peers}
|
||||
self.availability.update(v)
|
||||
return v
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob)
|
||||
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
|
||||
d.addCallback(lambda peers: _save_peer_info(blob, peers))
|
||||
return d
|
||||
|
||||
def _get_most_popular(self):
|
||||
dl = []
|
||||
for (hash, _) in self._dht_node.get_most_popular_hashes(100):
|
||||
encoded = hash.encode('hex')
|
||||
dl.append(self._update_peers_for_blob(encoded))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
def _update_most_popular(self):
|
||||
d = self._get_most_popular()
|
||||
d.addCallback(lambda _: self._get_mean_peers())
|
||||
|
||||
def _update_mine(self):
|
||||
def _get_peers(blobs):
|
||||
dl = []
|
||||
for hash in blobs:
|
||||
dl.append(self._update_peers_for_blob(hash))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
d = self._blob_manager.get_all_verified_blobs()
|
||||
d.addCallback(_get_peers)
|
||||
d.addCallback(lambda _: self._get_mean_peers())
|
||||
|
||||
def _get_mean_peers(self):
|
||||
num_peers = [len(self.availability[blob]) for blob in self.availability]
|
||||
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
|
||||
self._last_mean_availability = mean
|
||||
+246
-84
@@ -1,14 +1,19 @@
|
||||
import logging
|
||||
import os
|
||||
import leveldb
|
||||
import time
|
||||
import json
|
||||
from twisted.internet import threads, defer, reactor, task
|
||||
import sqlite3
|
||||
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.HashBlob import BlobFile, TempBlob, BlobFileCreator, TempBlobCreator
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.Error import NoSuchBlobError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager(DHTHashSupplier):
|
||||
@@ -62,30 +67,43 @@ class BlobManager(DHTHashSupplier):
|
||||
def blob_paid_for(self, blob_hash, amount):
|
||||
pass
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
pass
|
||||
|
||||
def add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
pass
|
||||
|
||||
def add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
pass
|
||||
|
||||
|
||||
class DiskBlobManager(BlobManager):
|
||||
"""This class stores blobs on the hard disk"""
|
||||
def __init__(self, hash_announcer, blob_dir, db_dir):
|
||||
BlobManager.__init__(self, hash_announcer)
|
||||
self.blob_dir = blob_dir
|
||||
self.db_dir = db_dir
|
||||
self.db = None
|
||||
self.db_file = os.path.join(db_dir, "blobs.db")
|
||||
self.db_conn = None
|
||||
self.blob_type = BlobFile
|
||||
self.blob_creator_type = BlobFileCreator
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
self._next_manage_call = None
|
||||
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
log.info("Setting up the DiskBlobManager. blob_dir: %s, db_file: %s", str(self.blob_dir),
|
||||
str(self.db_file))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping the DiskBlobManager")
|
||||
if self._next_manage_call is not None and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
self.db = None
|
||||
#d = self.db_conn.close()
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length=None):
|
||||
@@ -101,7 +119,7 @@ class DiskBlobManager(BlobManager):
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
|
||||
blob = self.blob_type(self.blob_dir, blob_hash, upload_allowed, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
d = threads.deferToThread(self._completed_blobs, [blob_hash])
|
||||
d = self._completed_blobs([blob_hash])
|
||||
|
||||
def check_completed(completed_blobs):
|
||||
|
||||
@@ -110,7 +128,7 @@ class DiskBlobManager(BlobManager):
|
||||
|
||||
if len(completed_blobs) == 1 and completed_blobs[0] == blob_hash:
|
||||
blob.verified = True
|
||||
inner_d = threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
inner_d = self._get_blob_length(blob_hash)
|
||||
inner_d.addCallback(set_length)
|
||||
inner_d.addCallback(lambda _: blob)
|
||||
else:
|
||||
@@ -123,18 +141,18 @@ class DiskBlobManager(BlobManager):
|
||||
def blob_completed(self, blob, next_announce_time=None):
|
||||
if next_announce_time is None:
|
||||
next_announce_time = time.time()
|
||||
return threads.deferToThread(self._add_completed_blob, blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
return self._add_completed_blob(blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
return threads.deferToThread(self._completed_blobs, blobs_to_check)
|
||||
return self._completed_blobs(blobs_to_check)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_blobs_to_announce, next_announce_time)
|
||||
return self._get_blobs_to_announce(next_announce_time)
|
||||
|
||||
def creator_finished(self, blob_creator):
|
||||
logging.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
assert blob_creator.blob_hash is not None
|
||||
assert blob_creator.blob_hash not in self.blobs
|
||||
assert blob_creator.length is not None
|
||||
@@ -155,18 +173,31 @@ class DiskBlobManager(BlobManager):
|
||||
self.blob_hashes_to_delete[blob_hash] = False
|
||||
|
||||
def update_all_last_verified_dates(self, timestamp):
|
||||
return threads.deferToThread(self._update_all_last_verified_dates, timestamp)
|
||||
return self._update_all_last_verified_dates(timestamp)
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
d = threads.deferToThread(self._get_all_verified_blob_hashes)
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.hash_announcer.immediate_announce)
|
||||
return d
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
return threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
return self._get_blob_length(blob_hash)
|
||||
|
||||
def check_consistency(self):
|
||||
return threads.deferToThread(self._check_consistency)
|
||||
return self._check_consistency()
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
def add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
d = self._add_blob_to_download_history(blob_hash, host, rate)
|
||||
return d
|
||||
|
||||
def add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
d = self._add_blob_to_upload_history(blob_hash, host, rate)
|
||||
return d
|
||||
|
||||
def _manage(self):
|
||||
from twisted.internet import reactor
|
||||
@@ -185,19 +216,19 @@ class DiskBlobManager(BlobManager):
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return err
|
||||
|
||||
def delete_from_db(result):
|
||||
b_hs = [r[1] for r in result if r[0] is True]
|
||||
if b_hs:
|
||||
d = threads.deferToThread(self._delete_blobs_from_db, b_hs)
|
||||
d = self._delete_blobs_from_db(b_hs)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
log.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
@@ -221,72 +252,144 @@ class DiskBlobManager(BlobManager):
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "blobs.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', self.db_file, check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" last_verified_time real, " +
|
||||
" next_announce_time real)")
|
||||
|
||||
transaction.execute("create table if not exists download (" +
|
||||
" id integer primary key autoincrement, " +
|
||||
" blob text, " +
|
||||
" host text, " +
|
||||
" rate float, " +
|
||||
" ts integer)")
|
||||
|
||||
transaction.execute("create table if not exists upload (" +
|
||||
" id integer primary key autoincrement, " +
|
||||
" blob text, " +
|
||||
" host text, " +
|
||||
" rate float, " +
|
||||
" ts integer)")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_completed_blob(self, blob_hash, length, timestamp, next_announce_time=None):
|
||||
logging.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
if next_announce_time is None:
|
||||
next_announce_time = timestamp
|
||||
self.db.Put(blob_hash, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into blobs values (?, ?, ?, ?)",
|
||||
(blob_hash, length, timestamp, next_announce_time))
|
||||
d.addErrback(lambda err: err.trap(sqlite3.IntegrityError))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _completed_blobs(self, blobs_to_check):
|
||||
blobs = []
|
||||
for b in blobs_to_check:
|
||||
if is_valid_blobhash(b):
|
||||
try:
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(b))
|
||||
except KeyError:
|
||||
continue
|
||||
blobs_to_check = filter(is_valid_blobhash, blobs_to_check)
|
||||
|
||||
def get_blobs_in_db(db_transaction):
|
||||
blobs_in_db = [] # [(blob_hash, last_verified_time)]
|
||||
for b in blobs_to_check:
|
||||
result = db_transaction.execute("select last_verified_time from blobs where blob_hash = ?",
|
||||
(b,))
|
||||
row = result.fetchone()
|
||||
if row is not None:
|
||||
blobs_in_db.append((b, row[0]))
|
||||
return blobs_in_db
|
||||
|
||||
def get_valid_blobs(blobs_in_db):
|
||||
|
||||
def check_blob_verified_date(b, verified_time):
|
||||
file_path = os.path.join(self.blob_dir, b)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blobs.append(b)
|
||||
return blobs
|
||||
return True
|
||||
return False
|
||||
|
||||
def return_valid_blobs(results):
|
||||
valid_blobs = []
|
||||
for (b, verified_date), (success, result) in zip(blobs_in_db, results):
|
||||
if success is True and result is True:
|
||||
valid_blobs.append(b)
|
||||
return valid_blobs
|
||||
|
||||
ds = []
|
||||
for b, verified_date in blobs_in_db:
|
||||
ds.append(threads.deferToThread(check_blob_verified_date, b, verified_date))
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(return_valid_blobs)
|
||||
return dl
|
||||
|
||||
d = self.db_conn.runInteraction(get_blobs_in_db)
|
||||
d.addCallback(get_valid_blobs)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_length(self, blob):
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
return length
|
||||
d = self.db_conn.runQuery("select blob_length from blobs where blob_hash = ?", (blob,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else Failure(NoSuchBlobError(blob)))
|
||||
return d
|
||||
|
||||
#length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
#return length
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_blob_verified_timestamp(self, blob, timestamp):
|
||||
length, old_verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
self.db.Put(blob, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(blob, timestamp))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blobs_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
blobs = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(blob_hash, json.dumps((length, verified_time, next_announce_time)))
|
||||
blobs.append(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
return blobs
|
||||
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select blob_hash from blobs " +
|
||||
"where next_announce_time < ? and blob_hash is not null",
|
||||
(timestamp,))
|
||||
blobs = [b for b, in r.fetchall()]
|
||||
transaction.execute("update blobs set next_announce_time = ? where next_announce_time < ?",
|
||||
(next_announce_time, timestamp))
|
||||
return blobs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_all_last_verified_dates(self, timestamp):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
batch.Put(blob_hash, json.dumps((length, timestamp, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_date = ?", (timestamp,))
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_blobs_from_db(self, blob_hashes):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash in blob_hashes:
|
||||
batch.Delete(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
|
||||
def delete_blobs(transaction):
|
||||
for b in blob_hashes:
|
||||
transaction.execute("delete from blobs where blob_hash = ?", (b,))
|
||||
|
||||
return self.db_conn.runInteraction(delete_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_consistency(self):
|
||||
batch = leveldb.WriteBatch()
|
||||
|
||||
ALREADY_VERIFIED = 1
|
||||
NEWLY_VERIFIED = 2
|
||||
INVALID = 3
|
||||
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
d = self.db_conn.runQuery("select blob_hash, blob_length, last_verified_time from blobs")
|
||||
|
||||
def check_blob(blob_hash, blob_length, verified_time):
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time < os.path.getctime(file_path):
|
||||
if verified_time >= os.path.getctime(file_path):
|
||||
return ALREADY_VERIFIED
|
||||
else:
|
||||
h = get_lbry_hash_obj()
|
||||
len_so_far = 0
|
||||
f = open(file_path)
|
||||
@@ -296,19 +399,75 @@ class DiskBlobManager(BlobManager):
|
||||
break
|
||||
h.update(data)
|
||||
len_so_far += len(data)
|
||||
if len_so_far == length and h.hexdigest() == blob_hash:
|
||||
batch.Put(blob_hash, json.dumps((length, current_time, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
if len_so_far == blob_length and h.hexdigest() == blob_hash:
|
||||
return NEWLY_VERIFIED
|
||||
return INVALID
|
||||
|
||||
def do_check(blobs):
|
||||
already_verified = []
|
||||
newly_verified = []
|
||||
invalid = []
|
||||
for blob_hash, blob_length, verified_time in blobs:
|
||||
status = check_blob(blob_hash, blob_length, verified_time)
|
||||
if status == ALREADY_VERIFIED:
|
||||
already_verified.append(blob_hash)
|
||||
elif status == NEWLY_VERIFIED:
|
||||
newly_verified.append(blob_hash)
|
||||
else:
|
||||
invalid.append(blob_hash)
|
||||
return already_verified, newly_verified, invalid
|
||||
|
||||
def update_newly_verified(transaction, blobs):
|
||||
for b in blobs:
|
||||
transaction.execute("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(current_time, b))
|
||||
|
||||
def check_blobs(blobs):
|
||||
|
||||
@rerun_if_locked
|
||||
def update_and_return(status_lists):
|
||||
|
||||
already_verified, newly_verified, invalid = status_lists
|
||||
|
||||
d = self.db_conn.runInteraction(update_newly_verified, newly_verified)
|
||||
d.addCallback(lambda _: status_lists)
|
||||
return d
|
||||
|
||||
d = threads.deferToThread(do_check, blobs)
|
||||
|
||||
d.addCallback(update_and_return)
|
||||
return d
|
||||
|
||||
d.addCallback(check_blobs)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
blob_hashes = []
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blob_hashes.append(blob_hash)
|
||||
return blob_hashes
|
||||
d = self.db_conn.runQuery("select blob_hash, last_verified_time from blobs")
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash, verified_time in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blob_to_download_history(self, blob_hash, host, rate):
|
||||
ts = int(time.time())
|
||||
d = self.db_conn.runQuery("insert into download values (null, ?, ?, ?, ?) ", (blob_hash, str(host), float(rate), ts))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blob_to_upload_history(self, blob_hash, host, rate):
|
||||
ts = int(time.time())
|
||||
d = self.db_conn.runQuery("insert into upload values (null, ?, ?, ?, ?) ", (blob_hash, str(host), float(rate), ts))
|
||||
return d
|
||||
|
||||
|
||||
class TempBlobManager(BlobManager):
|
||||
@@ -354,6 +513,10 @@ class TempBlobManager(BlobManager):
|
||||
blobs = [b.blob_hash for b in self.blobs.itervalues() if b.blob_hash in blobs_to_check and b.is_validated()]
|
||||
return defer.succeed(blobs)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self.completed_blobs(self.blobs)
|
||||
return d
|
||||
|
||||
def hashes_to_announce(self):
|
||||
now = time.time()
|
||||
blobs = [blob_hash for blob_hash, announce_time in self.blob_next_announces.iteritems() if announce_time < now]
|
||||
@@ -389,7 +552,7 @@ class TempBlobManager(BlobManager):
|
||||
if blob_hash in self.blobs:
|
||||
if self.blobs[blob_hash].length is not None:
|
||||
return defer.succeed(self.blobs[blob_hash].length)
|
||||
return defer.fail(ValueError("No such blob hash is known"))
|
||||
return defer.fail(NoSuchBlobError(blob_hash))
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
|
||||
@@ -400,20 +563,19 @@ class TempBlobManager(BlobManager):
|
||||
d = self._delete_blobs_marked_for_deletion()
|
||||
|
||||
def set_next_manage_call():
|
||||
logging.info("Setting the next manage call in %s", str(self))
|
||||
log.info("Setting the next manage call in %s", str(self))
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def _delete_blobs_marked_for_deletion(self):
|
||||
|
||||
def remove_from_list(b_h):
|
||||
del self.blob_hashes_to_delete[b_h]
|
||||
logging.info("Deleted blob %s", blob_hash)
|
||||
log.info("Deleted blob %s", blob_hash)
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return b_h
|
||||
|
||||
@@ -422,7 +584,7 @@ class TempBlobManager(BlobManager):
|
||||
if being_deleted is False:
|
||||
if blob_hash in self.blobs:
|
||||
self.blob_hashes_to_delete[blob_hash] = True
|
||||
logging.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
log.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
blob = self.blobs[blob_hash]
|
||||
d = blob.delete()
|
||||
|
||||
@@ -432,7 +594,7 @@ class TempBlobManager(BlobManager):
|
||||
ds.append(d)
|
||||
else:
|
||||
remove_from_list(blob_hash)
|
||||
d = defer.fail(Failure(ValueError("No such blob known")))
|
||||
logging.warning("Blob %s cannot be deleted because it is unknown")
|
||||
d = defer.fail(Failure(NoSuchBlobError(blob_hash)))
|
||||
log.warning("Blob %s cannot be deleted because it is unknown")
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds)
|
||||
return defer.DeferredList(ds)
|
||||
@@ -1,6 +1,21 @@
|
||||
class DownloadOptionChoice(object):
|
||||
"""A possible choice that can be picked for some option.
|
||||
|
||||
An option can have one or more choices that can be picked from.
|
||||
"""
|
||||
def __init__(self, value, short_description, long_description, bool_options_description=None):
|
||||
self.value = value
|
||||
self.short_description = short_description
|
||||
self.long_description = long_description
|
||||
self.bool_options_description = bool_options_description
|
||||
|
||||
|
||||
class DownloadOption(object):
|
||||
def __init__(self, option_types, long_description, short_description, default):
|
||||
"""An option for a user to select a value from several different choices."""
|
||||
def __init__(self, option_types, long_description, short_description, default_value,
|
||||
default_value_description):
|
||||
self.option_types = option_types
|
||||
self.long_description = long_description
|
||||
self.short_description = short_description
|
||||
self.default = default
|
||||
self.default_value = default_value
|
||||
self.default_value_description = default_value_description
|
||||
+58
-2
@@ -22,14 +22,42 @@ class ConnectionClosedBeforeResponseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowed(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownNameError(Exception):
|
||||
def __init__(self, name):
|
||||
Exception.__init__(self, 'Name {} is unknown'.format(name))
|
||||
self.name = name
|
||||
|
||||
|
||||
class InvalidName(Exception):
|
||||
def __init__(self, name, invalid_characters):
|
||||
self.name = name
|
||||
self.invalid_characters = invalid_characters
|
||||
Exception.__init__(
|
||||
self, 'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
|
||||
|
||||
|
||||
class UnknownStreamTypeError(Exception):
|
||||
def __init__(self, stream_type):
|
||||
self.stream_type = stream_type
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.stream_type)
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidStreamInfoError(Exception):
|
||||
def __init__(self, name):
|
||||
def __init__(self, name, stream_info):
|
||||
msg = '{} has claim with invalid stream info: {}'.format(name, stream_info)
|
||||
Exception.__init__(self, msg)
|
||||
self.name = name
|
||||
self.stream_info = stream_info
|
||||
|
||||
|
||||
class MisbehavingPeerError(Exception):
|
||||
@@ -45,4 +73,32 @@ class NoResponseError(MisbehavingPeerError):
|
||||
|
||||
|
||||
class InvalidResponseError(MisbehavingPeerError):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchBlobError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchStreamHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidBlobHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidHeaderError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidAuthenticationToken(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class SubhandlerError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NegotiationError(Exception):
|
||||
pass
|
||||
+16
-11
@@ -8,11 +8,14 @@ from twisted.internet import interfaces, defer, threads
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
implements(interfaces.IConsumer)
|
||||
|
||||
@@ -56,6 +59,9 @@ class HashBlobWriter(object):
|
||||
" %s to %s" % (str(self.len_so_far),
|
||||
str(self.length_getter())))))
|
||||
else:
|
||||
if self.write_handle is None:
|
||||
log.debug("Tried to write to a write_handle that was None.")
|
||||
return
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb(self)
|
||||
@@ -81,10 +87,10 @@ class HashBlob(object):
|
||||
def set_length(self, length):
|
||||
if self.length is not None and length == self.length:
|
||||
return True
|
||||
if self.length is None and 0 <= length <= BLOB_SIZE:
|
||||
if self.length is None and 0 <= length <= settings.BLOB_SIZE:
|
||||
self.length = length
|
||||
return True
|
||||
logging.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
return False
|
||||
|
||||
def get_length(self):
|
||||
@@ -126,8 +132,8 @@ class HashBlob(object):
|
||||
finished_deferred.callback(self)
|
||||
del self.writers[p]
|
||||
return True
|
||||
logging.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
log.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
@@ -199,14 +205,14 @@ class BlobFile(HashBlob):
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
if not peer in self.writers:
|
||||
logging.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
write_file = tempfile.NamedTemporaryFile(delete=False, dir=self.blob_dir)
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(write_file, self.get_length, self.writer_finished)
|
||||
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return finished_deferred, writer.write, writer.cancel
|
||||
logging.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
@@ -232,7 +238,7 @@ class BlobFile(HashBlob):
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
log.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
@@ -247,7 +253,7 @@ class BlobFile(HashBlob):
|
||||
|
||||
def _close_writer(self, writer):
|
||||
if writer.write_handle is not None:
|
||||
logging.debug("Closing %s", str(self))
|
||||
log.debug("Closing %s", str(self))
|
||||
name = writer.write_handle.name
|
||||
writer.write_handle.close()
|
||||
threads.deferToThread(os.remove, name)
|
||||
@@ -335,7 +341,6 @@ class HashBlobCreator(object):
|
||||
else:
|
||||
self.blob_hash = self.hashsum.hexdigest()
|
||||
d = self._close()
|
||||
|
||||
if self.blob_hash is not None:
|
||||
d.addCallback(lambda _: self.blob_manager.creator_finished(self))
|
||||
d.addCallback(lambda _: self.blob_hash)
|
||||
@@ -388,4 +393,4 @@ class TempBlobCreator(HashBlobCreator):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _write(self, data):
|
||||
self.data_buffer += data
|
||||
self.data_buffer += data
|
||||
@@ -1,468 +0,0 @@
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest
|
||||
from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
|
||||
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
|
||||
from twisted.internet import threads, reactor, defer, task
|
||||
from twisted.python.failure import Failure
|
||||
from collections import defaultdict, deque
|
||||
from zope.interface import implements
|
||||
from decimal import Decimal
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
import subprocess
|
||||
import socket
|
||||
import time
|
||||
import os
|
||||
|
||||
|
||||
class ReservedPoints(object):
|
||||
def __init__(self, identifier, amount):
|
||||
self.identifier = identifier
|
||||
self.amount = amount
|
||||
|
||||
|
||||
class LBRYcrdWallet(object):
|
||||
"""This class implements the LBRYWallet interface for the LBRYcrd payment system"""
|
||||
implements(ILBRYWallet)
|
||||
|
||||
def __init__(self, rpc_user, rpc_pass, rpc_url, rpc_port, start_lbrycrdd=False,
|
||||
wallet_dir=None, wallet_conf=None):
|
||||
self.rpc_conn_string = "http://%s:%s@%s:%s" % (rpc_user, rpc_pass, rpc_url, str(rpc_port))
|
||||
self.next_manage_call = None
|
||||
self.wallet_balance = Decimal(0.0)
|
||||
self.total_reserved_points = Decimal(0.0)
|
||||
self.peer_addresses = {} # {Peer: string}
|
||||
self.queued_payments = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.expected_balances = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.current_address_given_to_peer = {} # {Peer: address(string)}
|
||||
self.expected_balance_at_time = deque() # (Peer, address(string), amount(Decimal), time(datetime), count(int),
|
||||
# incremental_amount(float))
|
||||
self.max_expected_payment_time = datetime.timedelta(minutes=3)
|
||||
self.stopped = True
|
||||
self.start_lbrycrdd = start_lbrycrdd
|
||||
self.started_lbrycrdd = False
|
||||
self.wallet_dir = wallet_dir
|
||||
self.wallet_conf = wallet_conf
|
||||
self.lbrycrdd = None
|
||||
self.manage_running = False
|
||||
|
||||
def start(self):
|
||||
|
||||
def make_connection():
|
||||
if self.start_lbrycrdd is True:
|
||||
self._start_daemon()
|
||||
logging.info("Trying to connect to %s", self.rpc_conn_string)
|
||||
self.rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Connected!")
|
||||
|
||||
def start_manage():
|
||||
self.stopped = False
|
||||
self.manage()
|
||||
return True
|
||||
|
||||
d = threads.deferToThread(make_connection)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
self.stopped = True
|
||||
# If self.next_manage_call is None, then manage is currently running or else
|
||||
# start has not been called, so set stopped and do nothing else.
|
||||
if self.next_manage_call is not None:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
|
||||
d = self.manage()
|
||||
if self.start_lbrycrdd is True:
|
||||
d.addBoth(lambda _: self._stop_daemon())
|
||||
return d
|
||||
|
||||
def manage(self):
|
||||
logging.info("Doing manage")
|
||||
self.next_manage_call = None
|
||||
have_set_manage_running = [False]
|
||||
|
||||
def check_if_manage_running():
|
||||
|
||||
d = defer.Deferred()
|
||||
|
||||
def fire_if_not_running():
|
||||
if self.manage_running is False:
|
||||
self.manage_running = True
|
||||
have_set_manage_running[0] = True
|
||||
d.callback(True)
|
||||
else:
|
||||
task.deferLater(reactor, 1, fire_if_not_running)
|
||||
|
||||
fire_if_not_running()
|
||||
return d
|
||||
|
||||
d = check_if_manage_running()
|
||||
|
||||
d.addCallback(lambda _: self._check_expected_balances())
|
||||
|
||||
d.addCallback(lambda _: self._send_payments())
|
||||
|
||||
d.addCallback(lambda _: threads.deferToThread(self._get_wallet_balance))
|
||||
|
||||
def set_wallet_balance(balance):
|
||||
self.wallet_balance = balance
|
||||
|
||||
d.addCallback(set_wallet_balance)
|
||||
|
||||
def set_next_manage_call():
|
||||
if not self.stopped:
|
||||
self.next_manage_call = reactor.callLater(60, self.manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def log_error(err):
|
||||
logging.error("Something went wrong during manage. Error message: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
|
||||
def set_manage_not_running(arg):
|
||||
if have_set_manage_running[0] is True:
|
||||
self.manage_running = False
|
||||
return arg
|
||||
|
||||
d.addBoth(set_manage_not_running)
|
||||
return d
|
||||
|
||||
def get_info_exchanger(self):
|
||||
return LBRYcrdAddressRequester(self)
|
||||
|
||||
def get_wallet_info_query_handler_factory(self):
|
||||
return LBRYcrdAddressQueryHandlerFactory(self)
|
||||
|
||||
def get_balance(self):
|
||||
d = threads.deferToThread(self._get_wallet_balance)
|
||||
return d
|
||||
|
||||
def reserve_points(self, peer, amount):
|
||||
"""
|
||||
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
|
||||
|
||||
@param peer: The peer to which the payment will ultimately be sent
|
||||
|
||||
@param amount: The amount of points to reserve
|
||||
|
||||
@return: A ReservedPoints object which is given to send_points once the service has been rendered
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
#if peer in self.peer_addresses:
|
||||
if self.wallet_balance >= self.total_reserved_points + rounded_amount:
|
||||
self.total_reserved_points += rounded_amount
|
||||
return ReservedPoints(peer, rounded_amount)
|
||||
return None
|
||||
|
||||
def cancel_point_reservation(self, reserved_points):
|
||||
"""
|
||||
Return all of the points that were reserved previously for some ReservedPoints object
|
||||
|
||||
@param reserved_points: ReservedPoints previously returned by reserve_points
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self.total_reserved_points -= reserved_points.amount
|
||||
|
||||
def send_points(self, reserved_points, amount):
|
||||
"""
|
||||
Schedule a payment to be sent to a peer
|
||||
|
||||
@param reserved_points: ReservedPoints object previously returned by reserve_points
|
||||
|
||||
@param amount: amount of points to actually send, must be less than or equal to the
|
||||
amount reserved in reserved_points
|
||||
|
||||
@return: Deferred which fires when the payment has been scheduled
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
peer = reserved_points.identifier
|
||||
assert(rounded_amount <= reserved_points.amount)
|
||||
assert(peer in self.peer_addresses)
|
||||
self.queued_payments[self.peer_addresses[peer]] += rounded_amount
|
||||
# make any unused points available
|
||||
self.total_reserved_points -= (reserved_points.amount - rounded_amount)
|
||||
logging.info("ordering that %s points be sent to %s", str(rounded_amount),
|
||||
str(self.peer_addresses[peer]))
|
||||
peer.update_stats('points_sent', amount)
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_expected_payment(self, peer, amount):
|
||||
"""Increase the number of points expected to be paid by a peer"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
assert(peer in self.current_address_given_to_peer)
|
||||
address = self.current_address_given_to_peer[peer]
|
||||
logging.info("expecting a payment at address %s in the amount of %s", str(address), str(rounded_amount))
|
||||
self.expected_balances[address] += rounded_amount
|
||||
expected_balance = self.expected_balances[address]
|
||||
expected_time = datetime.datetime.now() + self.max_expected_payment_time
|
||||
self.expected_balance_at_time.append((peer, address, expected_balance, expected_time, 0, amount))
|
||||
peer.update_stats('expected_points', amount)
|
||||
|
||||
def update_peer_address(self, peer, address):
|
||||
self.peer_addresses[peer] = address
|
||||
|
||||
def get_new_address_for_peer(self, peer):
|
||||
def set_address_for_peer(address):
|
||||
self.current_address_given_to_peer[peer] = address
|
||||
return address
|
||||
d = threads.deferToThread(self._get_new_address)
|
||||
d.addCallback(set_address_for_peer)
|
||||
return d
|
||||
|
||||
def get_stream_info_for_name(self, name):
|
||||
|
||||
def get_stream_info_from_value(result):
|
||||
r_dict = {}
|
||||
if 'value' in result:
|
||||
value = result['value']
|
||||
try:
|
||||
value_dict = json.loads(value)
|
||||
except ValueError:
|
||||
return Failure(InvalidStreamInfoError(name))
|
||||
if 'stream_hash' in value_dict:
|
||||
r_dict['stream_hash'] = value_dict['stream_hash']
|
||||
if 'name' in value_dict:
|
||||
r_dict['name'] = value_dict['name']
|
||||
if 'description' in value_dict:
|
||||
r_dict['description'] = value_dict['description']
|
||||
return r_dict
|
||||
return Failure(UnknownNameError(name))
|
||||
|
||||
d = threads.deferToThread(self._get_value_for_name, name)
|
||||
d.addCallback(get_stream_info_from_value)
|
||||
return d
|
||||
|
||||
def claim_name(self, stream_hash, name, amount):
|
||||
value = json.dumps({"stream_hash": stream_hash})
|
||||
d = threads.deferToThread(self._claim_name, name, value, amount)
|
||||
return d
|
||||
|
||||
def get_available_balance(self):
|
||||
return float(self.wallet_balance - self.total_reserved_points)
|
||||
|
||||
def get_new_address(self):
|
||||
return threads.deferToThread(self._get_new_address)
|
||||
|
||||
def _start_daemon(self):
|
||||
|
||||
if os.name == "nt":
|
||||
si = subprocess.STARTUPINFO
|
||||
si.dwFlags = subprocess.STARTF_USESHOWWINDOW
|
||||
si.wShowWindow = subprocess.SW_HIDE
|
||||
self.lbrycrdd = subprocess.Popen(["lbrycrdd.exe", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf], startupinfo=si)
|
||||
else:
|
||||
self.lbrycrdd = subprocess.Popen(["./lbrycrdd", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf])
|
||||
self.started_lbrycrdd = True
|
||||
|
||||
tries = 0
|
||||
while tries < 5:
|
||||
try:
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.getinfo()
|
||||
break
|
||||
except (socket.error, JSONRPCException):
|
||||
tries += 1
|
||||
logging.warning("Failed to connect to lbrycrdd.")
|
||||
if tries < 5:
|
||||
time.sleep(2 ** tries)
|
||||
logging.warning("Trying again in %d seconds", 2 ** tries)
|
||||
else:
|
||||
logging.warning("Giving up.")
|
||||
else:
|
||||
self.lbrycrdd.terminate()
|
||||
raise ValueError("Couldn't open lbrycrdd")
|
||||
|
||||
def _stop_daemon(self):
|
||||
if self.lbrycrdd is not None and self.started_lbrycrdd is True:
|
||||
d = threads.deferToThread(self._rpc_stop)
|
||||
return d
|
||||
return defer.succeed(True)
|
||||
|
||||
def _check_expected_balances(self):
|
||||
now = datetime.datetime.now()
|
||||
balances_to_check = []
|
||||
try:
|
||||
while self.expected_balance_at_time[0][3] < now:
|
||||
balances_to_check.append(self.expected_balance_at_time.popleft())
|
||||
except IndexError:
|
||||
pass
|
||||
ds = []
|
||||
for balance_to_check in balances_to_check:
|
||||
d = threads.deferToThread(self._check_expected_balance, balance_to_check)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def handle_checks(results):
|
||||
from future_builtins import zip
|
||||
for balance, (success, result) in zip(balances_to_check, results):
|
||||
peer = balance[0]
|
||||
if success is True:
|
||||
if result is False:
|
||||
if balance[4] <= 1: # first or second strike, give them another chance
|
||||
new_expected_balance = (balance[0],
|
||||
balance[1],
|
||||
balance[2],
|
||||
datetime.datetime.now() + self.max_expected_payment_time,
|
||||
balance[4] + 1,
|
||||
balance[5])
|
||||
self.expected_balance_at_time.append(new_expected_balance)
|
||||
peer.update_score(-5.0)
|
||||
else:
|
||||
peer.update_score(-50.0)
|
||||
else:
|
||||
if balance[4] == 0:
|
||||
peer.update_score(balance[5])
|
||||
peer.update_stats('points_received', balance[5])
|
||||
else:
|
||||
logging.warning("Something went wrong checking a balance. Peer: %s, account: %s,"
|
||||
"expected balance: %s, expected time: %s, count: %s, error: %s",
|
||||
str(balance[0]), str(balance[1]), str(balance[2]), str(balance[3]),
|
||||
str(balance[4]), str(result.getErrorMessage()))
|
||||
|
||||
dl.addCallback(handle_checks)
|
||||
return dl
|
||||
|
||||
def _check_expected_balance(self, expected_balance):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Checking balance of address %s", str(expected_balance[1]))
|
||||
balance = rpc_conn.getreceivedbyaddress(expected_balance[1])
|
||||
logging.debug("received balance: %s", str(balance))
|
||||
logging.debug("expected balance: %s", str(expected_balance[2]))
|
||||
return balance >= expected_balance[2]
|
||||
|
||||
def _send_payments(self):
|
||||
logging.info("Trying to send payments, if there are any to be sent")
|
||||
|
||||
def do_send(payments):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.sendmany("", payments)
|
||||
|
||||
payments_to_send = {}
|
||||
for address, points in self.queued_payments.items():
|
||||
logging.info("Should be sending %s points to %s", str(points), str(address))
|
||||
payments_to_send[address] = float(points)
|
||||
self.total_reserved_points -= points
|
||||
self.wallet_balance -= points
|
||||
del self.queued_payments[address]
|
||||
if payments_to_send:
|
||||
logging.info("Creating a transaction with outputs %s", str(payments_to_send))
|
||||
return threads.deferToThread(do_send, payments_to_send)
|
||||
logging.info("There were no payments to send")
|
||||
return defer.succeed(True)
|
||||
|
||||
def _get_wallet_balance(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getbalance("")
|
||||
|
||||
def _get_new_address(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getnewaddress()
|
||||
|
||||
def _get_value_for_name(self, name):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getvalueforname(name)
|
||||
|
||||
def _claim_name(self, name, value, amount):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return str(rpc_conn.claimname(name, value, amount))
|
||||
|
||||
def _rpc_stop(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.stop()
|
||||
self.lbrycrdd.wait()
|
||||
|
||||
|
||||
class LBRYcrdAddressRequester(object):
|
||||
implements([IRequestCreator])
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self._protocols = []
|
||||
|
||||
######### IRequestCreator #########
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
|
||||
if not protocol in self._protocols:
|
||||
r = ClientRequest({'lbrycrd_address': True}, 'lbrycrd_address')
|
||||
d = protocol.add_request(r)
|
||||
d.addCallback(self._handle_address_response, peer, r, protocol)
|
||||
d.addErrback(self._request_failed, peer)
|
||||
self._protocols.append(protocol)
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
return defer.succeed(False)
|
||||
|
||||
######### internal calls #########
|
||||
|
||||
def _handle_address_response(self, response_dict, peer, request, protocol):
|
||||
assert request.response_identifier in response_dict, \
|
||||
"Expected %s in dict but did not get it" % request.response_identifier
|
||||
assert protocol in self._protocols, "Responding protocol is not in our list of protocols"
|
||||
address = response_dict[request.response_identifier]
|
||||
self.wallet.update_peer_address(peer, address)
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = LBRYcrdAddressQueryHandler(self.wallet)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'lbrycrd_address'
|
||||
|
||||
def get_description(self):
|
||||
return "LBRYcrd Address - an address for receiving payments via LBRYcrd"
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandler(object):
|
||||
implements(IQueryHandler)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = ['lbrycrd_address']
|
||||
self.address = None
|
||||
self.peer = None
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
self.peer = peer
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
|
||||
def create_response(address):
|
||||
self.address = address
|
||||
fields = {'lbrycrd_address': address}
|
||||
return fields
|
||||
|
||||
if self.query_identifiers[0] in queries:
|
||||
d = self.wallet.get_new_address_for_peer(self.peer)
|
||||
d.addCallback(create_response)
|
||||
return d
|
||||
if self.address is None:
|
||||
logging.warning("Expected a request for an address, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive an address request")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
@@ -0,0 +1,62 @@
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class Offer(object):
|
||||
"""A rate offer to download blobs from a host."""
|
||||
|
||||
RATE_ACCEPTED = "RATE_ACCEPTED"
|
||||
RATE_TOO_LOW = "RATE_TOO_LOW"
|
||||
RATE_UNSET = "RATE_UNSET"
|
||||
|
||||
def __init__(self, offer):
|
||||
self._state = None
|
||||
self.rate = None
|
||||
if isinstance(offer, Decimal):
|
||||
self.rate = round(offer, 5)
|
||||
elif isinstance(offer, float):
|
||||
self.rate = round(Decimal(offer), 5)
|
||||
if self.rate is None or self.rate < Decimal(0.0):
|
||||
self.unset()
|
||||
|
||||
@property
|
||||
def is_accepted(self):
|
||||
return self._state is Offer.RATE_ACCEPTED
|
||||
|
||||
@property
|
||||
def is_too_low(self):
|
||||
return self._state is Offer.RATE_TOO_LOW
|
||||
|
||||
@property
|
||||
def is_unset(self):
|
||||
return self._state is Offer.RATE_UNSET
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
if self.is_accepted:
|
||||
return Offer.RATE_ACCEPTED
|
||||
elif self.is_too_low:
|
||||
return Offer.RATE_TOO_LOW
|
||||
elif self.is_unset:
|
||||
return Offer.RATE_UNSET
|
||||
return None
|
||||
|
||||
def accept(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_ACCEPTED
|
||||
|
||||
def reject(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_TOO_LOW
|
||||
|
||||
def unset(self):
|
||||
self._state = Offer.RATE_UNSET
|
||||
|
||||
def handle(self, reply_message):
|
||||
if reply_message == Offer.RATE_TOO_LOW:
|
||||
self.reject()
|
||||
elif reply_message == Offer.RATE_ACCEPTED:
|
||||
self.accept()
|
||||
elif reply_message == Offer.RATE_UNSET:
|
||||
self.unset()
|
||||
else:
|
||||
raise Exception("Unknown offer reply %s" % str(reply_message))
|
||||
+32
-24
@@ -1,24 +1,27 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import leveldb
|
||||
import os
|
||||
import unqlite
|
||||
import time
|
||||
from Crypto.Hash import SHA512
|
||||
from Crypto.PublicKey import RSA
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet
|
||||
from lbrynet.pointtraderclient import pointtraderclient
|
||||
from twisted.internet import defer, threads
|
||||
from zope.interface import implements
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.LBRYcrdWallet import ReservedPoints
|
||||
from lbrynet.core.Wallet import ReservedPoints
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PTCWallet(object):
|
||||
"""This class sends payments to peers and also ensures that expected payments are received.
|
||||
This class is only intended to be used for testing."""
|
||||
implements(ILBRYWallet)
|
||||
implements(IWallet)
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
@@ -71,7 +74,7 @@ class PTCWallet(object):
|
||||
|
||||
def save_key(success, private_key):
|
||||
if success is True:
|
||||
threads.deferToThread(self.save_private_key, private_key.exportKey())
|
||||
self._save_private_key(private_key.exportKey())
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -95,8 +98,8 @@ class PTCWallet(object):
|
||||
def start_manage():
|
||||
self.manage()
|
||||
return True
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d.addCallback(lambda _: threads.deferToThread(self.get_wallet_private_key))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._get_wallet_private_key())
|
||||
d.addCallback(ensure_private_key_exists)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
@@ -160,8 +163,8 @@ class PTCWallet(object):
|
||||
ds.append(d)
|
||||
del self.queued_payments[peer]
|
||||
else:
|
||||
logging.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def get_balance(self):
|
||||
@@ -207,20 +210,25 @@ class PTCWallet(object):
|
||||
if self.peer_pub_keys[peer] in self.received_payments:
|
||||
received_balance = sum([a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
|
||||
if min_expected_balance > received_balance:
|
||||
logging.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
log.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
def open_db():
|
||||
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
return threads.deferToThread(open_db)
|
||||
|
||||
def save_private_key(self, private_key):
|
||||
self.db.Put("private_key", private_key)
|
||||
def _save_private_key(self, private_key):
|
||||
def save_key():
|
||||
self.db['private_key'] = private_key
|
||||
return threads.deferToThread(save_key)
|
||||
|
||||
def get_wallet_private_key(self):
|
||||
try:
|
||||
return self.db.Get("private_key")
|
||||
except KeyError:
|
||||
def _get_wallet_private_key(self):
|
||||
def get_key():
|
||||
if 'private_key' in self.db:
|
||||
return self.db['private_key']
|
||||
return None
|
||||
return threads.deferToThread(get_key)
|
||||
|
||||
|
||||
class PointTraderKeyExchanger(object):
|
||||
@@ -256,9 +264,9 @@ class PointTraderKeyExchanger(object):
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
return err
|
||||
|
||||
|
||||
class PointTraderKeyQueryHandlerFactory(object):
|
||||
@@ -301,15 +309,15 @@ class PointTraderKeyQueryHandler(object):
|
||||
try:
|
||||
RSA.importKey(new_encoded_pub_key)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
logging.warning("Client sent an invalid public key.")
|
||||
log.warning("Client sent an invalid public key.")
|
||||
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
|
||||
self.public_key = new_encoded_pub_key
|
||||
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
|
||||
logging.debug("Received the client's public key: %s", str(self.public_key))
|
||||
log.debug("Received the client's public key: %s", str(self.public_key))
|
||||
fields = {'public_key': self.wallet.encoded_public_key}
|
||||
return defer.succeed(fields)
|
||||
if self.public_key is None:
|
||||
logging.warning("Expected a public key, but did not receive one")
|
||||
log.warning("Expected a public key, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
@@ -1,6 +1,12 @@
|
||||
from lbrynet.core.Strategy import get_default_strategy
|
||||
from lbrynet.conf import settings
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class BasePaymentRateManager(object):
|
||||
def __init__(self, rate):
|
||||
def __init__(self, rate=settings.data_rate, info_rate=settings.min_info_rate):
|
||||
self.min_blob_data_payment_rate = rate
|
||||
self.min_blob_info_payment_rate = info_rate
|
||||
|
||||
|
||||
class PaymentRateManager(object):
|
||||
@@ -26,4 +32,50 @@ class PaymentRateManager(object):
|
||||
return self.min_blob_data_payment_rate
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
self.points_paid += amount
|
||||
|
||||
|
||||
class NegotiatedPaymentRateManager(object):
|
||||
def __init__(self, base, availability_tracker, generous=settings.is_generous_host):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
@param availability_tracker: a BlobAvailabilityTracker
|
||||
@param rate: the min blob data payment rate
|
||||
"""
|
||||
|
||||
self.base = base
|
||||
self.points_paid = 0.0
|
||||
self.blob_tracker = availability_tracker
|
||||
self.generous = generous
|
||||
self.strategy = get_default_strategy(self.blob_tracker,
|
||||
base_price=self.base.min_blob_data_payment_rate,
|
||||
is_generous=generous)
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
return offer.is_too_low and round(Decimal.from_float(offer.rate), 5) >= round(self.strategy.max_rate, 5)
|
||||
return False
|
||||
@@ -1,5 +1,6 @@
|
||||
from collections import defaultdict
|
||||
import datetime
|
||||
from collections import defaultdict
|
||||
from lbrynet.core import utils
|
||||
|
||||
|
||||
class Peer(object):
|
||||
@@ -12,8 +13,7 @@ class Peer(object):
|
||||
self.stats = defaultdict(float) # {string stat_type, float count}
|
||||
|
||||
def is_available(self):
|
||||
if (self.attempt_connection_at is None or
|
||||
datetime.datetime.today() > self.attempt_connection_at):
|
||||
if self.attempt_connection_at is None or utils.today() > self.attempt_connection_at:
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -24,7 +24,7 @@ class Peer(object):
|
||||
def report_down(self):
|
||||
self.down_count += 1
|
||||
timeout_time = datetime.timedelta(seconds=60 * self.down_count)
|
||||
self.attempt_connection_at = datetime.datetime.today() + timeout_time
|
||||
self.attempt_connection_at = utils.today() + timeout_time
|
||||
|
||||
def update_score(self, score_change):
|
||||
self.score += score_change
|
||||
@@ -33,4 +33,7 @@ class Peer(object):
|
||||
self.stats[stat_type] += count
|
||||
|
||||
def __str__(self):
|
||||
return self.host + ":" + str(self.port)
|
||||
return '{}:{}'.format(self.host, self.port)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Peer({!r}, {!r})'.format(self.host, self.port)
|
||||
@@ -0,0 +1,29 @@
|
||||
import platform
|
||||
import simplejson as json
|
||||
|
||||
from urllib2 import urlopen
|
||||
|
||||
from lbrynet import __version__ as lbrynet_version
|
||||
from lbrynet import build_type
|
||||
from lbryum.version import LBRYUM_VERSION as lbryum_version
|
||||
|
||||
|
||||
def get_platform():
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"lbryum_version": lbryum_version,
|
||||
"ui_version": "not loaded yet",
|
||||
"build": build_type.BUILD, # travis sets this during build step
|
||||
}
|
||||
|
||||
try:
|
||||
p['ip'] = json.load(urlopen('http://jsonip.com'))['ip']
|
||||
except:
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
@@ -0,0 +1,50 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.interfaces import IBlobPriceModel
|
||||
from lbrynet.conf import settings
|
||||
|
||||
|
||||
def get_default_price_model(blob_tracker, base_price, **kwargs):
|
||||
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
|
||||
|
||||
|
||||
class MeanAvailabilityWeightedPrice(object):
|
||||
"""
|
||||
Calculate mean-blob-availability and stream-position weighted price for a blob
|
||||
|
||||
Attributes:
|
||||
base_price (float): base price
|
||||
alpha (float): constant, > 0.0 and <= 1.0, used to more highly value blobs at the beginning of a stream.
|
||||
alpha defaults to 1.0, which has a null effect
|
||||
blob_tracker (BlobAvailabilityTracker): blob availability tracker
|
||||
"""
|
||||
implementer(IBlobPriceModel)
|
||||
|
||||
def __init__(self, tracker, base_price=settings.data_rate, alpha=1.0):
|
||||
self.blob_tracker = tracker
|
||||
self.base_price = Decimal(base_price)
|
||||
self.alpha = Decimal(alpha)
|
||||
|
||||
def calculate_price(self, blob):
|
||||
mean_availability = self.blob_tracker.last_mean_availability
|
||||
availability = self.blob_tracker.availability.get(blob, [])
|
||||
index = 0 # blob.index
|
||||
price = self.base_price * self._get_availability_multiplier(mean_availability, availability) / self._frontload(index)
|
||||
return round(price, 5)
|
||||
|
||||
def _get_availability_multiplier(self, mean_availability, availability):
|
||||
return Decimal(max(1, mean_availability) / Decimal(max(1, len(availability))))
|
||||
|
||||
def _frontload(self, index):
|
||||
"""
|
||||
Get front-load multiplier, used to weight prices of blobs in a stream towards the front of the stream.
|
||||
|
||||
At index 0, returns 1.0
|
||||
As index increases, return value approaches 2.0
|
||||
|
||||
@param index: blob position in stream
|
||||
@return: front-load multiplier
|
||||
"""
|
||||
|
||||
return Decimal(2.0) - (self.alpha ** index)
|
||||
+40
-92
@@ -1,5 +1,11 @@
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IRateLimiter
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyRateLimiter(object):
|
||||
@@ -10,22 +16,20 @@ class DummyRateLimiter(object):
|
||||
self.total_ul_bytes = 0
|
||||
self.target_dl = 0
|
||||
self.target_ul = 0
|
||||
self.ul_delay = 0.00
|
||||
self.dl_delay = 0.00
|
||||
self.next_tick = None
|
||||
self.tick_call = None
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(1)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
pass
|
||||
@@ -33,12 +37,6 @@ class DummyRateLimiter(object):
|
||||
def set_ul_limit(self, limit):
|
||||
pass
|
||||
|
||||
def ul_wait_time(self):
|
||||
return self.ul_delay
|
||||
|
||||
def dl_wait_time(self):
|
||||
return self.dl_delay
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
@@ -58,66 +56,34 @@ class RateLimiter(object):
|
||||
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
|
||||
self.max_dl_bytes = max_dl_bytes
|
||||
self.max_ul_bytes = max_ul_bytes
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.next_tick = None
|
||||
self.next_unthrottle_dl = None
|
||||
self.next_unthrottle_ul = None
|
||||
|
||||
self.next_dl_check = None
|
||||
self.next_ul_check = None
|
||||
|
||||
self.dl_check_interval = 1.0
|
||||
self.ul_check_interval = 1.0
|
||||
self.tick_call = None
|
||||
self.tick_interval = 0.1
|
||||
|
||||
self.dl_throttled = False
|
||||
self.ul_throttled = False
|
||||
|
||||
self.protocols = []
|
||||
|
||||
def start(self):
|
||||
log.info("Starting %s", self)
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(self.tick_interval)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
# happens once per second
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
if self.max_dl_bytes is not None:
|
||||
if self.dl_bytes_this_second == 0:
|
||||
self.dl_check_interval = 1.0
|
||||
else:
|
||||
self.dl_check_interval = min(1.0, self.dl_check_interval *
|
||||
self.max_dl_bytes / self.dl_bytes_this_second)
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
if self.max_ul_bytes is not None:
|
||||
if self.ul_bytes_this_second == 0:
|
||||
self.ul_check_interval = 1.0
|
||||
else:
|
||||
self.ul_check_interval = min(1.0, self.ul_check_interval *
|
||||
self.max_ul_bytes / self.ul_bytes_this_second)
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.unthrottle_dl()
|
||||
self.unthrottle_ul()
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
log.info("Stopping %s", self)
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
self.max_dl_bytes = limit
|
||||
@@ -129,27 +95,15 @@ class RateLimiter(object):
|
||||
|
||||
def check_dl(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_dl_check = None
|
||||
|
||||
if self.dl_bytes_this_second > self.max_dl_bytes:
|
||||
self.throttle_dl()
|
||||
else:
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
self.dl_check_interval = min(self.dl_check_interval * 2, 1.0)
|
||||
if self.max_dl_bytes is not None and self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_dl)
|
||||
|
||||
def check_ul(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_ul_check = None
|
||||
|
||||
if self.ul_bytes_this_second > self.max_ul_bytes:
|
||||
self.throttle_ul()
|
||||
else:
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.ul_check_interval = min(self.ul_check_interval * 2, 1.0)
|
||||
if self.max_ul_bytes is not None and self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_ul)
|
||||
|
||||
def throttle_dl(self):
|
||||
if self.dl_throttled is False:
|
||||
@@ -175,23 +129,17 @@ class RateLimiter(object):
|
||||
protocol.unthrottle_upload()
|
||||
self.ul_throttled = False
|
||||
|
||||
#deprecated
|
||||
|
||||
def ul_wait_time(self):
|
||||
return 0
|
||||
|
||||
def dl_wait_time(self):
|
||||
return 0
|
||||
|
||||
#called by protocols
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.dl_bytes_this_interval += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
self.check_dl()
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_second += num_bytes
|
||||
self.ul_bytes_this_interval += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
self.check_ul()
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
if protocol not in self.protocols:
|
||||
@@ -203,4 +151,4 @@ class RateLimiter(object):
|
||||
|
||||
def unregister_protocol(self, protocol):
|
||||
if protocol in self.protocols:
|
||||
self.protocols.remove(protocol)
|
||||
self.protocols.remove(protocol)
|
||||
+177
-73
@@ -9,65 +9,96 @@ from lbrynet.core.client.DHTPeerFinder import DHTPeerFinder
|
||||
from lbrynet.core.HashAnnouncer import DummyHashAnnouncer
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager
|
||||
from lbrynet.core.PaymentRateManager import BasePaymentRateManager, NegotiatedPaymentRateManager
|
||||
from lbrynet.core.BlobAvailability import BlobAvailabilityTracker
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
|
||||
class LBRYSession(object):
|
||||
"""This class manages all important services common to any application that uses the network:
|
||||
the hash announcer, which informs other peers that this peer is associated with some hash. Usually,
|
||||
this means this peer has a blob identified by the hash in question, but it can be used for other
|
||||
purposes.
|
||||
the peer finder, which finds peers that are associated with some hash.
|
||||
the blob manager, which keeps track of which blobs have been downloaded and provides access to them,
|
||||
the rate limiter, which attempts to ensure download and upload rates stay below a set maximum,
|
||||
and upnp, which opens holes in compatible firewalls so that remote peers can connect to this peer."""
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None, lbryid=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None, hash_announcer=None,
|
||||
blob_dir=None, blob_manager=None, peer_port=None, use_upnp=True,
|
||||
rate_limiter=None, wallet=None):
|
||||
"""
|
||||
@param blob_data_payment_rate: The default payment rate for blob data
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Session(object):
|
||||
"""This class manages all important services common to any application that uses the network.
|
||||
|
||||
the hash announcer, which informs other peers that this peer is
|
||||
associated with some hash. Usually, this means this peer has a
|
||||
blob identified by the hash in question, but it can be used for
|
||||
other purposes.
|
||||
|
||||
the peer finder, which finds peers that are associated with some
|
||||
hash.
|
||||
|
||||
the blob manager, which keeps track of which blobs have been
|
||||
downloaded and provides access to them,
|
||||
|
||||
the rate limiter, which attempts to ensure download and upload
|
||||
rates stay below a set maximum
|
||||
|
||||
upnp, which opens holes in compatible firewalls so that remote
|
||||
peers can connect to this peer.
|
||||
"""
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None,
|
||||
lbryid=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None,
|
||||
hash_announcer=None, blob_dir=None,
|
||||
blob_manager=None, peer_port=None, use_upnp=True,
|
||||
rate_limiter=None, wallet=None,
|
||||
dht_node_class=node.Node, blob_tracker_class=None,
|
||||
payment_rate_manager_class=None, is_generous=True):
|
||||
"""@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
@param db_dir: The directory in which levelDB files should be stored
|
||||
|
||||
@param lbryid: The unique ID of this node
|
||||
|
||||
@param peer_manager: An object which keeps track of all known peers. If None, a PeerManager will be created
|
||||
@param peer_manager: An object which keeps track of all known
|
||||
peers. If None, a PeerManager will be created
|
||||
|
||||
@param dht_node_port: The port on which the dht node should listen for incoming connections
|
||||
@param dht_node_port: The port on which the dht node should
|
||||
listen for incoming connections
|
||||
|
||||
@param known_dht_nodes: A list of nodes which the dht node should use to bootstrap into the dht
|
||||
@param known_dht_nodes: A list of nodes which the dht node
|
||||
should use to bootstrap into the dht
|
||||
|
||||
@param peer_finder: An object which is used to look up peers that are associated with some hash. If None,
|
||||
a DHTPeerFinder will be used, which looks for peers in the distributed hash table.
|
||||
@param peer_finder: An object which is used to look up peers
|
||||
that are associated with some hash. If None, a
|
||||
DHTPeerFinder will be used, which looks for peers in the
|
||||
distributed hash table.
|
||||
|
||||
@param hash_announcer: An object which announces to other peers that this peer is associated with some hash.
|
||||
If None, and peer_port is not None, a DHTHashAnnouncer will be used. If None and
|
||||
peer_port is None, a DummyHashAnnouncer will be used, which will not actually announce
|
||||
anything.
|
||||
@param hash_announcer: An object which announces to other
|
||||
peers that this peer is associated with some hash. If
|
||||
None, and peer_port is not None, a DHTHashAnnouncer will
|
||||
be used. If None and peer_port is None, a
|
||||
DummyHashAnnouncer will be used, which will not actually
|
||||
announce anything.
|
||||
|
||||
@param blob_dir: The directory in which blobs will be stored. If None and blob_manager is None, blobs will
|
||||
be stored in memory only.
|
||||
@param blob_dir: The directory in which blobs will be
|
||||
stored. If None and blob_manager is None, blobs will be
|
||||
stored in memory only.
|
||||
|
||||
@param blob_manager: An object which keeps track of downloaded blobs and provides access to them. If None,
|
||||
and blob_dir is not None, a DiskBlobManager will be used, with the given blob_dir.
|
||||
If None and blob_dir is None, a TempBlobManager will be used, which stores blobs in
|
||||
memory only.
|
||||
@param blob_manager: An object which keeps track of downloaded
|
||||
blobs and provides access to them. If None, and blob_dir
|
||||
is not None, a DiskBlobManager will be used, with the
|
||||
given blob_dir. If None and blob_dir is None, a
|
||||
TempBlobManager will be used, which stores blobs in memory
|
||||
only.
|
||||
|
||||
@param peer_port: The port on which other peers should connect to this peer
|
||||
@param peer_port: The port on which other peers should connect
|
||||
to this peer
|
||||
|
||||
@param use_upnp: Whether or not to try to open a hole in the firewall so that outside peers can connect to
|
||||
this peer's peer_port and dht_node_port
|
||||
@param use_upnp: Whether or not to try to open a hole in the
|
||||
firewall so that outside peers can connect to this peer's
|
||||
peer_port and dht_node_port
|
||||
|
||||
@param rate_limiter: An object which keeps track of the amount of data transferred to and from this peer,
|
||||
and can limit that rate if desired
|
||||
@param rate_limiter: An object which keeps track of the amount
|
||||
of data transferred to and from this peer, and can limit
|
||||
that rate if desired
|
||||
|
||||
@param wallet: An object which will be used to keep track of expected payments and which will pay peers.
|
||||
If None, a wallet which uses the Point Trader system will be used, which is meant for testing
|
||||
only
|
||||
@param wallet: An object which will be used to keep track of
|
||||
expected payments and which will pay peers. If None, a
|
||||
wallet which uses the Point Trader system will be used,
|
||||
which is meant for testing only
|
||||
|
||||
@return:
|
||||
"""
|
||||
self.db_dir = db_dir
|
||||
|
||||
@@ -85,6 +116,9 @@ class LBRYSession(object):
|
||||
self.blob_dir = blob_dir
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
self.blob_tracker = None
|
||||
self.blob_tracker_class = blob_tracker_class or BlobAvailabilityTracker
|
||||
|
||||
self.peer_port = peer_port
|
||||
|
||||
self.use_upnp = use_upnp
|
||||
@@ -92,19 +126,22 @@ class LBRYSession(object):
|
||||
self.rate_limiter = rate_limiter
|
||||
|
||||
self.external_ip = '127.0.0.1'
|
||||
self.upnp_handler = None
|
||||
self.upnp_redirects_set = False
|
||||
|
||||
self.upnp_redirects = []
|
||||
|
||||
self.wallet = wallet
|
||||
|
||||
self.dht_node_class = dht_node_class
|
||||
self.dht_node = None
|
||||
|
||||
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
|
||||
self.payment_rate_manager = None
|
||||
self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager
|
||||
self.is_generous = is_generous
|
||||
|
||||
def setup(self):
|
||||
"""Create the blob directory and database if necessary, start all desired services"""
|
||||
|
||||
logging.debug("Setting up the lbry session")
|
||||
log.debug("Setting up the lbry session")
|
||||
|
||||
if self.lbryid is None:
|
||||
self.lbryid = generate_id()
|
||||
@@ -124,7 +161,7 @@ class LBRYSession(object):
|
||||
d.addCallback(lambda _: self._setup_dht())
|
||||
else:
|
||||
if self.hash_announcer is None and self.peer_port is not None:
|
||||
logging.warning("The server has no way to advertise its available blobs.")
|
||||
log.warning("The server has no way to advertise its available blobs.")
|
||||
self.hash_announcer = DummyHashAnnouncer()
|
||||
|
||||
d.addCallback(lambda _: self._setup_other_components())
|
||||
@@ -132,44 +169,78 @@ class LBRYSession(object):
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop all services"""
|
||||
log.info('Shutting down %s', self)
|
||||
ds = []
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_tracker.stop))
|
||||
if self.dht_node is not None:
|
||||
ds.append(defer.maybeDeferred(self.dht_node.stop))
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.upnp_redirects_set is True:
|
||||
if self.rate_limiter is not None:
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
if self.peer_finder is not None:
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
if self.hash_announcer is not None:
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
if self.wallet is not None:
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.use_upnp is True:
|
||||
ds.append(defer.maybeDeferred(self._unset_upnp))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _try_upnp(self):
|
||||
|
||||
logging.debug("In _try_upnp")
|
||||
log.debug("In _try_upnp")
|
||||
|
||||
def threaded_try_upnp():
|
||||
if self.use_upnp is False:
|
||||
logging.debug("Not using upnp")
|
||||
log.debug("Not using upnp")
|
||||
return False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
self.upnp_handler = u
|
||||
u.selectigd()
|
||||
external_ip = u.externalipaddress()
|
||||
if external_ip != '0.0.0.0':
|
||||
self.external_ip = external_ip
|
||||
if self.peer_port is not None:
|
||||
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
|
||||
if u.getspecificportmapping(self.peer_port, 'TCP') is None:
|
||||
u.addportmapping(
|
||||
self.peer_port, 'TCP', u.lanaddr, self.peer_port,
|
||||
'LBRY peer port', '')
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
log.info("Set UPnP redirect for TCP port %d", self.peer_port)
|
||||
else:
|
||||
# see comment below
|
||||
log.warning("UPnP redirect already set for TCP port %d", self.peer_port)
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
if self.dht_node_port is not None:
|
||||
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
|
||||
self.upnp_redirects_set = True
|
||||
if u.getspecificportmapping(self.dht_node_port, 'UDP') is None:
|
||||
u.addportmapping(
|
||||
self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port,
|
||||
'LBRY DHT port', '')
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
log.info("Set UPnP redirect for UPD port %d", self.dht_node_port)
|
||||
else:
|
||||
# TODO: check that the existing redirect was
|
||||
# put up by an old lbrynet session before
|
||||
# grabbing it if such a disconnected redirect
|
||||
# exists, then upnp won't work unless the
|
||||
# redirect is appended or is torn down and set
|
||||
# back up. a bad shutdown of lbrynet could
|
||||
# leave such a redirect up and cause problems
|
||||
# on the next start. this could be
|
||||
# problematic if a previous lbrynet session
|
||||
# didn't make the redirect, and it was made by
|
||||
# another application
|
||||
log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port)
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
return True
|
||||
return False
|
||||
|
||||
def upnp_failed(err):
|
||||
logging.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
return False
|
||||
|
||||
d = threads.deferToThread(threaded_try_upnp)
|
||||
@@ -180,7 +251,7 @@ class LBRYSession(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("Starting the dht")
|
||||
log.debug("Starting the dht")
|
||||
|
||||
def match_port(h, p):
|
||||
return h, p
|
||||
@@ -204,8 +275,11 @@ class LBRYSession(object):
|
||||
d.addCallback(match_port, port)
|
||||
ds.append(d)
|
||||
|
||||
self.dht_node = node.Node(udpPort=self.dht_node_port, lbryid=self.lbryid,
|
||||
externalIP=self.external_ip)
|
||||
self.dht_node = self.dht_node_class(
|
||||
udpPort=self.dht_node_port,
|
||||
lbryid=self.lbryid,
|
||||
externalIP=self.external_ip
|
||||
)
|
||||
self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager)
|
||||
if self.hash_announcer is None:
|
||||
self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port)
|
||||
@@ -216,7 +290,7 @@ class LBRYSession(object):
|
||||
return dl
|
||||
|
||||
def _setup_other_components(self):
|
||||
logging.debug("Setting up the rest of the components")
|
||||
log.debug("Setting up the rest of the components")
|
||||
|
||||
if self.rate_limiter is None:
|
||||
self.rate_limiter = RateLimiter()
|
||||
@@ -225,21 +299,51 @@ class LBRYSession(object):
|
||||
if self.blob_dir is None:
|
||||
self.blob_manager = TempBlobManager(self.hash_announcer)
|
||||
else:
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer, self.blob_dir, self.db_dir)
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer,
|
||||
self.blob_dir,
|
||||
self.db_dir)
|
||||
|
||||
self.rate_limiter.tick()
|
||||
if self.blob_tracker is None:
|
||||
self.blob_tracker = self.blob_tracker_class(self.blob_manager,
|
||||
self.peer_finder,
|
||||
self.dht_node)
|
||||
if self.payment_rate_manager is None:
|
||||
self.payment_rate_manager = self.payment_rate_manager_class(
|
||||
self.base_payment_rate_manager,
|
||||
self.blob_tracker,
|
||||
self.is_generous)
|
||||
|
||||
self.rate_limiter.start()
|
||||
d1 = self.blob_manager.setup()
|
||||
d2 = self.wallet.start()
|
||||
return defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
|
||||
dl.addCallback(lambda _: self.blob_tracker.start())
|
||||
|
||||
dl.addErrback(self._subfailure)
|
||||
return dl
|
||||
|
||||
def _unset_upnp(self):
|
||||
|
||||
log.info("Unsetting upnp for %s", self)
|
||||
def threaded_unset_upnp():
|
||||
u = self.upnp_handler
|
||||
if self.peer_port is not None:
|
||||
u.deleteportmapping(self.peer_port, 'TCP')
|
||||
if self.dht_node_port is not None:
|
||||
u.deleteportmapping(self.dht_node_port, 'UDP')
|
||||
self.upnp_redirects_set = False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
for port, protocol in self.upnp_redirects:
|
||||
if u.getspecificportmapping(port, protocol) is None:
|
||||
log.warning(
|
||||
"UPnP redirect for %s %d was removed by something else.",
|
||||
protocol, port)
|
||||
else:
|
||||
u.deleteportmapping(port, protocol)
|
||||
log.info("Removed UPnP redirect for %s %d.", protocol, port)
|
||||
self.upnp_redirects = []
|
||||
|
||||
return threads.deferToThread(threaded_unset_upnp)
|
||||
d = threads.deferToThread(threaded_unset_upnp)
|
||||
d.addErrback(lambda err: str(err))
|
||||
return d
|
||||
|
||||
def _subfailure(self, err):
|
||||
log.error(err.getTraceback())
|
||||
return err.value
|
||||
@@ -0,0 +1,130 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.interfaces import INegotiationStrategy
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice
|
||||
|
||||
|
||||
def get_default_strategy(blob_tracker, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
|
||||
|
||||
|
||||
class Strategy(object):
|
||||
"""
|
||||
Base for negotiation strategies
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous=settings.is_generous_host):
|
||||
self.price_model = price_model
|
||||
self.is_generous = is_generous
|
||||
self.accepted_offers = {}
|
||||
self.pending_sent_offers = {}
|
||||
self.offers_sent = {}
|
||||
self.offers_received = {}
|
||||
self.max_rate = max_rate or Decimal(self.price_model.base_price * 50)
|
||||
self.min_rate = Decimal(min_rate)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def make_offer(self, peer, blobs):
|
||||
offer_count = self.offers_sent.get(peer, 0)
|
||||
self._add_offer_sent(peer)
|
||||
if peer in self.accepted_offers:
|
||||
# if there was a previous accepted offer, use that
|
||||
offer = self.accepted_offers[peer]
|
||||
if peer in self.pending_sent_offers:
|
||||
del self.pending_sent_offers[peer]
|
||||
elif offer_count == 0 and self.is_generous:
|
||||
# Try asking for it for free
|
||||
offer = Offer(Decimal(0.0))
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
else:
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._make_rate_offer(rates, offer_count)
|
||||
offer = Offer(price)
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
return offer
|
||||
|
||||
def respond_to_offer(self, offer, peer, blobs):
|
||||
offer_count = self.offers_received.get(peer, 0)
|
||||
self._add_offer_received(peer)
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._get_response_rate(rates, offer_count)
|
||||
if peer in self.accepted_offers:
|
||||
offer = self.accepted_offers[peer]
|
||||
elif offer.rate == 0.0 and offer_count == 0 and self.is_generous:
|
||||
# give blobs away for free by default on the first request
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
elif offer.rate >= price:
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
else:
|
||||
offer.reject()
|
||||
if peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
return offer
|
||||
|
||||
def update_accepted_offers(self, peer, offer):
|
||||
if not offer.is_accepted and peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
if offer.is_accepted:
|
||||
self.accepted_offers.update({peer: offer})
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
|
||||
def _add_offer_sent(self, peer):
|
||||
turn = self.offers_sent.get(peer, 0) + 1
|
||||
self.offers_sent.update({peer: turn})
|
||||
|
||||
def _add_offer_received(self, peer):
|
||||
turn = self.offers_received.get(peer, 0) + 1
|
||||
self.offers_received.update({peer: turn})
|
||||
|
||||
def _bounded_price(self, price):
|
||||
price_for_return = Decimal(min(self.max_rate, max(price, self.min_rate)))
|
||||
return price_for_return
|
||||
|
||||
|
||||
class BasicAvailabilityWeightedStrategy(Strategy):
|
||||
"""
|
||||
Basic strategy to target blob prices based on supply relative to mean supply
|
||||
|
||||
Discount price target with each incoming request, and raise it with each outgoing from the modeled price
|
||||
until the rate is accepted or a threshold is reached
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, blob_tracker, acceleration=1.25, deceleration=0.9, max_rate=None, min_rate=0.0,
|
||||
is_generous=settings.is_generous_host, base_price=0.0001, alpha=1.0):
|
||||
price_model = MeanAvailabilityWeightedPrice(blob_tracker, base_price=base_price, alpha=alpha)
|
||||
Strategy.__init__(self, price_model, max_rate, min_rate, is_generous)
|
||||
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
|
||||
self._deceleration = Decimal(deceleration)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
mean_rate = Decimal(sum(rates)) / Decimal(max(len(rates), 1))
|
||||
return mean_rate
|
||||
|
||||
def _premium(self, rate, turn):
|
||||
return rate * (self._acceleration ** Decimal(turn))
|
||||
|
||||
def _discount(self, rate, turn):
|
||||
return rate * (self._deceleration ** Decimal(turn))
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
discounted = self._discount(rate, offer_count)
|
||||
rounded_price = round(discounted, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
with_premium = self._premium(rate, offer_count)
|
||||
rounded_price = round(with_premium, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
@@ -3,6 +3,9 @@ from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamCreator(object):
|
||||
"""Classes which derive from this class create a 'stream', which can be any
|
||||
collection of associated blobs and associated metadata. These classes
|
||||
@@ -42,7 +45,7 @@ class StreamCreator(object):
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
logging.debug("stop has been called for StreamCreator")
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
current_blob = self.current_blob
|
||||
@@ -62,12 +65,10 @@ class StreamCreator(object):
|
||||
pass
|
||||
|
||||
def write(self, data):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self._write(data)
|
||||
if self.stopped is False and self.streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def _write(self, data):
|
||||
pass
|
||||
pass
|
||||
@@ -3,6 +3,10 @@ import json
|
||||
import logging
|
||||
from twisted.internet import threads
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamDescriptorReader(object):
|
||||
@@ -81,7 +85,7 @@ class PlainStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
|
||||
def write_file():
|
||||
logging.debug("Writing the sd file to disk")
|
||||
log.debug("Writing the sd file to disk")
|
||||
with open(self.sd_file_name, 'w') as sd_file:
|
||||
sd_file.write(raw_data)
|
||||
return self.sd_file_name
|
||||
@@ -96,13 +100,26 @@ class BlobStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
logging.debug("Creating the new blob for the stream descriptor")
|
||||
log.debug("Creating the new blob for the stream descriptor")
|
||||
blob_creator = self.blob_manager.get_blob_creator()
|
||||
blob_creator.write(raw_data)
|
||||
logging.debug("Wrote the data to the new blob")
|
||||
log.debug("Wrote the data to the new blob")
|
||||
return blob_creator.close()
|
||||
|
||||
|
||||
class StreamMetadata(object):
|
||||
FROM_BLOB = 1
|
||||
FROM_PLAIN = 2
|
||||
|
||||
def __init__(self, validator, options, factories):
|
||||
self.validator = validator
|
||||
self.options = options
|
||||
self.factories = factories
|
||||
self.metadata_source = None
|
||||
self.source_blob_hash = None
|
||||
self.source_file = None
|
||||
|
||||
|
||||
class StreamDescriptorIdentifier(object):
|
||||
"""Tries to determine the type of stream described by the stream descriptor using the
|
||||
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
|
||||
@@ -110,9 +127,10 @@ class StreamDescriptorIdentifier(object):
|
||||
"""
|
||||
def __init__(self):
|
||||
self._sd_info_validators = {} # {stream_type: IStreamDescriptorValidator}
|
||||
self._stream_options = {} # {stream_type: IStreamOptions}
|
||||
self._stream_downloader_factories = defaultdict(list) # {stream_type: [IStreamDownloaderFactory]}
|
||||
|
||||
def add_stream_info_validator(self, stream_type, sd_info_validator):
|
||||
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
|
||||
"""
|
||||
This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
|
||||
@@ -126,9 +144,13 @@ class StreamDescriptorIdentifier(object):
|
||||
will then be called. If the validation step fails, an exception will be thrown, preventing the stream
|
||||
descriptor from being further processed.
|
||||
|
||||
@param stream_options: A class implementing the IStreamOptions interface. This class's constructor will be
|
||||
passed the sd_info_validator object containing the raw metadata from the stream descriptor file.
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self._sd_info_validators[stream_type] = sd_info_validator
|
||||
self._stream_options[stream_type] = stream_options
|
||||
|
||||
def add_stream_downloader_factory(self, stream_type, factory):
|
||||
"""
|
||||
@@ -146,35 +168,59 @@ class StreamDescriptorIdentifier(object):
|
||||
"""
|
||||
self._stream_downloader_factories[stream_type].append(factory)
|
||||
|
||||
def get_info_and_factories_for_sd_file(self, sd_path):
|
||||
def _return_metadata(self, options_validator_factories, source_type, source):
|
||||
validator, options, factories = options_validator_factories
|
||||
m = StreamMetadata(validator, options, factories)
|
||||
m.metadata_source = source_type
|
||||
if source_type == StreamMetadata.FROM_BLOB:
|
||||
m.source_blob_hash = source
|
||||
if source_type == StreamMetadata.FROM_PLAIN:
|
||||
m.source_file = source
|
||||
return m
|
||||
|
||||
def get_metadata_for_sd_file(self, sd_path):
|
||||
sd_reader = PlainStreamDescriptorReader(sd_path)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
|
||||
return d
|
||||
|
||||
def get_info_and_factories_for_sd_blob(self, sd_blob):
|
||||
def get_metadata_for_sd_blob(self, sd_blob):
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
|
||||
return d
|
||||
|
||||
def _get_factories(self, stream_type):
|
||||
assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_downloader_factories[stream_type]
|
||||
|
||||
def _get_validator(self, stream_type):
|
||||
assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._sd_info_validators:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._sd_info_validators[stream_type]
|
||||
|
||||
def _return_info_and_factories(self, sd_info):
|
||||
assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.'
|
||||
def _get_options(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_options[stream_type]
|
||||
|
||||
def _return_options_and_validator_and_factories(self, sd_info):
|
||||
if not 'stream_type' in sd_info:
|
||||
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
|
||||
stream_type = sd_info['stream_type']
|
||||
factories = self._get_factories(stream_type)
|
||||
validator = self._get_validator(stream_type)(sd_info)
|
||||
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
|
||||
|
||||
d = validator.validate()
|
||||
|
||||
d.addCallback(lambda _: (validator, factories))
|
||||
def get_options():
|
||||
options = self._get_options(stream_type)
|
||||
return validator, options, factories
|
||||
|
||||
d.addCallback(lambda _: get_options())
|
||||
return d
|
||||
|
||||
|
||||
@@ -192,4 +238,4 @@ def download_sd_blob(session, blob_hash, payment_rate_manager):
|
||||
"""
|
||||
downloader = StandaloneBlobDownloader(blob_hash, session.blob_manager, session.peer_finder,
|
||||
session.rate_limiter, payment_rate_manager, session.wallet)
|
||||
return downloader.download()
|
||||
return downloader.download()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,12 +1,41 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from decimal import Decimal
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
|
||||
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
|
||||
from lbrynet.interfaces import IRequestCreator
|
||||
from lbrynet.core.Offer import Offer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_points(num_bytes, rate):
|
||||
if isinstance(rate, float):
|
||||
return 1.0 * num_bytes * rate / 2**20
|
||||
elif isinstance(rate, Decimal):
|
||||
return 1.0 * num_bytes * float(rate) / 2**20
|
||||
else:
|
||||
raise Exception("Unknown rate type")
|
||||
|
||||
|
||||
def cache(fn):
|
||||
"""Caches the function call for each instance"""
|
||||
attr = '__{}_value'.format(fn.__name__)
|
||||
|
||||
def helper(self):
|
||||
if not hasattr(self, attr):
|
||||
value = fn(self)
|
||||
setattr(self, attr, value)
|
||||
return getattr(self, attr)
|
||||
return helper
|
||||
|
||||
|
||||
class BlobRequester(object):
|
||||
@@ -22,59 +51,18 @@ class BlobRequester(object):
|
||||
self._available_blobs = defaultdict(list) # {Peer: [blob_hash]}
|
||||
self._unavailable_blobs = defaultdict(list) # {Peer: [blob_hash]}}
|
||||
self._protocol_prices = {} # {ClientProtocol: price}
|
||||
self._protocol_offers = {}
|
||||
self._price_disagreements = [] # [Peer]
|
||||
self._protocol_tries = {}
|
||||
self._maxed_out_peers = []
|
||||
self._incompatible_peers = []
|
||||
|
||||
######## IRequestCreator #########
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
sent_request = False
|
||||
if self._blobs_to_download() and self._should_send_request_to(peer):
|
||||
a_r = self._get_availability_request(peer)
|
||||
d_r = self._get_download_request(peer)
|
||||
p_r = None
|
||||
|
||||
if a_r is not None or d_r is not None:
|
||||
p_r = self._get_price_request(peer, protocol)
|
||||
|
||||
if a_r is not None:
|
||||
d1 = protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, peer, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request", peer)
|
||||
sent_request = True
|
||||
if d_r is not None:
|
||||
reserved_points = self._reserve_points(peer, protocol, d_r.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
d_r.finished_deferred.addCallbacks(self._download_succeeded, self._download_failed,
|
||||
callbackArgs=(peer, d_r.blob),
|
||||
errbackArgs=(peer,))
|
||||
d_r.finished_deferred.addBoth(self._pay_or_cancel_payment, protocol, reserved_points, d_r.blob)
|
||||
d_r.finished_deferred.addErrback(self._handle_download_error, peer, d_r.blob)
|
||||
|
||||
d2 = protocol.add_blob_request(d_r)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d2.addCallback(self._handle_incoming_blob, peer, d_r)
|
||||
d2.addErrback(self._request_failed, "download request", peer)
|
||||
|
||||
sent_request = True
|
||||
else:
|
||||
d_r.cancel(InsufficientFundsError())
|
||||
return defer.fail(InsufficientFundsError())
|
||||
if sent_request is True:
|
||||
if p_r is not None:
|
||||
d3 = protocol.add_request(p_r)
|
||||
d3.addCallback(self._handle_price_response, peer, p_r, protocol)
|
||||
d3.addErrback(self._request_failed, "price request", peer)
|
||||
return defer.succeed(sent_request)
|
||||
"""Makes an availability request, download request and price request"""
|
||||
if not self.should_send_next_request(peer):
|
||||
return defer.succeed(False)
|
||||
return self._send_next_request(peer, protocol)
|
||||
|
||||
def get_new_peers(self):
|
||||
d = self._get_hash_for_peer_search()
|
||||
@@ -82,36 +70,35 @@ class BlobRequester(object):
|
||||
return d
|
||||
|
||||
######### internal calls #########
|
||||
def should_send_next_request(self, peer):
|
||||
return (
|
||||
self._blobs_to_download() and
|
||||
self._should_send_request_to(peer)
|
||||
)
|
||||
|
||||
def _download_succeeded(self, arg, peer, blob):
|
||||
logging.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
|
||||
self._update_local_score(peer, 5.0)
|
||||
peer.update_stats('blobs_downloaded', 1)
|
||||
peer.update_score(5.0)
|
||||
self.blob_manager.blob_completed(blob)
|
||||
return arg
|
||||
def _send_next_request(self, peer, protocol):
|
||||
log.debug('Sending a blob request for %s and %s', peer, protocol)
|
||||
availability = AvailabilityRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
download = DownloadRequest(self, peer, protocol, self.payment_rate_manager, self.wallet)
|
||||
price = PriceRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
|
||||
def _download_failed(self, reason, peer):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self._update_local_score(peer, -10.0)
|
||||
return reason
|
||||
sent_request = False
|
||||
if availability.can_make_request():
|
||||
availability.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if price.can_make_request():
|
||||
# TODO: document why a PriceRequest is only made if an
|
||||
# Availability or Download request was made
|
||||
price.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if download.can_make_request():
|
||||
try:
|
||||
download.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
except InsufficientFundsError as err:
|
||||
return defer.fail(err)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, protocol, reserved_points, blob):
|
||||
if blob.length != 0 and (not isinstance(arg, Failure) or arg.check(DownloadCanceledError)):
|
||||
self._pay_peer(protocol, blob.length, reserved_points)
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _handle_download_error(self, err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
logging.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
return defer.succeed(sent_request)
|
||||
|
||||
def _get_hash_for_peer_search(self):
|
||||
r = None
|
||||
@@ -123,7 +110,7 @@ class BlobRequester(object):
|
||||
else:
|
||||
blob_hash = blobs_without_sources[0].blob_hash
|
||||
r = blob_hash
|
||||
logging.debug("Blob requester peer search response: %s", str(r))
|
||||
log.debug("Blob requester peer search response: %s", str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -134,9 +121,17 @@ class BlobRequester(object):
|
||||
|
||||
def choose_best_peers(peers):
|
||||
bad_peers = self._get_bad_peers()
|
||||
return [p for p in peers if not p in bad_peers]
|
||||
without_bad_peers = [p for p in peers if not p in bad_peers]
|
||||
without_maxed_out_peers = [p for p in without_bad_peers if p not in self._maxed_out_peers]
|
||||
return without_maxed_out_peers
|
||||
|
||||
d.addCallback(choose_best_peers)
|
||||
|
||||
def lookup_failed(err):
|
||||
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
|
||||
return []
|
||||
|
||||
d.addErrback(lookup_failed)
|
||||
return d
|
||||
|
||||
def _should_send_request_to(self, peer):
|
||||
@@ -167,141 +162,407 @@ class BlobRequester(object):
|
||||
return sorted(needed_blobs, key=lambda b: b.is_downloading())
|
||||
|
||||
def _blobs_without_sources(self):
|
||||
return [b for b in self.download_manager.needed_blobs() if not self._hash_available(b.blob_hash)]
|
||||
|
||||
def _get_availability_request(self, peer):
|
||||
all_needed = [b.blob_hash for b in self._blobs_to_download() if not b.blob_hash in self._available_blobs[peer]]
|
||||
# sort them so that the peer will be asked first for blobs it hasn't said it doesn't have
|
||||
to_request = sorted(all_needed, key=lambda b: b in self._unavailable_blobs[peer])[:20]
|
||||
if to_request:
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
return None
|
||||
|
||||
def _get_download_request(self, peer):
|
||||
request = None
|
||||
to_download = [b for b in self._blobs_to_download() if self._hash_available_on(b.blob_hash, peer)]
|
||||
while to_download and request is None:
|
||||
blob_to_download = to_download[0]
|
||||
to_download = to_download[1:]
|
||||
if not blob_to_download.is_validated():
|
||||
d, write_func, cancel_func = blob_to_download.open_for_writing(peer)
|
||||
|
||||
def counting_write_func(data):
|
||||
peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return write_func(data)
|
||||
|
||||
if d is not None:
|
||||
|
||||
request_dict = {'requested_blob': blob_to_download.blob_hash}
|
||||
response_identifier = 'incoming_blob'
|
||||
|
||||
request = ClientBlobRequest(request_dict, response_identifier, counting_write_func, d,
|
||||
cancel_func, blob_to_download)
|
||||
|
||||
logging.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
|
||||
return request
|
||||
return [
|
||||
b for b in self.download_manager.needed_blobs()
|
||||
if not self._hash_available(b.blob_hash)
|
||||
]
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
if protocol in self._protocol_prices:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_price_request(self, peer, protocol):
|
||||
request = None
|
||||
if not protocol in self._protocol_prices:
|
||||
self._protocol_prices[protocol] = self.payment_rate_manager.get_rate_blob_data(peer)
|
||||
request_dict = {'blob_data_payment_rate': self._protocol_prices[protocol]}
|
||||
request = ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
|
||||
|
||||
class RequestHelper(object):
|
||||
def __init__(self, requestor, peer, protocol, payment_rate_manager):
|
||||
self.requestor = requestor
|
||||
self.peer = peer
|
||||
self.protocol = protocol
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
|
||||
@property
|
||||
def protocol_prices(self):
|
||||
return self.requestor._protocol_prices
|
||||
|
||||
@property
|
||||
def protocol_offers(self):
|
||||
return self.requestor._protocol_offers
|
||||
|
||||
@property
|
||||
def available_blobs(self):
|
||||
return self.requestor._available_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def unavailable_blobs(self):
|
||||
return self.requestor._unavailable_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def maxed_out_peers(self):
|
||||
return self.requestor._maxed_out_peers
|
||||
|
||||
def update_local_score(self, score):
|
||||
self.requestor._update_local_score(self.peer, score)
|
||||
|
||||
def _request_failed(self, reason, request_type):
|
||||
if reason.check(RequestCanceledError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self.requestor._incompatible_peers.append(self.peer)
|
||||
log.warning("A request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
request_type, reason.getErrorMessage(), reason.type)
|
||||
self.update_local_score(-10.0)
|
||||
if isinstance(reason, InvalidResponseError) or isinstance(reason, NoResponseError):
|
||||
self.peer.update_score(-10.0)
|
||||
else:
|
||||
self.peer.update_score(-2.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
|
||||
def get_and_save_rate(self):
|
||||
if self.payment_rate_manager.price_limit_reached(self.peer):
|
||||
if self.peer not in self.maxed_out_peers:
|
||||
self.maxed_out_peers.append(self.peer)
|
||||
return None
|
||||
rate = self.protocol_prices.get(self.protocol)
|
||||
if rate is None:
|
||||
if self.peer in self.payment_rate_manager.strategy.pending_sent_offers:
|
||||
pending = self.payment_rate_manager.strategy.pending_sent_offers[self.peer]
|
||||
if not pending.is_too_low and not pending.is_accepted:
|
||||
return pending.rate
|
||||
rate = self.payment_rate_manager.get_rate_blob_data(self.peer, self.available_blobs)
|
||||
self.protocol_offers[self.protocol] = rate
|
||||
return rate
|
||||
|
||||
|
||||
def _handle_incoming_blob(response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not type(response_dict[request.response_identifier]) == dict:
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
type(response_dict[request.response_identifier]))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if not 'blob_hash' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError(
|
||||
"Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob'])
|
||||
)
|
||||
if not 'length' in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
|
||||
|
||||
def _handle_download_error(err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
log.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
|
||||
|
||||
class AvailabilityRequest(RequestHelper):
|
||||
"""Ask a peer what blobs it has available.
|
||||
|
||||
Results are saved in `_available_blobs` and `_unavailable_blobs`
|
||||
on the parent BlobRequester.
|
||||
"""
|
||||
def can_make_request(self):
|
||||
return self.get_top_needed_blobs()
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
to_request = self.get_top_needed_blobs()
|
||||
if not to_request:
|
||||
raise Exception('Unable to make a request without available blobs')
|
||||
return self._make_request(to_request)
|
||||
|
||||
@cache
|
||||
def get_top_needed_blobs(self, limit=20):
|
||||
all_needed = [
|
||||
b.blob_hash for b in self.requestor._blobs_to_download()
|
||||
if not self.is_available(b)
|
||||
]
|
||||
# sort them so that the peer will be asked first for blobs it
|
||||
# hasn't said it doesn't have
|
||||
sorted_needed = sorted(
|
||||
all_needed,
|
||||
key=lambda b: b in self.unavailable_blobs
|
||||
)
|
||||
return sorted_needed[:limit]
|
||||
|
||||
def is_available(self, blob):
|
||||
return blob.blob_hash in self.available_blobs
|
||||
|
||||
def _make_request(self, to_request):
|
||||
log.debug('Requesting blobs: %s', to_request)
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
def _handle_request(self, a_r):
|
||||
log.debug('making an availability request')
|
||||
d1 = self.protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request")
|
||||
|
||||
def _reserve_points(self, peer, protocol, max_bytes):
|
||||
assert protocol in self._protocol_prices
|
||||
points_to_reserve = 1.0 * max_bytes * self._protocol_prices[protocol] / 2**20
|
||||
return self.wallet.reserve_points(peer, points_to_reserve)
|
||||
def _handle_availability(self, response_dict, request):
|
||||
assert request.response_identifier == 'available_blobs'
|
||||
if 'available_blobs' not in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
log.debug("Received a response to the availability request")
|
||||
# save available blobs
|
||||
blob_hashes = response_dict['available_blobs']
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
self.process_available_blob_hash(blob_hash, request)
|
||||
# everything left in the request is missing
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self.unavailable_blobs.append(blob_hash)
|
||||
return True
|
||||
|
||||
def _pay_peer(self, protocol, num_bytes, reserved_points):
|
||||
def process_available_blob_hash(self, blob_hash, request):
|
||||
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self.available_blobs.append(blob_hash)
|
||||
self.remove_from_unavailable_blobs(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
|
||||
def remove_from_unavailable_blobs(self, blob_hash):
|
||||
if blob_hash in self.unavailable_blobs:
|
||||
self.unavailable_blobs.remove(blob_hash)
|
||||
|
||||
|
||||
class PriceRequest(RequestHelper):
|
||||
"""Ask a peer if a certain price is acceptable"""
|
||||
def can_make_request(self):
|
||||
if len(self.available_blobs) and not self.protocol in self.protocol_prices:
|
||||
return self.get_and_save_rate() is not None
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_price_request()
|
||||
self._handle_price_request(request)
|
||||
|
||||
def _get_price_request(self):
|
||||
rate = self.get_and_save_rate()
|
||||
if rate is None:
|
||||
log.debug("No blobs to request from %s", self.peer)
|
||||
raise Exception('Cannot make a price request without a payment rate')
|
||||
log.debug("Offer rate %s to %s for %i blobs", rate, self.peer, len(self.available_blobs))
|
||||
|
||||
request_dict = {'blob_data_payment_rate': rate}
|
||||
return ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
|
||||
def _handle_price_request(self, price_request):
|
||||
d = self.protocol.add_request(price_request)
|
||||
d.addCallback(self._handle_price_response, price_request)
|
||||
d.addErrback(self._request_failed, "price request")
|
||||
|
||||
def _handle_price_response(self, response_dict, request):
|
||||
assert request.response_identifier == 'blob_data_payment_rate'
|
||||
if 'blob_data_payment_rate' not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
assert self.protocol in self.protocol_offers
|
||||
offer = Offer(self.protocol_offers[self.protocol])
|
||||
offer.handle(response_dict['blob_data_payment_rate'])
|
||||
self.payment_rate_manager.record_offer_reply(self.peer, offer)
|
||||
if offer.is_accepted:
|
||||
log.info("Offered rate %f/mb accepted by %s", offer.rate, self.peer.host)
|
||||
self.protocol_prices[self.protocol] = offer.rate
|
||||
return True
|
||||
elif offer.is_too_low:
|
||||
log.debug("Offered rate %f/mb rejected by %s", offer.rate, self.peer.host)
|
||||
return not self.payment_rate_manager.price_limit_reached(self.peer)
|
||||
else:
|
||||
log.warning("Price disagreement")
|
||||
self.requestor._price_disagreements.append(self.peer)
|
||||
return False
|
||||
|
||||
|
||||
class DownloadRequest(RequestHelper):
|
||||
"""Choose a blob and download it from a peer and also pay the peer for the data."""
|
||||
def __init__(self, requester, peer, protocol, payment_rate_manager, wallet):
|
||||
RequestHelper.__init__(self, requester, peer, protocol, payment_rate_manager)
|
||||
self.wallet = wallet
|
||||
|
||||
def can_make_request(self):
|
||||
if self.protocol in self.protocol_prices:
|
||||
return self.get_blob_details()
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_download_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
blob_details = self.get_blob_details()
|
||||
if not blob_details:
|
||||
raise Exception('No blobs available to download')
|
||||
return self._make_request(blob_details)
|
||||
|
||||
@cache
|
||||
def get_blob_details(self):
|
||||
"""Open a blob for writing and return the details.
|
||||
|
||||
If no blob can be opened, returns None.
|
||||
"""
|
||||
to_download = self.get_available_blobs()
|
||||
return self.find_blob(to_download)
|
||||
|
||||
def get_available_blobs(self):
|
||||
available_blobs = [
|
||||
b for b in self.requestor._blobs_to_download()
|
||||
if self.requestor._hash_available_on(b.blob_hash, self.peer)
|
||||
]
|
||||
log.debug('available blobs: %s', available_blobs)
|
||||
return available_blobs
|
||||
|
||||
def find_blob(self, to_download):
|
||||
"""Return the first blob in `to_download` that is successfully opened for write."""
|
||||
for blob in to_download:
|
||||
if blob.is_validated():
|
||||
log.debug('Skipping blob %s as its already validated', blob)
|
||||
continue
|
||||
d, write_func, cancel_func = blob.open_for_writing(self.peer)
|
||||
if d is not None:
|
||||
return BlobDownloadDetails(blob, d, write_func, cancel_func, self.peer)
|
||||
log.debug('Skipping blob %s as there was an issue opening it for writing', blob)
|
||||
return None
|
||||
|
||||
def _make_request(self, blob_details):
|
||||
blob = blob_details.blob
|
||||
request = ClientBlobRequest(
|
||||
{'requested_blob': blob.blob_hash},
|
||||
'incoming_blob',
|
||||
blob_details.counting_write_func,
|
||||
blob_details.deferred,
|
||||
blob_details.cancel_func,
|
||||
blob
|
||||
)
|
||||
log.info("Requesting blob %s from %s", blob.blob_hash, self.peer)
|
||||
return request
|
||||
|
||||
def _handle_download_request(self, client_blob_request):
|
||||
reserved_points = self.reserve_funds_or_cancel(client_blob_request)
|
||||
self.add_callbacks_to_download_request(client_blob_request, reserved_points)
|
||||
self.create_add_blob_request(client_blob_request)
|
||||
|
||||
def reserve_funds_or_cancel(self, client_blob_request):
|
||||
reserved_points = self._reserve_points(client_blob_request.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
return reserved_points
|
||||
client_blob_request.cancel(InsufficientFundsError())
|
||||
client_blob_request.finished_deferred.addErrback(lambda _: True)
|
||||
raise InsufficientFundsError()
|
||||
|
||||
def add_callbacks_to_download_request(self, client_blob_request, reserved_points):
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
client_blob_request.finished_deferred.addCallbacks(
|
||||
self._download_succeeded,
|
||||
self._download_failed,
|
||||
callbackArgs=(client_blob_request.blob,),
|
||||
)
|
||||
client_blob_request.finished_deferred.addBoth(
|
||||
self._pay_or_cancel_payment, reserved_points, client_blob_request.blob)
|
||||
client_blob_request.finished_deferred.addErrback(
|
||||
_handle_download_error, self.peer, client_blob_request.blob)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, reserved_points, blob):
|
||||
if self._can_pay_peer(blob, arg):
|
||||
self._pay_peer(blob.length, reserved_points)
|
||||
d = self.requestor.blob_manager.add_blob_to_download_history(str(blob),
|
||||
str(self.peer.host),
|
||||
float(self.protocol_prices[self.protocol]))
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _can_pay_peer(self, blob, arg):
|
||||
return (
|
||||
blob.length != 0 and
|
||||
(not isinstance(arg, Failure) or arg.check(DownloadCanceledError))
|
||||
)
|
||||
|
||||
def _pay_peer(self, num_bytes, reserved_points):
|
||||
assert num_bytes != 0
|
||||
assert protocol in self._protocol_prices
|
||||
point_amount = 1.0 * num_bytes * self._protocol_prices[protocol] / 2**20
|
||||
rate = self.get_and_save_rate()
|
||||
point_amount = get_points(num_bytes, rate)
|
||||
self.wallet.send_points(reserved_points, point_amount)
|
||||
self.payment_rate_manager.record_points_paid(point_amount)
|
||||
|
||||
def _cancel_points(self, reserved_points):
|
||||
self.wallet.cancel_point_reservation(reserved_points)
|
||||
|
||||
def _handle_availability(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
logging.debug("Received a response to the availability request")
|
||||
blob_hashes = response_dict[request.response_identifier]
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
logging.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self._available_blobs[peer].append(blob_hash)
|
||||
if blob_hash in self._unavailable_blobs[peer]:
|
||||
self._unavailable_blobs[peer].remove(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self._unavailable_blobs[peer].append(blob_hash)
|
||||
return True
|
||||
def create_add_blob_request(self, client_blob_request):
|
||||
d = self.protocol.add_blob_request(client_blob_request)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d.addCallback(_handle_incoming_blob, self.peer, client_blob_request)
|
||||
d.addErrback(self._request_failed, "download request")
|
||||
|
||||
def _handle_incoming_blob(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not type(response_dict[request.response_identifier]) == dict:
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
(type(response_dict[request.response_identifier]),))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if not 'blob_hash' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError("Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob']))
|
||||
if not 'length' in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
def _reserve_points(self, num_bytes):
|
||||
# jobevers: there was an assertion here, but I don't think it
|
||||
# was a valid assertion to make. It is possible for a rate to
|
||||
# not yet been set for this protocol or for it to have been
|
||||
# removed so instead I switched it to check if a rate has been set
|
||||
# and calculate it if it has not
|
||||
rate = self.get_and_save_rate()
|
||||
points_to_reserve = get_points(num_bytes, rate)
|
||||
return self.wallet.reserve_points(self.peer, points_to_reserve)
|
||||
|
||||
def _handle_price_response(self, response_dict, peer, request, protocol):
|
||||
if not request.response_identifier in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
assert protocol in self._protocol_prices
|
||||
response = response_dict[request.response_identifier]
|
||||
if response == "RATE_ACCEPTED":
|
||||
return True
|
||||
else:
|
||||
del self._protocol_prices[protocol]
|
||||
self._price_disagreements.append(peer)
|
||||
return True
|
||||
def _download_succeeded(self, arg, blob):
|
||||
log.info("Blob %s has been successfully downloaded from %s", blob, self.peer)
|
||||
self.update_local_score(5.0)
|
||||
self.peer.update_stats('blobs_downloaded', 1)
|
||||
self.peer.update_score(5.0)
|
||||
self.requestor.blob_manager.blob_completed(blob)
|
||||
return arg
|
||||
|
||||
def _request_failed(self, reason, request_type, peer):
|
||||
if reason.check(RequestCanceledError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
str(request_type), reason.getErrorMessage(), reason.type)
|
||||
self._update_local_score(peer, -10.0)
|
||||
if isinstance(reason, InvalidResponseError):
|
||||
peer.update_score(-10.0)
|
||||
else:
|
||||
peer.update_score(-2.0)
|
||||
return reason
|
||||
def _download_failed(self, reason):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self.update_local_score(-10.0)
|
||||
return reason
|
||||
|
||||
|
||||
class BlobDownloadDetails(object):
|
||||
"""Contains the information needed to make a ClientBlobRequest from an open blob"""
|
||||
def __init__(self, blob, deferred, write_func, cancel_func, peer):
|
||||
self.blob = blob
|
||||
self.deferred = deferred
|
||||
self.write_func = write_func
|
||||
self.cancel_func = cancel_func
|
||||
self.peer = peer
|
||||
|
||||
def counting_write_func(self, data):
|
||||
self.peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return self.write_func(data)
|
||||
@@ -1,9 +1,10 @@
|
||||
import json
|
||||
import logging
|
||||
from twisted.internet import error, defer, reactor
|
||||
from decimal import Decimal
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.python import failure
|
||||
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
|
||||
from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
@@ -11,6 +12,15 @@ from lbrynet.interfaces import IRequestSender, IRateLimited
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_decimal(obj):
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
raise TypeError(repr(obj) + " is not JSON serializable")
|
||||
|
||||
|
||||
class ClientProtocol(Protocol):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
|
||||
@@ -38,8 +48,8 @@ class ClientProtocol(Protocol):
|
||||
self._blob_download_request.write(data)
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > MAX_RESPONSE_SIZE:
|
||||
logging.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
if len(self._response_buff) > settings.MAX_RESPONSE_INFO_SIZE:
|
||||
log.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
response, extra_data = self._get_valid_response(self._response_buff)
|
||||
if response is not None:
|
||||
@@ -55,7 +65,7 @@ class ClientProtocol(Protocol):
|
||||
else:
|
||||
err = reason
|
||||
#if self._response_deferreds:
|
||||
# logging.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
# log.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
for key, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
@@ -70,7 +80,7 @@ class ClientProtocol(Protocol):
|
||||
return defer.fail(failure.Failure(ValueError("There is already a request for that response active")))
|
||||
self._next_request.update(request.request_dict)
|
||||
d = defer.Deferred()
|
||||
logging.debug("Adding a request. Request: %s", str(request))
|
||||
log.debug("Adding a request. Request: %s", str(request.request_dict))
|
||||
self._response_deferreds[request.response_identifier] = d
|
||||
return d
|
||||
|
||||
@@ -102,12 +112,14 @@ class ClientProtocol(Protocol):
|
||||
######### Internal request handling #########
|
||||
|
||||
def _handle_request_error(self, err):
|
||||
logging.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
log.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
self.transport.loseConnection()
|
||||
|
||||
def _ask_for_request(self):
|
||||
|
||||
log.debug("In _ask_for_request")
|
||||
|
||||
if self.connection_closed is True or self.connection_closing is True:
|
||||
return
|
||||
|
||||
@@ -117,7 +129,7 @@ class ClientProtocol(Protocol):
|
||||
self._send_request_message(request_msg)
|
||||
else:
|
||||
# The connection manager has indicated that this connection should be terminated
|
||||
logging.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
log.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
self.transport.loseConnection()
|
||||
|
||||
d = self._connection_manager.get_next_request(self.peer, self)
|
||||
@@ -127,7 +139,7 @@ class ClientProtocol(Protocol):
|
||||
def _send_request_message(self, request_msg):
|
||||
# TODO: compare this message to the last one. If they're the same,
|
||||
# TODO: incrementally delay this message.
|
||||
m = json.dumps(request_msg)
|
||||
m = json.dumps(request_msg, default=encode_decimal)
|
||||
self.transport.write(m)
|
||||
|
||||
def _get_valid_response(self, response_msg):
|
||||
@@ -153,14 +165,15 @@ class ClientProtocol(Protocol):
|
||||
# If an error gets to this point, log it and kill the connection.
|
||||
if not err.check(MisbehavingPeerError, ConnectionClosedBeforeResponseError, DownloadCanceledError,
|
||||
RequestCanceledError):
|
||||
logging.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError):
|
||||
self.transport.loseConnection()
|
||||
log.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError): # The connection manager is closing the connection, so
|
||||
# there's no need to do it here.
|
||||
return err
|
||||
|
||||
def _handle_response(self, response):
|
||||
ds = []
|
||||
logging.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
log.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
for key, val in response.items():
|
||||
if key in self._response_deferreds:
|
||||
d = self._response_deferreds[key]
|
||||
@@ -178,12 +191,29 @@ class ClientProtocol(Protocol):
|
||||
d.addErrback(self._handle_response_error)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
|
||||
dl.addCallback(lambda _: self._ask_for_request())
|
||||
def get_next_request(results):
|
||||
failed = False
|
||||
for success, result in results:
|
||||
if success is False:
|
||||
failed = True
|
||||
if not isinstance(result.value, DownloadCanceledError):
|
||||
log.info(result.value)
|
||||
log.info("The connection is closing due to an error: %s", str(result.getTraceback()))
|
||||
if failed is False:
|
||||
log.debug("Asking for another request.")
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self._ask_for_request)
|
||||
#self._ask_for_request()
|
||||
else:
|
||||
log.debug("Not asking for another request.")
|
||||
self.transport.loseConnection()
|
||||
|
||||
dl.addCallback(get_next_request)
|
||||
|
||||
def _downloading_finished(self, arg):
|
||||
logging.debug("The blob has finished downloading")
|
||||
log.debug("The blob has finished downloading")
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return arg
|
||||
@@ -194,8 +224,8 @@ class ClientProtocol(Protocol):
|
||||
# TODO: always be this way. it's done this way now because the client has no other way
|
||||
# TODO: of telling the server it wants the download to stop. It would be great if the
|
||||
# TODO: protocol had such a mechanism.
|
||||
logging.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
log.debug("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
#self.transport.loseConnection()
|
||||
#return True
|
||||
return err
|
||||
@@ -232,4 +262,4 @@ class ClientProtocolFactory(ClientFactory):
|
||||
p = self.protocol()
|
||||
p.factory = self
|
||||
self.p = p
|
||||
return p
|
||||
return p
|
||||
@@ -1,4 +1,4 @@
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet.conf import settings
|
||||
|
||||
|
||||
class ClientRequest(object):
|
||||
@@ -17,7 +17,7 @@ class ClientBlobRequest(ClientPaidRequest):
|
||||
def __init__(self, request_dict, response_identifier, write_func, finished_deferred,
|
||||
cancel_func, blob):
|
||||
if blob.length is None:
|
||||
max_pay_units = BLOB_SIZE
|
||||
max_pay_units = settings.BLOB_SIZE
|
||||
else:
|
||||
max_pay_units = blob.length
|
||||
ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units)
|
||||
|
||||
@@ -2,54 +2,67 @@ import logging
|
||||
from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
from lbrynet.conf import MAX_CONNECTIONS_PER_STREAM
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnectionHandler(object):
|
||||
def __init__(self, request_creators, factory):
|
||||
self.request_creators = request_creators
|
||||
self.factory = factory
|
||||
self.connection = None
|
||||
|
||||
|
||||
class ConnectionManager(object):
|
||||
implements(interfaces.IConnectionManager)
|
||||
|
||||
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
self.primary_request_creators = primary_request_creators
|
||||
self.secondary_request_creators = secondary_request_creators
|
||||
self.peer_connections = {} # {Peer: {'connection': connection,
|
||||
# 'request_creators': [IRequestCreator if using this connection]}}
|
||||
self.connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self.next_manage_call = None
|
||||
self._primary_request_creators = primary_request_creators
|
||||
self._secondary_request_creators = secondary_request_creators
|
||||
self._peer_connections = {} # {Peer: PeerConnectionHandler}
|
||||
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self._next_manage_call = None
|
||||
self.stopped = True
|
||||
|
||||
def start(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = reactor.callLater(0, self._manage)
|
||||
self.stopped = False
|
||||
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = reactor.callLater(0, self._manage)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
self.stopped = True
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
closing_deferreds = []
|
||||
for peer in self.peer_connections.keys():
|
||||
for peer in self._peer_connections.keys():
|
||||
|
||||
def close_connection(p):
|
||||
logging.info("Abruptly closing a connection to %s due to downloading being paused",
|
||||
str(p))
|
||||
log.info(
|
||||
"Abruptly closing a connection to %s due to downloading being paused", p)
|
||||
|
||||
if self.peer_connections[p]['factory'].p is not None:
|
||||
d = self.peer_connections[p]['factory'].p.cancel_requests()
|
||||
if self._peer_connections[p].factory.p is not None:
|
||||
d = self._peer_connections[p].factory.p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def disconnect_peer():
|
||||
self.peer_connections[p]['connection'].disconnect()
|
||||
if p in self.peer_connections:
|
||||
del self.peer_connections[p]
|
||||
d = defer.Deferred()
|
||||
self.connections_closing[p] = d
|
||||
self._connections_closing[p] = d
|
||||
self._peer_connections[p].connection.disconnect()
|
||||
if p in self._peer_connections:
|
||||
del self._peer_connections[p]
|
||||
return d
|
||||
|
||||
d.addBoth(lambda _: disconnect_peer())
|
||||
@@ -60,26 +73,27 @@ class ConnectionManager(object):
|
||||
|
||||
def get_next_request(self, peer, protocol):
|
||||
|
||||
logging.debug("Trying to get the next request for peer %s", str(peer))
|
||||
log.debug("Trying to get the next request for peer %s", peer)
|
||||
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("The peer has already been told to shut down.")
|
||||
if not peer in self._peer_connections or self.stopped is True:
|
||||
log.debug("The peer has already been told to shut down.")
|
||||
return defer.succeed(False)
|
||||
|
||||
def handle_error(err):
|
||||
if err.check(InsufficientFundsError):
|
||||
self.downloader.insufficient_funds()
|
||||
return False
|
||||
else:
|
||||
return err
|
||||
err.trap(InsufficientFundsError)
|
||||
self.downloader.insufficient_funds(err)
|
||||
return False
|
||||
|
||||
def check_if_request_sent(request_sent, request_creator):
|
||||
if peer not in self._peer_connections:
|
||||
# This can happen if the connection is told to close
|
||||
return False
|
||||
if request_sent is False:
|
||||
if request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].remove(request_creator)
|
||||
if request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.remove(request_creator)
|
||||
else:
|
||||
if not request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].append(request_creator)
|
||||
if not request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.append(request_creator)
|
||||
return request_sent
|
||||
|
||||
def check_requests(requests):
|
||||
@@ -89,7 +103,7 @@ class ConnectionManager(object):
|
||||
def get_secondary_requests_if_necessary(have_request):
|
||||
if have_request is True:
|
||||
ds = []
|
||||
for s_r_c in self.secondary_request_creators:
|
||||
for s_r_c in self._secondary_request_creators:
|
||||
d = s_r_c.send_next_request(peer, protocol)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
@@ -100,7 +114,7 @@ class ConnectionManager(object):
|
||||
|
||||
ds = []
|
||||
|
||||
for p_r_c in self.primary_request_creators:
|
||||
for p_r_c in self._primary_request_creators:
|
||||
d = p_r_c.send_next_request(peer, protocol)
|
||||
d.addErrback(handle_error)
|
||||
d.addCallback(check_if_request_sent, p_r_c)
|
||||
@@ -112,11 +126,11 @@ class ConnectionManager(object):
|
||||
return dl
|
||||
|
||||
def protocol_disconnected(self, peer, protocol):
|
||||
if peer in self.peer_connections:
|
||||
del self.peer_connections[peer]
|
||||
if peer in self.connections_closing:
|
||||
d = self.connections_closing[peer]
|
||||
del self.connections_closing[peer]
|
||||
if peer in self._peer_connections:
|
||||
del self._peer_connections[peer]
|
||||
if peer in self._connections_closing:
|
||||
d = self._connections_closing[peer]
|
||||
del self._connections_closing[peer]
|
||||
d.callback(True)
|
||||
|
||||
def _rank_request_creator_connections(self):
|
||||
@@ -125,30 +139,30 @@ class ConnectionManager(object):
|
||||
connections open that it likes
|
||||
"""
|
||||
def count_peers(request_creator):
|
||||
return len([p for p in self.peer_connections.itervalues() if request_creator in p['request_creators']])
|
||||
return len([p for p in self._peer_connections.itervalues() if request_creator in p.request_creators])
|
||||
|
||||
return sorted(self.primary_request_creators, key=count_peers)
|
||||
return sorted(self._primary_request_creators, key=count_peers)
|
||||
|
||||
def _connect_to_peer(self, peer):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if peer is not None:
|
||||
logging.debug("Trying to connect to %s", str(peer))
|
||||
if peer is not None and self.stopped is False:
|
||||
log.debug("Trying to connect to %s", peer)
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
|
||||
factory)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory)
|
||||
self.peer_connections[peer] = {'connection': connection,
|
||||
'request_creators': self.primary_request_creators[:],
|
||||
'factory': factory}
|
||||
self._peer_connections[peer].connection = connection
|
||||
|
||||
def _manage(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
def get_new_peers(request_creators):
|
||||
logging.debug("Trying to get a new peer to connect to")
|
||||
log.debug("Trying to get a new peer to connect to")
|
||||
if len(request_creators) > 0:
|
||||
logging.debug("Got a creator to check: %s", str(request_creators[0]))
|
||||
log.debug("Got a creator to check: %s", request_creators[0])
|
||||
d = request_creators[0].get_new_peers()
|
||||
d.addCallback(lambda h: h if h is not None else get_new_peers(request_creators[1:]))
|
||||
return d
|
||||
@@ -158,20 +172,21 @@ class ConnectionManager(object):
|
||||
def pick_best_peer(peers):
|
||||
# TODO: Eventually rank them based on past performance/reputation. For now
|
||||
# TODO: just pick the first to which we don't have an open connection
|
||||
logging.debug("Got a list of peers to choose from: %s", str(peers))
|
||||
|
||||
log.debug("Got a list of peers to choose from: %s", peers)
|
||||
if peers is None:
|
||||
return None
|
||||
for peer in peers:
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("Got a good peer. Returning peer %s", str(peer))
|
||||
if not peer in self._peer_connections:
|
||||
log.debug("Got a good peer. Returning peer %s", peer)
|
||||
return peer
|
||||
logging.debug("Couldn't find a good peer to connect to")
|
||||
log.debug("Couldn't find a good peer to connect to")
|
||||
return None
|
||||
|
||||
if len(self.peer_connections) < MAX_CONNECTIONS_PER_STREAM:
|
||||
if len(self._peer_connections) < settings.max_connections_per_stream:
|
||||
ordered_request_creators = self._rank_request_creator_connections()
|
||||
d = get_new_peers(ordered_request_creators)
|
||||
d.addCallback(pick_best_peer)
|
||||
d.addCallback(self._connect_to_peer)
|
||||
|
||||
self.next_manage_call = reactor.callLater(1, self._manage)
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
@@ -1,8 +1,13 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IPeerFinder
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTPeerFinder(object):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
implements(IPeerFinder)
|
||||
@@ -21,6 +26,7 @@ class DHTPeerFinder(object):
|
||||
self.next_manage_call = reactor.callLater(60, self.run_manage_loop)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self.next_manage_call is not None and self.next_manage_call.active():
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
@@ -32,8 +38,9 @@ class DHTPeerFinder(object):
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
|
||||
def filter_peers(peer_list):
|
||||
peers = set(peer_list)
|
||||
good_peers = []
|
||||
for host, port in peer_list:
|
||||
for host, port in peers:
|
||||
peer = self.peer_manager.get_peer(host, port)
|
||||
if peer.is_available() is True:
|
||||
good_peers.append(peer)
|
||||
@@ -44,4 +51,4 @@ class DHTPeerFinder(object):
|
||||
return d
|
||||
|
||||
def get_most_popular_hashes(self, num_to_return):
|
||||
return self.dht_node.get_most_popular_hashes(num_to_return)
|
||||
return self.dht_node.get_most_popular_hashes(num_to_return)
|
||||
@@ -5,6 +5,9 @@ from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadManager(object):
|
||||
implements(interfaces.IDownloadManager)
|
||||
|
||||
@@ -24,7 +27,7 @@ class DownloadManager(object):
|
||||
|
||||
def start_downloading(self):
|
||||
d = self.blob_info_finder.get_initial_blobs()
|
||||
logging.debug("Requested the initial blobs from the info finder")
|
||||
log.debug("Requested the initial blobs from the info finder")
|
||||
d.addCallback(self.add_blobs_to_download)
|
||||
d.addCallback(lambda _: self.resume_downloading())
|
||||
return d
|
||||
@@ -33,7 +36,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_start(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
log.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -49,7 +52,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_stop(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to stop the %s: %s", manager. result.getErrorMessage())
|
||||
log.error("Failed to stop the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -63,21 +66,21 @@ class DownloadManager(object):
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
logging.debug("Adding %s to blobs", str(blob_infos))
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
logging.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
log.debug("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
|
||||
def error_during_add(err):
|
||||
logging.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
log.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
logging.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
log.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, self.upload_allowed, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
@@ -112,4 +115,4 @@ class DownloadManager(object):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
@@ -5,9 +5,15 @@ from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.Error import InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleBlobMetadataHandler(object):
|
||||
implements(interfaces.IMetadataHandler)
|
||||
|
||||
@@ -18,7 +24,7 @@ class SingleBlobMetadataHandler(object):
|
||||
######## IMetadataHandler #########
|
||||
|
||||
def get_initial_blobs(self):
|
||||
logging.debug("Returning the blob info")
|
||||
log.debug("Returning the blob info")
|
||||
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
|
||||
|
||||
def final_blob_num(self):
|
||||
@@ -72,7 +78,7 @@ class SingleProgressManager(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
log.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
if self.finished is False:
|
||||
self.finished = True
|
||||
reactor.callLater(0, self.finished_callback, blob)
|
||||
@@ -87,7 +93,6 @@ class DummyBlobHandler(object):
|
||||
|
||||
|
||||
class StandaloneBlobDownloader(object):
|
||||
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_manager = blob_manager
|
||||
@@ -99,6 +104,9 @@ class StandaloneBlobDownloader(object):
|
||||
self.finished_deferred = None
|
||||
|
||||
def download(self):
|
||||
if not is_valid_blobhash(self.blob_hash):
|
||||
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
|
||||
|
||||
def cancel_download(d):
|
||||
self.stop()
|
||||
|
||||
@@ -127,7 +135,10 @@ class StandaloneBlobDownloader(object):
|
||||
|
||||
def _blob_downloaded(self, blob):
|
||||
self.stop()
|
||||
self.finished_deferred.callback(blob)
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.callback(blob)
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(err)
|
||||
@@ -4,6 +4,9 @@ from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamProgressManager(object):
|
||||
implements(IProgressManager)
|
||||
|
||||
@@ -61,18 +64,18 @@ class StreamProgressManager(object):
|
||||
return defer.succeed(None)
|
||||
|
||||
def _finished_with_blob(self, blob_num):
|
||||
logging.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
if self.delete_blob_after_finished is True:
|
||||
logging.debug("delete_blob_after_finished is True")
|
||||
log.debug("delete_blob_after_finished is True")
|
||||
blobs = self.download_manager.blobs
|
||||
if blob_num in blobs:
|
||||
logging.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
log.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
|
||||
else:
|
||||
logging.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
log.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
else:
|
||||
logging.debug("delete_blob_after_finished is False")
|
||||
log.debug("delete_blob_after_finished is False")
|
||||
|
||||
|
||||
class FullStreamProgressManager(StreamProgressManager):
|
||||
@@ -115,6 +118,8 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
|
||||
def finished_outputting_blob():
|
||||
self.last_blob_outputted += 1
|
||||
|
||||
def check_if_finished():
|
||||
final_blob_num = self.download_manager.final_blob_num()
|
||||
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
|
||||
self._finished_outputting()
|
||||
@@ -126,14 +131,18 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
current_blob_num = self.last_blob_outputted + 1
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
log.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
self.provided_blob_nums.append(self.last_blob_outputted + 1)
|
||||
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
d.addCallback(lambda _: check_if_finished())
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
log.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
if self.outputting_d is not None and not self.outputting_d.called:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
|
||||
d.addErrback(log_error)
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import requests
|
||||
from requests_futures.sessions import FuturesSession
|
||||
|
||||
import lbrynet
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.core import utils
|
||||
|
||||
|
||||
session = FuturesSession()
|
||||
|
||||
|
||||
def bg_cb(sess, resp):
|
||||
""" Don't do anything with the response """
|
||||
pass
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None):
|
||||
logging.Handler.__init__(self)
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
payload = self.format(record)
|
||||
session.post(self.url, data=payload, background_callback=bg_cb)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except:
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
for handler in log.handlers:
|
||||
if handler.name == handler_name:
|
||||
log.removeHandler(handler)
|
||||
|
||||
|
||||
def _log_decorator(fn):
|
||||
"""Configure a logging handler.
|
||||
|
||||
`fn` is a function that returns a logging handler. The returned
|
||||
handler has its log-level set and is attached to the specified
|
||||
logger or the root logger.
|
||||
"""
|
||||
def helper(*args, **kwargs):
|
||||
log = kwargs.pop('log', logging.getLogger())
|
||||
level = kwargs.pop('level', logging.INFO)
|
||||
if not isinstance(level, int):
|
||||
# despite the name, getLevelName returns
|
||||
# the numeric level when passed a text level
|
||||
level = logging.getLevelName(level)
|
||||
handler = fn(*args, **kwargs)
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
return handler
|
||||
return helper
|
||||
|
||||
|
||||
def disable_third_party_loggers():
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
logging.getLogger('urllib3').setLevel(logging.WARNING)
|
||||
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
|
||||
logging.getLogger('lbryum').setLevel(logging.WARNING)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_console(**kwargs):
|
||||
"""Convenience function to configure a log-handler that outputs to stdout"""
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'console'
|
||||
return handler
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_file_handler(file_name, **kwargs):
|
||||
"""Convenience function to configure a log-handler that writes to `file_name`"""
|
||||
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'file'
|
||||
return handler
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(settings.LOGGLY_TOKEN)
|
||||
version = version or lbrynet.__version__
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_loggly_handler(url=None, **kwargs):
|
||||
url = url or get_loggly_url()
|
||||
formatter = JsonFormatter(**kwargs)
|
||||
handler = HTTPSHandler(url)
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def failure(failure, log, msg, *args):
|
||||
"""Log a failure message from a deferred.
|
||||
|
||||
Args:
|
||||
failure: twisted.python.failure.Failure
|
||||
log: a python logger instance
|
||||
msg: the message to log. Can use normal logging string interpolation.
|
||||
the last argument will be set to the error message from the failure.
|
||||
args: values to substitute into `msg`
|
||||
"""
|
||||
args += (failure.getErrorMessage(),)
|
||||
exc_info = (failure.type, failure.value, failure.getTracebackObject())
|
||||
log.error(msg, *args, exc_info=exc_info)
|
||||
|
||||
|
||||
def convert_verbose(verbose):
|
||||
"""Convert the results of the --verbose flag into a list of logger names
|
||||
|
||||
if --verbose is not provided, args.verbose will be None and logging
|
||||
should be at the info level.
|
||||
if --verbose is provided, but not followed by any arguments, then
|
||||
args.verbose = [] and debug logging should be enabled for all of lbrynet
|
||||
along with info logging on lbryum.
|
||||
if --verbose is provided and followed by arguments, those arguments
|
||||
will be in a list
|
||||
"""
|
||||
if verbose is None:
|
||||
return []
|
||||
if verbose == []:
|
||||
return ['lbrynet', 'lbryum']
|
||||
return verbose
|
||||
|
||||
|
||||
def configure_logging(file_name, console, verbose=None):
|
||||
"""Apply the default logging configuration.
|
||||
|
||||
Enables two log-handlers at the INFO level: a file logger and a loggly logger.
|
||||
Optionally turns on a console logger that defaults to the INFO level, with
|
||||
specified loggers being set to the DEBUG level.
|
||||
|
||||
Args:
|
||||
file_name: the file to which logs should be saved
|
||||
console: If true, enable a console logger
|
||||
verbose: a list of loggers to set to debug level.
|
||||
See `convert_verbose` for more details.
|
||||
"""
|
||||
verbose = convert_verbose(verbose)
|
||||
configure_file_handler(file_name)
|
||||
configure_loggly_handler()
|
||||
disable_third_party_loggers()
|
||||
if console:
|
||||
# if there are some loggers at the debug level, we need
|
||||
# to enable the console to allow debug. Otherwise, only
|
||||
# allow info.
|
||||
level = 'DEBUG' if verbose else 'INFO'
|
||||
handler = configure_console(level=level)
|
||||
if 'lbryum' in verbose:
|
||||
# TODO: this enables lbryum logging on the other handlers
|
||||
# too which isn't consistent with how verbose logging
|
||||
# happens with other loggers. Should change the configuration
|
||||
# so that its only logging at the INFO level for the console.
|
||||
logging.getLogger('lbryum').setLevel(logging.INFO)
|
||||
verbose.remove('lbryum')
|
||||
if verbose:
|
||||
handler.addFilter(LoggerNameFilter(verbose))
|
||||
|
||||
|
||||
class LoggerNameFilter(object):
|
||||
"""Filter a log record based on its name.
|
||||
|
||||
Allows all info level and higher records to pass thru.
|
||||
Debug records pass if the log record name (or a parent) match
|
||||
the input list of logger names.
|
||||
"""
|
||||
def __init__(self, logger_names):
|
||||
self.logger_names = logger_names
|
||||
|
||||
def filter(self, record):
|
||||
if record.levelno >= logging.INFO:
|
||||
return True
|
||||
name = record.name
|
||||
while name:
|
||||
if name in self.logger_names:
|
||||
return True
|
||||
name = get_parent(name)
|
||||
return False
|
||||
|
||||
|
||||
def get_parent(logger_name):
|
||||
names = logger_name.split('.')
|
||||
if len(names) == 1:
|
||||
return ''
|
||||
names = names[:-1]
|
||||
return '.'.join(names)
|
||||
|
||||
|
||||
class LogUploader(object):
|
||||
def __init__(self, log_name, log_file, log_size):
|
||||
self.log_name = log_name
|
||||
self.log_file = log_file
|
||||
self.log_size = log_size
|
||||
|
||||
def upload(self, exclude_previous, id_hash, log_type):
|
||||
if not os.path.isfile(self.log_file):
|
||||
return
|
||||
log_contents = self.log_contents(exclude_previous)
|
||||
params = {
|
||||
'date': datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S'),
|
||||
'hash': id_hash,
|
||||
'sys': platform.system(),
|
||||
'type': self.get_type(log_type),
|
||||
'log': log_contents
|
||||
}
|
||||
requests.post(settings.LOG_POST_URL, params)
|
||||
|
||||
def log_contents(self, exclude_previous):
|
||||
with open(self.log_file) as f:
|
||||
if exclude_previous:
|
||||
f.seek(self.log_size)
|
||||
log_contents = f.read()
|
||||
else:
|
||||
log_contents = f.read()
|
||||
return log_contents
|
||||
|
||||
def get_type(self, log_type):
|
||||
if log_type:
|
||||
return "%s-%s" % (self.log_name, log_type)
|
||||
else:
|
||||
return self.log_name
|
||||
|
||||
@classmethod
|
||||
def load(cls, log_name, log_file):
|
||||
if os.path.isfile(log_file):
|
||||
with open(log_file, 'r') as f:
|
||||
log_size = len(f.read())
|
||||
else:
|
||||
log_size = 0
|
||||
return cls(log_name, log_file, log_size)
|
||||
@@ -0,0 +1,20 @@
|
||||
class LoopingCallManager(object):
|
||||
def __init__(self, calls=None):
|
||||
self.calls = calls or {}
|
||||
|
||||
def register_looping_call(self, name, call):
|
||||
assert name not in self.calls, '{} is already registered'.format(name)
|
||||
self.calls[name] = call
|
||||
|
||||
def start(self, name, *args):
|
||||
lcall = self.calls[name]
|
||||
if not lcall.running:
|
||||
lcall.start(*args)
|
||||
|
||||
def stop(self, name):
|
||||
self.calls[name].stop()
|
||||
|
||||
def shutdown(self):
|
||||
for lcall in self.calls.itervalues():
|
||||
if lcall.running:
|
||||
lcall.stop()
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -37,10 +40,11 @@ class BlobAvailabilityHandler(object):
|
||||
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
logging.debug("Received the client's list of requested blobs")
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
|
||||
@@ -1,23 +1,32 @@
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet import analytics
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequestHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager):
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, track):
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.track = track
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = BlobRequestHandler(self.blob_manager, self.wallet, self.payment_rate_manager)
|
||||
q_h = BlobRequestHandler(self.blob_manager, self.wallet, self.payment_rate_manager, self.track)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
@@ -29,18 +38,23 @@ class BlobRequestHandlerFactory(object):
|
||||
|
||||
class BlobRequestHandler(object):
|
||||
implements(IQueryHandler, IBlobSender)
|
||||
PAYMENT_RATE_QUERY = 'blob_data_payment_rate'
|
||||
BLOB_QUERY = 'requested_blob'
|
||||
AVAILABILITY_QUERY = 'requested_blobs'
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager):
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, track):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = ['blob_data_payment_rate', 'requested_blob']
|
||||
self.query_identifiers = [self.PAYMENT_RATE_QUERY, self.BLOB_QUERY, self.AVAILABILITY_QUERY]
|
||||
self.track = track
|
||||
self.peer = None
|
||||
self.blob_data_payment_rate = None
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
self.blob_bytes_uploaded = 0
|
||||
self._blobs_requested = []
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
@@ -50,44 +64,22 @@ class BlobRequestHandler(object):
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = {}
|
||||
if self.query_identifiers[0] in queries:
|
||||
if not self.handle_blob_data_payment_rate(queries[self.query_identifiers[0]]):
|
||||
response['blob_data_payment_rate'] = "RATE_TOO_LOW"
|
||||
else:
|
||||
response['blob_data_payment_rate'] = 'RATE_ACCEPTED'
|
||||
response = defer.succeed({})
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
logging.debug("Received the client's request to send a blob")
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response_fields['error'] = "RATE_UNSET"
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
|
||||
d = self.blob_manager.get_blob(queries[self.query_identifiers[1]], True)
|
||||
|
||||
def open_blob_for_reading(blob):
|
||||
if blob.is_validated():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
logging.debug("Sending %s to client", str(blob))
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
return response
|
||||
logging.debug("We can not send %s", str(blob))
|
||||
response_fields['error'] = "BLOB_UNAVAILABLE"
|
||||
return response
|
||||
|
||||
d.addCallback(open_blob_for_reading)
|
||||
|
||||
return d
|
||||
else:
|
||||
return defer.succeed(response)
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
|
||||
@@ -105,12 +97,91 @@ class BlobRequestHandler(object):
|
||||
|
||||
######### internal #########
|
||||
|
||||
def handle_blob_data_payment_rate(self, requested_payment_rate):
|
||||
if not self.payment_rate_manager.accept_rate_blob_data(self.peer, requested_payment_rate):
|
||||
return False
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
log.debug("Offered rate %f LBC/mb for %i blobs", offer.rate, len(blobs))
|
||||
reply = self.payment_rate_manager.reply_to_offer(self.peer, blobs, offer)
|
||||
if reply.is_accepted:
|
||||
self.blob_data_payment_rate = offer.rate
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_ACCEPTED"
|
||||
log.debug("Accepted rate: %f", offer.rate)
|
||||
elif reply.is_too_low:
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_TOO_LOW"
|
||||
log.debug("Reject rate: %f", offer.rate)
|
||||
elif reply.is_unset:
|
||||
log.warning("Rate unset")
|
||||
request['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
log.debug("Returning rate query result: %s", str(request))
|
||||
|
||||
return request
|
||||
|
||||
def _handle_blob_query(self, response, query):
|
||||
log.debug("Received the client's request to send a blob")
|
||||
response['incoming_blob'] = {}
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response['incoming_blob'] = {'error': "RATE_UNSET"}
|
||||
return response
|
||||
else:
|
||||
self.blob_data_payment_rate = requested_payment_rate
|
||||
return True
|
||||
return self._send_blob(response, query)
|
||||
|
||||
def _send_blob(self, response, query):
|
||||
d = self.blob_manager.get_blob(query, True)
|
||||
d.addCallback(self.open_blob_for_reading, response)
|
||||
return d
|
||||
|
||||
def open_blob_for_reading(self, blob, response):
|
||||
response_fields = {}
|
||||
d = defer.succeed(None)
|
||||
if blob.is_validated():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
log.info("Sending %s to client", str(blob))
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
response['incoming_blob'] = response_fields
|
||||
d.addCallback(lambda _: self.record_transaction(blob))
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
log.debug("We can not send %s", str(blob))
|
||||
response['incoming_blob'] = {'error': 'BLOB_UNAVAILABLE'}
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
|
||||
def record_transaction(self, blob):
|
||||
d = self.blob_manager.add_blob_to_upload_history(str(blob), self.peer.host, self.blob_data_payment_rate)
|
||||
return d
|
||||
|
||||
def _reply_to_send_request(self, response, incoming):
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
log.debug("Rate not set yet")
|
||||
response['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
log.debug("Requested blob: %s", str(incoming))
|
||||
d = self.blob_manager.get_blob(incoming, True)
|
||||
d.addCallback(lambda blob: self.open_blob_for_reading(blob, response))
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
@@ -123,20 +194,23 @@ class BlobRequestHandler(object):
|
||||
return inner_d
|
||||
|
||||
def count_bytes(data):
|
||||
self.blob_bytes_uploaded += len(data)
|
||||
self.peer.update_stats('blob_bytes_uploaded', len(data))
|
||||
uploaded = len(data)
|
||||
self.blob_bytes_uploaded += uploaded
|
||||
self.peer.update_stats('blob_bytes_uploaded', uploaded)
|
||||
self.track.add_observation(analytics.BLOB_BYTES_UPLOADED, uploaded)
|
||||
return data
|
||||
|
||||
def start_transfer():
|
||||
self.file_sender = FileSender()
|
||||
logging.info("Starting the file upload")
|
||||
log.debug("Starting the file upload")
|
||||
assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
|
||||
return d
|
||||
|
||||
def set_expected_payment():
|
||||
logging.info("Setting expected payment")
|
||||
log.debug("Setting expected payment")
|
||||
if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None:
|
||||
# TODO: explain why 2**20
|
||||
self.wallet.add_expected_payment(self.peer,
|
||||
self.currently_uploading.length * 1.0 *
|
||||
self.blob_data_payment_rate / 2**20)
|
||||
@@ -151,6 +225,6 @@ class BlobRequestHandler(object):
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
if reason is not None and isinstance(reason, Failure):
|
||||
logging.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
return _send_file()
|
||||
return _send_file()
|
||||
@@ -1,6 +1,11 @@
|
||||
import binascii
|
||||
from twisted.internet import defer, task, reactor
|
||||
import collections
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer, reactor
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTHashAnnouncer(object):
|
||||
@@ -22,6 +27,7 @@ class DHTHashAnnouncer(object):
|
||||
self.next_manage_call = reactor.callLater(60, self.run_manage_loop)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping %s", self)
|
||||
if self.next_manage_call is not None:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
@@ -78,4 +84,4 @@ class DHTHashSupplier(object):
|
||||
self.hash_reannounce_time = 60 * 60 # 1 hour
|
||||
|
||||
def hashes_to_announce(self):
|
||||
pass
|
||||
pass
|
||||
@@ -6,6 +6,9 @@ from zope.interface import implements
|
||||
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerProtocol(Protocol):
|
||||
"""ServerProtocol needs to:
|
||||
|
||||
@@ -26,7 +29,7 @@ class ServerProtocol(Protocol):
|
||||
#Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
logging.debug("Got a connection")
|
||||
log.debug("Got a connection")
|
||||
peer_info = self.transport.getPeer()
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.request_handler = ServerRequestHandler(self)
|
||||
@@ -34,7 +37,7 @@ class ServerProtocol(Protocol):
|
||||
if enabled is True:
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
logging.debug("Setting the request handler")
|
||||
log.debug("Setting the request handler")
|
||||
self.factory.rate_limiter.register_protocol(self)
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
@@ -42,10 +45,10 @@ class ServerProtocol(Protocol):
|
||||
self.request_handler.stopProducing()
|
||||
self.factory.rate_limiter.unregister_protocol(self)
|
||||
if not reason.check(error.ConnectionDone):
|
||||
logging.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
def dataReceived(self, data):
|
||||
logging.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
self.factory.rate_limiter.report_dl_bytes(len(data))
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.data_received(data)
|
||||
@@ -53,7 +56,7 @@ class ServerProtocol(Protocol):
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
logging.debug("Registering the producer")
|
||||
log.debug("Registering the producer")
|
||||
assert streaming is True
|
||||
|
||||
def unregisterProducer(self):
|
||||
@@ -61,7 +64,7 @@ class ServerProtocol(Protocol):
|
||||
self.transport.loseConnection()
|
||||
|
||||
def write(self, data):
|
||||
logging.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
log.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
self.transport.write(data)
|
||||
self.factory.rate_limiter.report_ul_bytes(len(data))
|
||||
|
||||
|
||||
@@ -5,10 +5,14 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IRequestHandler
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and return request for information about
|
||||
more blobs that are associated with streams"""
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and
|
||||
return request for information about more blobs that are
|
||||
associated with streams.
|
||||
"""
|
||||
implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
|
||||
|
||||
def __init__(self, consumer):
|
||||
@@ -52,7 +56,7 @@ class ServerRequestHandler(object):
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk != '':
|
||||
logging.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
log.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
|
||||
@@ -76,7 +80,7 @@ class ServerRequestHandler(object):
|
||||
|
||||
def get_more_data():
|
||||
if self.producer is not None:
|
||||
logging.debug("Requesting more data from the producer")
|
||||
log.debug("Requesting more data from the producer")
|
||||
self.producer.resumeProducing()
|
||||
|
||||
reactor.callLater(0, get_more_data)
|
||||
@@ -84,22 +88,29 @@ class ServerRequestHandler(object):
|
||||
#From Protocol
|
||||
|
||||
def data_received(self, data):
|
||||
logging.debug("Received data")
|
||||
logging.debug("%s", str(data))
|
||||
log.debug("Received data")
|
||||
log.debug("%s", str(data))
|
||||
if self.request_received is False:
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg is not None:
|
||||
self.request_buff = ''
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender is not None:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
else:
|
||||
logging.info("Request buff not a valid json message")
|
||||
logging.info("Request buff: %s", str(self.request_buff))
|
||||
return self._parse_data_and_maybe_send_blob(data)
|
||||
else:
|
||||
logging.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
log.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
|
||||
def _parse_data_and_maybe_send_blob(self, data):
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg:
|
||||
self.request_buff = ''
|
||||
self._process_msg(msg)
|
||||
else:
|
||||
log.debug("Request buff not a valid json message")
|
||||
log.debug("Request buff: %s", self.request_buff)
|
||||
|
||||
def _process_msg(self, msg):
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
|
||||
|
||||
######### IRequestHandler #########
|
||||
|
||||
@@ -112,7 +123,7 @@ class ServerRequestHandler(object):
|
||||
#response handling
|
||||
|
||||
def request_failure_handler(self, err):
|
||||
logging.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
self.stopProducing()
|
||||
return err
|
||||
|
||||
@@ -122,15 +133,15 @@ class ServerRequestHandler(object):
|
||||
|
||||
def send_response(self, msg):
|
||||
m = json.dumps(msg)
|
||||
logging.info("Sending a response of length %s", str(len(m)))
|
||||
logging.debug("Response: %s", str(m))
|
||||
log.debug("Sending a response of length %s", str(len(m)))
|
||||
log.debug("Response: %s", str(m))
|
||||
self.response_buff = self.response_buff + m
|
||||
self._produce_more()
|
||||
return True
|
||||
|
||||
def handle_request(self, msg):
|
||||
logging.debug("Handling a request")
|
||||
logging.debug(str(msg))
|
||||
log.debug("Handling a request")
|
||||
log.debug(str(msg))
|
||||
|
||||
def create_response_message(results):
|
||||
response = {}
|
||||
@@ -140,11 +151,11 @@ class ServerRequestHandler(object):
|
||||
else:
|
||||
# result is a Failure
|
||||
return result
|
||||
logging.debug("Finished making the response message. Response: %s", str(response))
|
||||
log.debug("Finished making the response message. Response: %s", str(response))
|
||||
return response
|
||||
|
||||
def log_errors(err):
|
||||
logging.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
def send_response(response):
|
||||
@@ -168,4 +179,4 @@ class ServerRequestHandler(object):
|
||||
msg = json.loads(request_buff)
|
||||
return msg
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -0,0 +1,23 @@
|
||||
import sqlite3
|
||||
from twisted.internet import task, reactor
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
|
||||
def rerun(err, *args, **kwargs):
|
||||
if err.check(sqlite3.OperationalError) and err.value.message == "database is locked":
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
return task.deferLater(reactor, 0, wrapper, *args, **kwargs)
|
||||
return err
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addErrback(rerun, *args, **kwargs)
|
||||
return d
|
||||
|
||||
return wrapper
|
||||
+75
-9
@@ -1,8 +1,39 @@
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
import base64
|
||||
import datetime
|
||||
import logging
|
||||
import random
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2 # digest_size is in bytes, and blob hashes are hex encoded
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# defining these time functions here allows for easier overriding in testing
|
||||
def now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def utcnow():
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def isonow():
|
||||
"""Return utc now in isoformat with timezone"""
|
||||
return utcnow().isoformat() + 'Z'
|
||||
|
||||
|
||||
def today():
|
||||
return datetime.datetime.today()
|
||||
|
||||
|
||||
def generate_id(num=None):
|
||||
@@ -14,15 +45,50 @@ def generate_id(num=None):
|
||||
return h.digest()
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: Whether the blobhash is the correct length and contains only valid characters (0-9, a-f)
|
||||
@return: True/False
|
||||
"""
|
||||
if len(blobhash) != blobhash_length:
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def version_is_greater_than(a, b):
|
||||
"""Returns True if version a is more recent than version b"""
|
||||
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
|
||||
|
||||
|
||||
def deobfuscate(obfustacated):
|
||||
return base64.b64decode(obfustacated.decode('rot13'))
|
||||
|
||||
|
||||
def obfuscate(plain):
|
||||
return base64.b64encode(plain).encode('rot13')
|
||||
|
||||
|
||||
def check_connection(server="www.lbry.io", port=80):
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
try:
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
host = socket.gethostbyname(server)
|
||||
s = socket.create_connection((host, port), 2)
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except Exception as ex:
|
||||
log.info(
|
||||
"Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port, exc_info=True)
|
||||
return False
|
||||
for l in blobhash:
|
||||
if l not in "0123456789abcdef":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def setup_certs_for_windows():
|
||||
if getattr(sys, 'frozen', False) and os.name == "nt":
|
||||
cert_path = os.path.join(os.path.dirname(sys.executable), "cacert.pem")
|
||||
os.environ["REQUESTS_CA_BUNDLE"] = cert_path
|
||||
@@ -1,10 +1,13 @@
|
||||
import binascii
|
||||
import logging
|
||||
from Crypto.Cipher import AES
|
||||
from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
@@ -64,7 +67,7 @@ class CryptStreamBlobMaker(object):
|
||||
self.length = 0
|
||||
|
||||
def write(self, data):
|
||||
max_bytes_to_write = BLOB_SIZE - self.length - 1
|
||||
max_bytes_to_write = settings.BLOB_SIZE - self.length - 1
|
||||
done = False
|
||||
if max_bytes_to_write <= len(data):
|
||||
num_bytes_to_write = max_bytes_to_write
|
||||
@@ -78,17 +81,17 @@ class CryptStreamBlobMaker(object):
|
||||
return done, num_bytes_to_write
|
||||
|
||||
def close(self):
|
||||
logging.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self._close_buffer()
|
||||
d = self.blob.close()
|
||||
d.addCallback(self._return_info)
|
||||
logging.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
return d
|
||||
|
||||
def _write_buffer(self):
|
||||
num_bytes_to_encrypt = (len(self.buff) // AES.block_size) * AES.block_size
|
||||
data_to_encrypt, self.buff = self.buff[:num_bytes_to_encrypt], self.buff[num_bytes_to_encrypt:]
|
||||
data_to_encrypt, self.buff = split(self.buff, num_bytes_to_encrypt)
|
||||
encrypted_data = self.cipher.encrypt(data_to_encrypt)
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
@@ -103,4 +106,7 @@ class CryptStreamBlobMaker(object):
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
def _return_info(self, blob_hash):
|
||||
return CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
return CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
|
||||
def split(buff, cutoff):
|
||||
return buff[:cutoff], buff[cutoff:]
|
||||
@@ -12,25 +12,29 @@ from lbrynet.core.StreamCreator import StreamCreator
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(StreamCreator):
|
||||
"""Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
Each blob is encrypted with the same key, but each blob has its own initialization vector
|
||||
which is associated with the blob when the blob is associated with the stream."""
|
||||
Each blob is encrypted with the same key, but each blob has its
|
||||
own initialization vector which is associated with the blob when
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""
|
||||
@param blob_manager: Object that stores and provides access to blobs.
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@type blob_manager: BlobManager
|
||||
|
||||
@param name: the name of the stream, which will be presented to the user
|
||||
@type name: string
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
|
||||
be generated.
|
||||
@param key: the raw AES key which will be used to encrypt the
|
||||
blobs. If None, a random key will be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
|
||||
once for each blob.
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: None
|
||||
@@ -57,22 +61,20 @@ class CryptStreamCreator(StreamCreator):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _finalize(self):
|
||||
logging.debug("_finalize has been called")
|
||||
log.debug("_finalize has been called")
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
final_blob_creator = self.blob_manager.get_blob_creator()
|
||||
logging.debug("Created the finished_deferred")
|
||||
log.debug("Created the finished_deferred")
|
||||
final_blob = self._get_blob_maker(iv, final_blob_creator)
|
||||
logging.debug("Created the final blob")
|
||||
logging.debug("Calling close on final blob")
|
||||
log.debug("Created the final blob")
|
||||
log.debug("Calling close on final blob")
|
||||
d = final_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
self.finished_deferreds.append(d)
|
||||
logging.debug("called close on final blob, returning from make_final_blob")
|
||||
return d
|
||||
log.debug("called close on final blob, returning from make_final_blob")
|
||||
|
||||
def _write(self, data):
|
||||
|
||||
def close_blob(blob):
|
||||
d = blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
@@ -91,4 +93,4 @@ class CryptStreamCreator(StreamCreator):
|
||||
self.current_blob = None
|
||||
|
||||
def _get_blob_maker(self, iv, blob_creator):
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IStreamDownloader
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
@@ -9,6 +10,9 @@ from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartFailedError(Exception):
|
||||
pass
|
||||
|
||||
@@ -44,9 +48,9 @@ class CryptStreamDownloader(object):
|
||||
|
||||
@param blob_manager: A BlobManager object
|
||||
|
||||
@param payment_rate_manager: A PaymentRateManager object
|
||||
@param payment_rate_manager: A NegotiatedPaymentRateManager object
|
||||
|
||||
@param wallet: An object which implements the ILBRYWallet interface
|
||||
@param wallet: An object which implements the IWallet interface
|
||||
|
||||
@return:
|
||||
"""
|
||||
@@ -71,6 +75,9 @@ class CryptStreamDownloader(object):
|
||||
|
||||
self.points_paid = 0.0
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stream_name)
|
||||
|
||||
def toggle_running(self):
|
||||
if self.stopped is True:
|
||||
return self.start()
|
||||
@@ -79,10 +86,6 @@ class CryptStreamDownloader(object):
|
||||
|
||||
def start(self):
|
||||
|
||||
def set_finished_deferred():
|
||||
self.finished_deferred = defer.Deferred()
|
||||
return self.finished_deferred
|
||||
|
||||
if self.starting is True:
|
||||
raise CurrentlyStartingError()
|
||||
if self.stopping is True:
|
||||
@@ -92,11 +95,13 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is None
|
||||
self.starting = True
|
||||
self.completed = False
|
||||
self.finished_deferred = defer.Deferred()
|
||||
fd = self.finished_deferred
|
||||
d = self._start()
|
||||
d.addCallback(lambda _: set_finished_deferred())
|
||||
d.addCallback(lambda _: fd)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
def stop(self, err=None):
|
||||
|
||||
def check_if_stop_succeeded(success):
|
||||
self.stopping = False
|
||||
@@ -112,8 +117,8 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is not None
|
||||
self.stopping = True
|
||||
d = self.download_manager.stop_downloading()
|
||||
self._fire_completed_deferred()
|
||||
d.addCallback(check_if_stop_succeeded)
|
||||
d.addCallback(lambda _: self._fire_completed_deferred(err))
|
||||
return d
|
||||
|
||||
def _start_failed(self):
|
||||
@@ -196,10 +201,15 @@ class CryptStreamDownloader(object):
|
||||
self._get_primary_request_creators(download_manager),
|
||||
self._get_secondary_request_creators(download_manager))
|
||||
|
||||
def _fire_completed_deferred(self):
|
||||
def _fire_completed_deferred(self, err=None):
|
||||
self.finished_deferred, d = None, self.finished_deferred
|
||||
if d is not None:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
if err is not None:
|
||||
d.errback(err)
|
||||
else:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
else:
|
||||
log.debug("Not firing the completed deferred because d is None")
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
return None
|
||||
@@ -209,5 +219,5 @@ class CryptStreamDownloader(object):
|
||||
self.completed = True
|
||||
return self.stop()
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop(err=err)
|
||||
Whitespace-only changes.
@@ -0,0 +1,27 @@
|
||||
import logging
|
||||
|
||||
|
||||
def migrate_db(db_dir, start, end):
|
||||
current = start
|
||||
while current < end:
|
||||
if current == 1:
|
||||
from lbrynet.db_migrator.migrate1to2 import do_migration
|
||||
do_migration(db_dir)
|
||||
else:
|
||||
raise Exception(
|
||||
"DB migration of version {} to {} is not available".format(current, current+1))
|
||||
current += 1
|
||||
return None
|
||||
|
||||
|
||||
def run_migration_script():
|
||||
import sys
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
|
||||
sys.stdout = open("migrator.out.log", 'w')
|
||||
sys.stderr = open("migrator.err.log", 'w')
|
||||
migrate_db(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_migration_script()
|
||||
@@ -0,0 +1,307 @@
|
||||
# import sqlite3
|
||||
# import unqlite
|
||||
# import leveldb
|
||||
# import shutil
|
||||
# import os
|
||||
# import logging
|
||||
# import json
|
||||
#
|
||||
#
|
||||
# log = logging.getLogger(__name__)
|
||||
#
|
||||
#
|
||||
# known_dbs = ['lbryfile_desc.db', 'lbryfiles.db', 'valuable_blobs.db', 'blobs.db',
|
||||
# 'lbryfile_blob.db', 'lbryfile_info.db', 'settings.db', 'blind_settings.db',
|
||||
# 'blind_peers.db', 'blind_info.db', 'lbryfile_info.db', 'lbryfile_manager.db',
|
||||
# 'live_stream.db', 'stream_info.db', 'stream_blob.db', 'stream_desc.db']
|
||||
#
|
||||
#
|
||||
# def do_move(from_dir, to_dir):
|
||||
# for known_db in known_dbs:
|
||||
# known_db_path = os.path.join(from_dir, known_db)
|
||||
# if os.path.exists(known_db_path):
|
||||
# log.debug("Moving %s to %s",
|
||||
# os.path.abspath(known_db_path),
|
||||
# os.path.abspath(os.path.join(to_dir, known_db)))
|
||||
# shutil.move(known_db_path, os.path.join(to_dir, known_db))
|
||||
# else:
|
||||
# log.debug("Did not find %s", os.path.abspath(known_db_path))
|
||||
#
|
||||
#
|
||||
# def do_migration(db_dir):
|
||||
# old_dir = os.path.join(db_dir, "_0_to_1_old")
|
||||
# new_dir = os.path.join(db_dir, "_0_to_1_new")
|
||||
# try:
|
||||
# log.info("Moving dbs from the real directory to %s", os.path.abspath(old_dir))
|
||||
# os.makedirs(old_dir)
|
||||
# do_move(db_dir, old_dir)
|
||||
# except:
|
||||
# log.error("An error occurred moving the old db files.")
|
||||
# raise
|
||||
# try:
|
||||
# log.info("Creating the new directory in %s", os.path.abspath(new_dir))
|
||||
# os.makedirs(new_dir)
|
||||
#
|
||||
# except:
|
||||
# log.error("An error occurred creating the new directory.")
|
||||
# raise
|
||||
# try:
|
||||
# log.info("Doing the migration")
|
||||
# migrate_blob_db(old_dir, new_dir)
|
||||
# migrate_lbryfile_db(old_dir, new_dir)
|
||||
# migrate_livestream_db(old_dir, new_dir)
|
||||
# migrate_ptc_db(old_dir, new_dir)
|
||||
# migrate_lbryfile_manager_db(old_dir, new_dir)
|
||||
# migrate_settings_db(old_dir, new_dir)
|
||||
# migrate_repeater_db(old_dir, new_dir)
|
||||
# log.info("Migration succeeded")
|
||||
# except:
|
||||
# log.error("An error occurred during the migration. Restoring.")
|
||||
# do_move(old_dir, db_dir)
|
||||
# raise
|
||||
# try:
|
||||
# log.info("Moving dbs in the new directory to the real directory")
|
||||
# do_move(new_dir, db_dir)
|
||||
# db_revision = open(os.path.join(db_dir, 'db_revision'), mode='w+')
|
||||
# db_revision.write("1")
|
||||
# db_revision.close()
|
||||
# os.rmdir(new_dir)
|
||||
# except:
|
||||
# log.error("An error occurred moving the new db files.")
|
||||
# raise
|
||||
# return old_dir
|
||||
#
|
||||
#
|
||||
# def migrate_blob_db(old_db_dir, new_db_dir):
|
||||
# old_blob_db_path = os.path.join(old_db_dir, "blobs.db")
|
||||
# if not os.path.exists(old_blob_db_path):
|
||||
# return True
|
||||
#
|
||||
# old_db = leveldb.LevelDB(old_blob_db_path)
|
||||
# new_db_conn = sqlite3.connect(os.path.join(new_db_dir, "blobs.db"))
|
||||
# c = new_db_conn.cursor()
|
||||
# c.execute("create table if not exists blobs (" +
|
||||
# " blob_hash text primary key, " +
|
||||
# " blob_length integer, " +
|
||||
# " last_verified_time real, " +
|
||||
# " next_announce_time real"
|
||||
# ")")
|
||||
# new_db_conn.commit()
|
||||
# c = new_db_conn.cursor()
|
||||
# for blob_hash, blob_info in old_db.RangeIter():
|
||||
# blob_length, verified_time, announce_time = json.loads(blob_info)
|
||||
# c.execute("insert into blobs values (?, ?, ?, ?)",
|
||||
# (blob_hash, blob_length, verified_time, announce_time))
|
||||
# new_db_conn.commit()
|
||||
# new_db_conn.close()
|
||||
#
|
||||
#
|
||||
# def migrate_lbryfile_db(old_db_dir, new_db_dir):
|
||||
# old_lbryfile_db_path = os.path.join(old_db_dir, "lbryfiles.db")
|
||||
# if not os.path.exists(old_lbryfile_db_path):
|
||||
# return True
|
||||
#
|
||||
# stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_info.db"))
|
||||
# stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_blob.db"))
|
||||
# stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_desc.db"))
|
||||
#
|
||||
# db_conn = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
|
||||
# c = db_conn.cursor()
|
||||
# c.execute("create table if not exists lbry_files (" +
|
||||
# " stream_hash text primary key, " +
|
||||
# " key text, " +
|
||||
# " stream_name text, " +
|
||||
# " suggested_file_name text" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists lbry_file_blobs (" +
|
||||
# " blob_hash text, " +
|
||||
# " stream_hash text, " +
|
||||
# " position integer, " +
|
||||
# " iv text, " +
|
||||
# " length integer, " +
|
||||
# " foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists lbry_file_descriptors (" +
|
||||
# " sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
# " stream_hash TEXT, " +
|
||||
# " foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
# ")")
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for stream_hash, stream_info in stream_info_db.RangeIter():
|
||||
# key, name, suggested_file_name = json.loads(stream_info)
|
||||
# c.execute("insert into lbry_files values (?, ?, ?, ?)",
|
||||
# (stream_hash, key, name, suggested_file_name))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
|
||||
# b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
# position, iv, length = json.loads(blob_info)
|
||||
# c.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
|
||||
# (b_h, s_h, position, iv, length))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
|
||||
# c.execute("insert into lbry_file_descriptors values (?, ?)",
|
||||
# (sd_blob_hash, stream_hash))
|
||||
# db_conn.commit()
|
||||
# db_conn.close()
|
||||
#
|
||||
#
|
||||
# def migrate_livestream_db(old_db_dir, new_db_dir):
|
||||
# old_db_path = os.path.join(old_db_dir, "stream_info.db")
|
||||
# if not os.path.exists(old_db_path):
|
||||
# return True
|
||||
# stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_info.db"))
|
||||
# stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_blob.db"))
|
||||
# stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_desc.db"))
|
||||
#
|
||||
# db_conn = sqlite3.connect(os.path.join(new_db_dir, "live_stream.db"))
|
||||
#
|
||||
# c = db_conn.cursor()
|
||||
#
|
||||
# c.execute("create table if not exists live_streams (" +
|
||||
# " stream_hash text primary key, " +
|
||||
# " public_key text, " +
|
||||
# " key text, " +
|
||||
# " stream_name text, " +
|
||||
# " next_announce_time real" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists live_stream_blobs (" +
|
||||
# " blob_hash text, " +
|
||||
# " stream_hash text, " +
|
||||
# " position integer, " +
|
||||
# " revision integer, " +
|
||||
# " iv text, " +
|
||||
# " length integer, " +
|
||||
# " signature text, " +
|
||||
# " foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists live_stream_descriptors (" +
|
||||
# " sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
# " stream_hash TEXT, " +
|
||||
# " foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
# ")")
|
||||
#
|
||||
# db_conn.commit()
|
||||
#
|
||||
# c = db_conn.cursor()
|
||||
# for stream_hash, stream_info in stream_info_db.RangeIter():
|
||||
# public_key, key, name, next_announce_time = json.loads(stream_info)
|
||||
# c.execute("insert into live_streams values (?, ?, ?, ?, ?)",
|
||||
# (stream_hash, public_key, key, name, next_announce_time))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
|
||||
# b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
# position, revision, iv, length, signature = json.loads(blob_info)
|
||||
# c.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
|
||||
# (b_h, s_h, position, revision, iv, length, signature))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
|
||||
# c.execute("insert into live_stream_descriptors values (?, ?)",
|
||||
# (sd_blob_hash, stream_hash))
|
||||
# db_conn.commit()
|
||||
# db_conn.close()
|
||||
#
|
||||
#
|
||||
# def migrate_ptc_db(old_db_dir, new_db_dir):
|
||||
# old_db_path = os.path.join(old_db_dir, "ptcwallet.db")
|
||||
# if not os.path.exists(old_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_db_path)
|
||||
# try:
|
||||
# p_key = old_db.Get("private_key")
|
||||
# new_db = unqlite.UnQLite(os.path.join(new_db_dir, "ptcwallet.db"))
|
||||
# new_db['private_key'] = p_key
|
||||
# except KeyError:
|
||||
# pass
|
||||
#
|
||||
#
|
||||
# def migrate_lbryfile_manager_db(old_db_dir, new_db_dir):
|
||||
# old_db_path = os.path.join(old_db_dir, "lbryfiles.db")
|
||||
# if not os.path.exists(old_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_db_path)
|
||||
# new_db = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
|
||||
# c = new_db.cursor()
|
||||
# c.execute("create table if not exists lbry_file_options (" +
|
||||
# " blob_data_rate real, " +
|
||||
# " status text," +
|
||||
# " stream_hash text,"
|
||||
# " foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
# ")")
|
||||
# new_db.commit()
|
||||
# FILE_STATUS = "t"
|
||||
# FILE_OPTIONS = "o"
|
||||
# c = new_db.cursor()
|
||||
# for k, v in old_db.RangeIter():
|
||||
# key_type, stream_hash = json.loads(k)
|
||||
# if key_type == FILE_STATUS:
|
||||
# try:
|
||||
# rate = json.loads(old_db.Get(json.dumps((FILE_OPTIONS, stream_hash))))[0]
|
||||
# except KeyError:
|
||||
# rate = None
|
||||
# c.execute("insert into lbry_file_options values (?, ?, ?)",
|
||||
# (rate, v, stream_hash))
|
||||
# new_db.commit()
|
||||
# new_db.close()
|
||||
#
|
||||
#
|
||||
# def migrate_settings_db(old_db_dir, new_db_dir):
|
||||
# old_settings_db_path = os.path.join(old_db_dir, "settings.db")
|
||||
# if not os.path.exists(old_settings_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_settings_db_path)
|
||||
# new_db = unqlite.UnQLite(os.path.join(new_db_dir, "settings.db"))
|
||||
# for k, v in old_db.RangeIter():
|
||||
# new_db[k] = v
|
||||
#
|
||||
#
|
||||
# def migrate_repeater_db(old_db_dir, new_db_dir):
|
||||
# old_repeater_db_path = os.path.join(old_db_dir, "valuable_blobs.db")
|
||||
# if not os.path.exists(old_repeater_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_repeater_db_path)
|
||||
# info_db = sqlite3.connect(os.path.join(new_db_dir, "blind_info.db"))
|
||||
# peer_db = sqlite3.connect(os.path.join(new_db_dir, "blind_peers.db"))
|
||||
# unql_db = unqlite.UnQLite(os.path.join(new_db_dir, "blind_settings.db"))
|
||||
# BLOB_INFO_TYPE = 'b'
|
||||
# SETTING_TYPE = 's'
|
||||
# PEER_TYPE = 'p'
|
||||
# info_c = info_db.cursor()
|
||||
# info_c.execute("create table if not exists valuable_blobs (" +
|
||||
# " blob_hash text primary key, " +
|
||||
# " blob_length integer, " +
|
||||
# " reference text, " +
|
||||
# " peer_host text, " +
|
||||
# " peer_port integer, " +
|
||||
# " peer_score text" +
|
||||
# ")")
|
||||
# info_db.commit()
|
||||
# peer_c = peer_db.cursor()
|
||||
# peer_c.execute("create table if not exists approved_peers (" +
|
||||
# " ip_address text, " +
|
||||
# " port integer" +
|
||||
# ")")
|
||||
# peer_db.commit()
|
||||
# info_c = info_db.cursor()
|
||||
# peer_c = peer_db.cursor()
|
||||
# for k, v in old_db.RangeIter():
|
||||
# key_type, key_rest = json.loads(k)
|
||||
# if key_type == PEER_TYPE:
|
||||
# host, port = key_rest
|
||||
# peer_c.execute("insert into approved_peers values (?, ?)",
|
||||
# (host, port))
|
||||
# elif key_type == SETTING_TYPE:
|
||||
# unql_db[key_rest] = v
|
||||
# elif key_type == BLOB_INFO_TYPE:
|
||||
# blob_hash = key_rest
|
||||
# length, reference, peer_host, peer_port, peer_score = json.loads(v)
|
||||
# info_c.execute("insert into valuable_blobs values (?, ?, ?, ?, ?, ?)",
|
||||
# (blob_hash, length, reference, peer_host, peer_port, peer_score))
|
||||
# info_db.commit()
|
||||
# peer_db.commit()
|
||||
# info_db.close()
|
||||
# peer_db.close()
|
||||
@@ -0,0 +1,76 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
UNSET_NOUT = -1
|
||||
|
||||
def do_migration(db_dir):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
def migrate_blockchainname_db(db_dir):
|
||||
blockchainname_db = os.path.join(db_dir, "blockchainname.db")
|
||||
# skip migration on fresh installs
|
||||
if not os.path.isfile(blockchainname_db):
|
||||
return
|
||||
temp_db = sqlite3.connect(":memory:")
|
||||
db_file = sqlite3.connect(blockchainname_db)
|
||||
file_cursor = db_file.cursor()
|
||||
mem_cursor = temp_db.cursor()
|
||||
|
||||
mem_cursor.execute("create table if not exists name_metadata ("
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer, "
|
||||
" sd_hash text)")
|
||||
mem_cursor.execute("create table if not exists claim_ids ("
|
||||
" claimId text, "
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer)")
|
||||
temp_db.commit()
|
||||
|
||||
name_metadata = file_cursor.execute("select * from name_metadata").fetchall()
|
||||
claim_metadata = file_cursor.execute("select * from claim_ids").fetchall()
|
||||
|
||||
# fill n as V1_UNSET_NOUT, Wallet.py will be responsible for filling in correct n
|
||||
for name, txid, sd_hash in name_metadata:
|
||||
mem_cursor.execute(
|
||||
"insert into name_metadata values (?, ?, ?, ?) ",
|
||||
(name, txid, UNSET_NOUT, sd_hash))
|
||||
|
||||
for claim_id, name, txid in claim_metadata:
|
||||
mem_cursor.execute(
|
||||
"insert into claim_ids values (?, ?, ?, ?)",
|
||||
(claim_id, name, txid, UNSET_NOUT))
|
||||
temp_db.commit()
|
||||
|
||||
new_name_metadata = mem_cursor.execute("select * from name_metadata").fetchall()
|
||||
new_claim_metadata = mem_cursor.execute("select * from claim_ids").fetchall()
|
||||
|
||||
file_cursor.execute("drop table name_metadata")
|
||||
file_cursor.execute("create table name_metadata ("
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer, "
|
||||
" sd_hash text)")
|
||||
|
||||
for name, txid, n, sd_hash in new_name_metadata:
|
||||
file_cursor.execute("insert into name_metadata values (?, ?, ?, ?) ", (name, txid, n, sd_hash))
|
||||
|
||||
file_cursor.execute("drop table claim_ids")
|
||||
file_cursor.execute("create table claim_ids ("
|
||||
" claimId text, "
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer)")
|
||||
|
||||
for claim_id, name, txid, n in new_claim_metadata:
|
||||
file_cursor.execute("insert into claim_ids values (?, ?, ?, ?)", (claim_id, name, txid, n))
|
||||
|
||||
db_file.commit()
|
||||
db_file.close()
|
||||
temp_db.close()
|
||||
@@ -8,10 +8,7 @@
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import UserDict
|
||||
#import sqlite3
|
||||
import cPickle as pickle
|
||||
import time
|
||||
import os
|
||||
import constants
|
||||
|
||||
|
||||
|
||||
@@ -99,7 +99,10 @@ class Bencode(Encoding):
|
||||
"""
|
||||
if len(data) == 0:
|
||||
raise DecodeError, 'Cannot decode empty string'
|
||||
return self._decodeRecursive(data)[0]
|
||||
try:
|
||||
return self._decodeRecursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError, e.message
|
||||
|
||||
@staticmethod
|
||||
def _decodeRecursive(data, startIndex=0):
|
||||
|
||||
+24
-15
@@ -7,9 +7,9 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import hashlib, random, struct, time, math, binascii
|
||||
import hashlib, random, struct, time, binascii
|
||||
import argparse
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import defer, error
|
||||
import constants
|
||||
import routingtable
|
||||
import datastore
|
||||
@@ -22,6 +22,9 @@ from hashwatcher import HashWatcher
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpcmethod(func):
|
||||
""" Decorator to expose Node methods as remote procedure calls
|
||||
|
||||
@@ -139,7 +142,13 @@ class Node(object):
|
||||
"""
|
||||
# Prepare the underlying Kademlia protocol
|
||||
if self.port is not None:
|
||||
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol) #IGNORE:E1101
|
||||
try:
|
||||
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running." % str(e))
|
||||
#IGNORE:E1101
|
||||
# Create temporary contact information for the list of addresses of known nodes
|
||||
if knownNodeAddresses != None:
|
||||
bootstrapContacts = []
|
||||
@@ -231,12 +240,12 @@ class Node(object):
|
||||
known_nodes = {}
|
||||
|
||||
def log_error(err, n):
|
||||
logging.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
|
||||
logging.error(binascii.hexlify(err.getErrorMessage()))
|
||||
logging.error(err.getTraceback())
|
||||
log.debug("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
|
||||
log.debug(err.getErrorMessage())
|
||||
log.debug(err.getTraceback())
|
||||
|
||||
def log_success(res):
|
||||
logging.debug("Response to store request: %s", str(res))
|
||||
log.debug("Response to store request: %s", str(res))
|
||||
return res
|
||||
|
||||
def announce_to_peer(responseTuple):
|
||||
@@ -332,7 +341,7 @@ class Node(object):
|
||||
# originalPublisherID = self.id
|
||||
#
|
||||
# def log_error(err):
|
||||
# logging.error(err.getErrorMessage())
|
||||
# log.error(err.getErrorMessage())
|
||||
#
|
||||
# # Prepare a callback for doing "STORE" RPC calls
|
||||
# def executeStoreRPCs(nodes):
|
||||
@@ -507,7 +516,7 @@ class Node(object):
|
||||
else:
|
||||
raise TypeError, 'No NodeID given. Therefore we can\'t store this node'
|
||||
|
||||
if self_store is True and self.externalIP is not None:
|
||||
if self_store is True and self.externalIP:
|
||||
contact = Contact(self.id, self.externalIP, self.port, None, None)
|
||||
compact_ip = contact.compact_ip()
|
||||
elif '_rpcNodeContact' in kwargs:
|
||||
@@ -563,7 +572,7 @@ class Node(object):
|
||||
def findNode(self, key, **kwargs):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
|
||||
@@ -581,17 +590,17 @@ class Node(object):
|
||||
contacts = self._routingTable.findCloseNodes(key, constants.k, rpcSenderID)
|
||||
contactTriples = []
|
||||
for contact in contacts:
|
||||
contactTriples.append( (contact.id, contact.address, contact.port) )
|
||||
contactTriples.append((contact.id, contact.address, contact.port))
|
||||
return contactTriples
|
||||
|
||||
@rpcmethod
|
||||
def findValue(self, key, **kwargs):
|
||||
""" Return the value associated with the specified key if present in
|
||||
this node's data, otherwise execute FIND_NODE for the key
|
||||
|
||||
|
||||
@param key: The hashtable key of the data to return
|
||||
@type key: str
|
||||
|
||||
|
||||
@return: A dictionary containing the requested key/value pair,
|
||||
or a list of contact triples closest to the requested key.
|
||||
@rtype: dict or list
|
||||
@@ -756,7 +765,7 @@ class Node(object):
|
||||
searchIteration()
|
||||
|
||||
def log_error(err):
|
||||
logging.error(err.getErrorMessage())
|
||||
log.error(err.getErrorMessage())
|
||||
|
||||
# Send parallel, asynchronous FIND_NODE RPCs to the shortlist of contacts
|
||||
def searchIteration():
|
||||
@@ -1008,4 +1017,4 @@ def main():
|
||||
twisted.internet.reactor.run()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main()
|
||||
+34
-11
@@ -7,6 +7,8 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import logging
|
||||
import binascii
|
||||
import time
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
@@ -19,10 +21,20 @@ import msgtypes
|
||||
import msgformat
|
||||
from contact import Contact
|
||||
|
||||
|
||||
reactor = twisted.internet.reactor
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TimeoutError(Exception):
|
||||
""" Raised when a RPC times out """
|
||||
def __init__(self, remote_contact_id):
|
||||
# remote_contact_id is a binary blob so we need to convert it
|
||||
# into something more readable
|
||||
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
|
||||
Exception.__init__(self, msg)
|
||||
self.remote_contact_id = remote_contact_id
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
@@ -110,10 +122,13 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
except encoding.DecodeError:
|
||||
# We received some rubbish here
|
||||
return
|
||||
|
||||
except IndexError:
|
||||
log.warning("Couldn't decode dht datagram from %s", address)
|
||||
return
|
||||
|
||||
message = self._translator.fromPrimitive(msgPrimitive)
|
||||
remoteContact = Contact(message.nodeID, address[0], address[1], self)
|
||||
|
||||
|
||||
# Refresh the remote node's details in the local node's k-buckets
|
||||
self._node.addContact(remoteContact)
|
||||
|
||||
@@ -129,7 +144,9 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
del self._sentMessages[message.id]
|
||||
|
||||
if hasattr(df, '_rpcRawResponse'):
|
||||
# The RPC requested that the raw response message and originating address be returned; do not interpret it
|
||||
# The RPC requested that the raw response message
|
||||
# and originating address be returned; do not
|
||||
# interpret it
|
||||
df.callback((message, address))
|
||||
elif isinstance(message, msgtypes.ErrorMessage):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
@@ -161,7 +178,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
def _send(self, data, rpcID, address):
|
||||
""" Transmit the specified data over UDP, breaking it up into several
|
||||
packets if necessary
|
||||
|
||||
|
||||
If the data is spread over multiple UDP datagrams, the packets have the
|
||||
following structure::
|
||||
| | | | | |||||||||||| 0x00 |
|
||||
@@ -169,7 +186,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
| type ID | of packets |of this packet | | indicator|
|
||||
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
|
||||
| | | | | |||||||||||| |
|
||||
|
||||
|
||||
@note: The header used for breaking up large data segments will
|
||||
possibly be moved out of the KademliaProtocol class in the
|
||||
future, into something similar to a message translator/encoder
|
||||
@@ -249,7 +266,8 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
try:
|
||||
##try:
|
||||
## # Try to pass the sender's node id to the function...
|
||||
result = func(*args, **{'_rpcNodeID': senderContact.id, '_rpcNodeContact': senderContact})
|
||||
kwargs = {'_rpcNodeID': senderContact.id, '_rpcNodeContact': senderContact}
|
||||
result = func(*args, **kwargs)
|
||||
##except TypeError:
|
||||
## # ...or simply call it if that fails
|
||||
## result = func(*args)
|
||||
@@ -259,7 +277,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df.callback(result)
|
||||
else:
|
||||
# No such exposed method
|
||||
df.errback( failure.Failure( AttributeError('Invalid method: %s' % method) ) )
|
||||
df.errback(failure.Failure(AttributeError('Invalid method: %s' % method)))
|
||||
|
||||
def _msgTimeout(self, messageID):
|
||||
""" Called when an RPC request message times out """
|
||||
@@ -287,19 +305,24 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df.errback(failure.Failure(TimeoutError(remoteContactID)))
|
||||
else:
|
||||
# This should never be reached
|
||||
print "ERROR: deferred timed out, but is not present in sent messages list!"
|
||||
log.error("deferred timed out, but is not present in sent messages list!")
|
||||
|
||||
def stopProtocol(self):
|
||||
""" Called when the transport is disconnected.
|
||||
|
||||
|
||||
Will only be called once, after all ports are disconnected.
|
||||
"""
|
||||
log.info('Stopping dht')
|
||||
for key in self._callLaterList.keys():
|
||||
try:
|
||||
if key > time.time():
|
||||
log.info('Cancelling %s', self._callLaterList[key])
|
||||
self._callLaterList[key].cancel()
|
||||
except Exception, e:
|
||||
print e
|
||||
log.exception('Failed to cancel %s', self._callLaterList[key])
|
||||
del self._callLaterList[key]
|
||||
#TODO: test: do we really need the reactor.iterate() call?
|
||||
|
||||
# not sure why this is needed, but taking this out sometimes causes
|
||||
# exceptions.AttributeError: 'Port' object has no attribute 'socket'
|
||||
# to happen on shutdown
|
||||
reactor.iterate()
|
||||
@@ -419,4 +419,5 @@ class OptimizedTreeRoutingTable(TreeRoutingTable):
|
||||
# Replace this stale contact with one from our replacemnent cache, if we have any
|
||||
if self._replacementCache.has_key(bucketIndex):
|
||||
if len(self._replacementCache[bucketIndex]) > 0:
|
||||
self._buckets[bucketIndex].addContact( self._replacementCache[bucketIndex].pop() )
|
||||
self._buckets[bucketIndex].addContact(
|
||||
self._replacementCache[bucketIndex].pop())
|
||||
+12
-9
@@ -6,6 +6,9 @@ import sys
|
||||
from lbrynet.core.utils import generate_id
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def print_usage():
|
||||
print "Usage:\n%s UDP_PORT KNOWN_NODE_IP KNOWN_NODE_PORT HASH"
|
||||
|
||||
@@ -13,15 +16,15 @@ def print_usage():
|
||||
def join_network(udp_port, known_nodes):
|
||||
lbryid = generate_id()
|
||||
|
||||
logging.info('Creating Node...')
|
||||
log.info('Creating Node...')
|
||||
node = Node(udpPort=udp_port, lbryid=lbryid)
|
||||
|
||||
logging.info('Joining network...')
|
||||
log.info('Joining network...')
|
||||
d = node.joinNetwork(known_nodes)
|
||||
|
||||
def log_network_size():
|
||||
logging.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
|
||||
logging.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
|
||||
log.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
|
||||
log.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
|
||||
|
||||
d.addCallback(lambda _: log_network_size())
|
||||
|
||||
@@ -36,7 +39,7 @@ def get_hosts(node, h):
|
||||
print "Hosts returned from the DHT: "
|
||||
print hosts
|
||||
|
||||
logging.info("Looking up %s", h)
|
||||
log.info("Looking up %s", h)
|
||||
d = node.getPeersForBlob(h)
|
||||
d.addCallback(print_hosts)
|
||||
return d
|
||||
@@ -48,9 +51,9 @@ def announce_hash(node, h):
|
||||
def log_results(results):
|
||||
for success, result in results:
|
||||
if success:
|
||||
logging.info("Succeeded: %s", str(result))
|
||||
log.info("Succeeded: %s", str(result))
|
||||
else:
|
||||
logging.info("Failed: %s", str(result.getErrorMessage()))
|
||||
log.info("Failed: %s", str(result.getErrorMessage()))
|
||||
|
||||
d.addCallback(log_results)
|
||||
return d
|
||||
@@ -80,11 +83,11 @@ def run_dht_script(dht_func):
|
||||
d.addCallback(run_dht_func)
|
||||
|
||||
def log_err(err):
|
||||
logging.error("An error occurred: %s", err.getTraceback())
|
||||
log.error("An error occurred: %s", err.getTraceback())
|
||||
return err
|
||||
|
||||
def shut_down():
|
||||
logging.info("Shutting down")
|
||||
log.info("Shutting down")
|
||||
reactor.stop()
|
||||
|
||||
d.addErrback(log_err)
|
||||
|
||||
+4
-4
@@ -20,7 +20,7 @@
|
||||
|
||||
|
||||
|
||||
import os, sys, time, signal, hashlib, random
|
||||
import sys, hashlib, random
|
||||
import twisted.internet.reactor
|
||||
from lbrynet.dht.node import Node
|
||||
#from entangled.kademlia.datastore import SQLiteDataStore
|
||||
@@ -106,7 +106,7 @@ def stop():
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
import sys, os
|
||||
import sys
|
||||
if len(sys.argv) < 2:
|
||||
print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0]
|
||||
print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0]
|
||||
@@ -145,9 +145,9 @@ if __name__ == '__main__':
|
||||
# If you wish to have a pure Kademlia network, use the entangled.kademlia.node.Node class instead
|
||||
print 'Creating Node...'
|
||||
#node = EntangledNode( udpPort=int(sys.argv[1]), dataStore=dataStore )
|
||||
node = Node( udpPort=int(sys.argv[1]), lbryid=lbryid)
|
||||
node = Node(udpPort=int(sys.argv[1]), lbryid=lbryid)
|
||||
|
||||
# Schedule the node to join the Kademlia/Entangled DHT
|
||||
# Schedule the node to join the Kademlia/Entangled DHT
|
||||
node.joinNetwork(knownNodes)
|
||||
# Schedule the "storeValue() call to be invoked after 2.5 seconds, using KEY and VALUE as arguments
|
||||
#twisted.internet.reactor.callLater(2.5, storeValue, KEY, VALUE)
|
||||
|
||||
+84
-11
@@ -445,10 +445,7 @@ class IQueryHandlerFactory(Interface):
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""
|
||||
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
|
||||
"""
|
||||
class IStreamDownloaderOptions(Interface):
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
"""
|
||||
Return the list of options that can be used to modify IStreamDownloader behavior
|
||||
@@ -459,8 +456,28 @@ class IStreamDownloaderFactory(Interface):
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: [(option_description, default)]
|
||||
@rtype: [(string, string)]
|
||||
@return: [DownloadOption]
|
||||
@rtype: [DownloadOption]
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""
|
||||
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
|
||||
"""
|
||||
|
||||
def can_download(self, sd_validator, payment_rate_manager):
|
||||
"""
|
||||
Decide whether the downloaders created by this factory can download the stream described by sd_validator
|
||||
|
||||
@param sd_validator: object containing stream metadata
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: True if the downloaders can download the stream, False otherwise
|
||||
@rtype: bool
|
||||
"""
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
@@ -470,10 +487,10 @@ class IStreamDownloaderFactory(Interface):
|
||||
@param sd_validator: object containing stream metadata which will be given to the IStreamDownloader
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param options: a list of strings that will be used by the IStreamDownloaderFactory to
|
||||
@param options: a list of values that will be used by the IStreamDownloaderFactory to
|
||||
construct the IStreamDownloader. the options are in the same order as they were given
|
||||
by get_downloader_options.
|
||||
@type options: [string]
|
||||
@type options: [Object]
|
||||
|
||||
@param payment_rate_manager: the PaymentRateManager which the IStreamDownloader should use.
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
@@ -504,7 +521,7 @@ class IStreamDownloader(Interface):
|
||||
@rtype: Deferred which fires with anything
|
||||
"""
|
||||
|
||||
def insufficient_funds(self):
|
||||
def insufficient_funds(self, err):
|
||||
"""
|
||||
this function informs the stream downloader that funds are too low to finish downloading.
|
||||
|
||||
@@ -531,13 +548,13 @@ class IStreamDescriptorValidator(Interface):
|
||||
"""
|
||||
|
||||
|
||||
class ILBRYWallet(Interface):
|
||||
class IWallet(Interface):
|
||||
"""
|
||||
Send and receive payments.
|
||||
|
||||
To send a payment, a payment reservation must be obtained first. This guarantees that a payment
|
||||
isn't promised if it can't be paid. When the service in question is rendered, the payment
|
||||
reservation must be given to the ILBRYWallet along with the final price. The reservation can also
|
||||
reservation must be given to the IWallet along with the final price. The reservation can also
|
||||
be canceled.
|
||||
"""
|
||||
def stop(self):
|
||||
@@ -630,4 +647,60 @@ class ILBRYWallet(Interface):
|
||||
@type amount: float
|
||||
|
||||
@return: None
|
||||
"""
|
||||
|
||||
|
||||
class IBlobPriceModel(Interface):
|
||||
"""
|
||||
A blob price model
|
||||
|
||||
Used by INegotiationStrategy classes
|
||||
"""
|
||||
|
||||
def calculate_price(self, blob):
|
||||
"""
|
||||
Calculate the price for a blob
|
||||
|
||||
@param blob: a blob hash
|
||||
@type blob: str
|
||||
|
||||
@return: blob price target
|
||||
@type: Decimal
|
||||
"""
|
||||
|
||||
|
||||
class INegotiationStrategy(Interface):
|
||||
"""
|
||||
Strategy to negotiate download payment rates
|
||||
"""
|
||||
|
||||
def make_offer(self, peer, blobs):
|
||||
"""
|
||||
Make a rate offer for the given peer and blobs
|
||||
|
||||
@param peer: peer to make an offer to
|
||||
@type: str
|
||||
|
||||
@param blobs: blob hashes to make an offer for
|
||||
@type: list
|
||||
|
||||
@return: rate offer
|
||||
@rtype: Offer
|
||||
"""
|
||||
|
||||
def respond_to_offer(self, offer, peer, blobs):
|
||||
"""
|
||||
Respond to a rate offer given by a peer
|
||||
|
||||
@param offer: offer to reply to
|
||||
@type: Offer
|
||||
|
||||
@param peer: peer to make an offer to
|
||||
@type: str
|
||||
|
||||
@param blobs: blob hashes to make an offer for
|
||||
@type: list
|
||||
|
||||
@return: accepted, rejected, or unset offer
|
||||
@rtype: Offer
|
||||
"""
|
||||
@@ -0,0 +1,336 @@
|
||||
import logging
|
||||
import sqlite3
|
||||
import os
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DBEncryptedFileMetadataManager(object):
|
||||
"""Store and provide access to LBRY file metadata using sqlite"""
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
|
||||
def setup(self):
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return self._get_all_streams()
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
d = self._store_stream(stream_hash, file_name, key, suggested_file_name)
|
||||
d.addCallback(lambda _: self.add_blobs_to_stream(stream_hash, blobs))
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return self._get_stream_info(stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return self._check_if_stream_exists(stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return self._delete_stream(stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
log.debug("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
d1 = defer.succeed(None)
|
||||
if end_blob is not None:
|
||||
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
d2 = defer.succeed(None)
|
||||
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
|
||||
def get_positions(results):
|
||||
start_num = None
|
||||
end_num = None
|
||||
if results[0][0] is True:
|
||||
start_num = results[0][1]
|
||||
if results[1][0] is True:
|
||||
end_num = results[1][1]
|
||||
return start_num, end_num
|
||||
|
||||
dl.addCallback(get_positions)
|
||||
return dl
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = get_positions_of_start_and_end()
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return self._get_stream_of_blobhash(blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return self._get_sd_blob_hashes_for_stream(stream_hash)
|
||||
|
||||
def _open_db(self):
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool("sqlite3", (os.path.join(self.db_dir, "lbryfile_info.db")),
|
||||
check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists lbry_files (" +
|
||||
" stream_hash text primary key, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" suggested_file_name text" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_stream(self, stream_hash):
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
|
||||
def do_delete(transaction, s_h):
|
||||
transaction.execute("delete from lbry_files where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_blobs where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_descriptors where stream_hash = ?", (s_h,))
|
||||
|
||||
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _store_stream(self, stream_hash, name, key, suggested_file_name):
|
||||
d = self.db_conn.runQuery("insert into lbry_files values (?, ?, ?, ?)",
|
||||
(stream_hash, key, name, suggested_file_name))
|
||||
|
||||
def check_duplicate(err):
|
||||
if err.check(sqlite3.IntegrityError):
|
||||
raise DuplicateStreamHashError(stream_hash)
|
||||
return err
|
||||
|
||||
d.addErrback(check_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_streams(self):
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files")
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_info(self, stream_hash):
|
||||
d = self.db_conn.runQuery("select key, stream_name, suggested_file_name from lbry_files where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda r: True if len(r) else False)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
d = self.db_conn.runQuery("select position from lbry_file_blobs where stream_hash = ? and blob_hash = ?",
|
||||
(stream_hash, blob_hash))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
params = []
|
||||
q_string = "select * from ("
|
||||
q_string += " select blob_hash, position, iv, length from lbry_file_blobs "
|
||||
q_string += " where stream_hash = ? "
|
||||
params.append(stream_hash)
|
||||
if start_num is not None:
|
||||
q_string += " and position > ? "
|
||||
params.append(start_num)
|
||||
if end_num is not None:
|
||||
q_string += " and position < ? "
|
||||
params.append(end_num)
|
||||
q_string += " order by position "
|
||||
if reverse is True:
|
||||
q_string += " DESC "
|
||||
if count is not None:
|
||||
q_string += " limit ? "
|
||||
params.append(count)
|
||||
q_string += ") order by position"
|
||||
# Order by position is done twice so that it always returns them from lowest position to
|
||||
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
|
||||
return self.db_conn.runQuery(q_string, tuple(params))
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
|
||||
def add_blobs(transaction):
|
||||
for blob_info in blob_infos:
|
||||
try:
|
||||
transaction.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
|
||||
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
|
||||
blob_info.iv, blob_info.length))
|
||||
except sqlite3.IntegrityError:
|
||||
if ignore_duplicate_error is False:
|
||||
raise
|
||||
|
||||
return self.db_conn.runInteraction(add_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_file_blobs where blob_hash = ?",
|
||||
(blob_hash,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
log.info("Saving sd blob hash %s to stream hash %s", str(sd_blob_hash), str(stream_hash))
|
||||
d = self.db_conn.runQuery("insert into lbry_file_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
|
||||
def ignore_duplicate(err):
|
||||
err.trap(sqlite3.IntegrityError)
|
||||
log.info("sd blob hash already known")
|
||||
|
||||
d.addErrback(ignore_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
log.debug("Looking up sd blob hashes for stream hash %s", str(stream_hash))
|
||||
d = self.db_conn.runQuery("select sd_blob_hash from lbry_file_descriptors where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
|
||||
class TempEncryptedFileMetadataManager(object):
|
||||
def __init__(self):
|
||||
self.streams = {}
|
||||
self.stream_blobs = {}
|
||||
self.sd_files = {}
|
||||
|
||||
def setup(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return defer.succeed(self.streams.keys())
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
self.streams[stream_hash] = {'suggested_file_name': suggested_file_name,
|
||||
'stream_name': file_name,
|
||||
'key': key}
|
||||
d = self.add_blobs_to_stream(stream_hash, blobs)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
stream_info = self.streams[stream_hash]
|
||||
return defer.succeed([stream_info['key'], stream_info['stream_name'],
|
||||
stream_info['suggested_file_name']])
|
||||
return defer.succeed(None)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
del self.streams[stream_hash]
|
||||
for (s_h, b_h) in self.stream_blobs.keys():
|
||||
if s_h == stream_hash:
|
||||
del self.stream_blobs[(s_h, b_h)]
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
assert stream_hash in self.streams, "Can't add blobs to a stream that isn't known"
|
||||
for blob in blobs:
|
||||
info = {}
|
||||
info['blob_num'] = blob.blob_num
|
||||
info['length'] = blob.length
|
||||
info['iv'] = blob.iv
|
||||
self.stream_blobs[(stream_hash, blob.blob_hash)] = info
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
end_num = None
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num, count, reverse)
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
for (s_h, b_h) in self.stream_blobs.iterkeys():
|
||||
if b_h == blob_hash:
|
||||
return defer.succeed(s_h)
|
||||
return defer.succeed(None)
|
||||
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for (s_h, b_h), info in self.stream_blobs.iteritems():
|
||||
if stream_hash == s_h:
|
||||
position = info['blob_num']
|
||||
length = info['length']
|
||||
iv = info['iv']
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return defer.succeed(blob_infos)
|
||||
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
if (stream_hash, blob_hash) in self.stream_blobs:
|
||||
return defer.succeed(self.stream_blobs[(stream_hash, blob_hash)]['blob_num'])
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.sd_files[sd_blob_hash] = stream_hash
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return defer.succeed([sd_hash for sd_hash, s_h in self.sd_files.iteritems() if stream_hash == s_h])
|
||||
@@ -1,268 +0,0 @@
|
||||
import logging
|
||||
import leveldb
|
||||
import json
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
|
||||
|
||||
class DBLBRYFileMetadataManager(object):
|
||||
"""Store and provide access to LBRY file metadata using leveldb files"""
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
|
||||
def setup(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
|
||||
def stop(self):
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return threads.deferToThread(self._get_all_streams)
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
d = threads.deferToThread(self._store_stream, stream_hash, file_name, key, suggested_file_name)
|
||||
d.addCallback(lambda _: self.add_blobs_to_stream(stream_hash, blobs))
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return threads.deferToThread(self._get_stream_info, stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_stream, stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
|
||||
def add_blobs():
|
||||
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
return threads.deferToThread(add_blobs)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
logging.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
end_num = None
|
||||
return start_num, end_num
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = threads.deferToThread(get_positions_of_start_and_end)
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
|
||||
|
||||
def _open_db(self):
|
||||
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_info.db"))
|
||||
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_blob.db"))
|
||||
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_desc.db"))
|
||||
|
||||
def _delete_stream(self, stream_hash):
|
||||
desc_batch = leveldb.WriteBatch()
|
||||
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
desc_batch.Delete(sd_blob_hash)
|
||||
self.stream_desc_db.Write(desc_batch, sync=True)
|
||||
|
||||
blob_batch = leveldb.WriteBatch()
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
blob_batch.Delete(blob_hash_stream_hash)
|
||||
self.stream_blob_db.Write(blob_batch, sync=True)
|
||||
|
||||
stream_batch = leveldb.WriteBatch()
|
||||
for s_h, stream_info in self.stream_info_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
stream_batch.Delete(s_h)
|
||||
self.stream_info_db.Write(stream_batch, sync=True)
|
||||
|
||||
def _store_stream(self, stream_hash, name, key, suggested_file_name):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
|
||||
except KeyError:
|
||||
pass
|
||||
self.stream_info_db.Put(stream_hash, json.dumps((key, name, suggested_file_name)), sync=True)
|
||||
|
||||
def _get_all_streams(self):
|
||||
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
|
||||
|
||||
def _get_stream_info(self, stream_hash):
|
||||
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
|
||||
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
|
||||
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
|
||||
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
position, iv, length = json.loads(blob_info)
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return blob_infos
|
||||
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_info in blob_infos:
|
||||
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
|
||||
try:
|
||||
self.stream_blob_db.Get(blob_hash_stream_hash)
|
||||
if ignore_duplicate_error is False:
|
||||
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
batch.Put(blob_hash_stream_hash,
|
||||
json.dumps((blob_info.blob_num,
|
||||
blob_info.iv,
|
||||
blob_info.length)))
|
||||
self.stream_blob_db.Write(batch, sync=True)
|
||||
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if blob_hash == b_h:
|
||||
return s_h
|
||||
return None
|
||||
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
|
||||
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
|
||||
|
||||
|
||||
class TempLBRYFileMetadataManager(object):
|
||||
def __init__(self):
|
||||
self.streams = {}
|
||||
self.stream_blobs = {}
|
||||
self.sd_files = {}
|
||||
|
||||
def setup(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return defer.succeed(self.streams.keys())
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
self.streams[stream_hash] = {'suggested_file_name': suggested_file_name,
|
||||
'stream_name': file_name,
|
||||
'key': key}
|
||||
d = self.add_blobs_to_stream(stream_hash, blobs)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
stream_info = self.streams[stream_hash]
|
||||
return defer.succeed([stream_info['key'], stream_info['stream_name'],
|
||||
stream_info['suggested_file_name']])
|
||||
return defer.succeed(None)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
if stream_hash in self.streams:
|
||||
del self.streams[stream_hash]
|
||||
for (s_h, b_h) in self.stream_blobs.keys():
|
||||
if s_h == stream_hash:
|
||||
del self.stream_blobs[(s_h, b_h)]
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
assert stream_hash in self.streams, "Can't add blobs to a stream that isn't known"
|
||||
for blob in blobs:
|
||||
info = {}
|
||||
info['blob_num'] = blob.blob_num
|
||||
info['length'] = blob.length
|
||||
info['iv'] = blob.iv
|
||||
self.stream_blobs[(stream_hash, blob.blob_hash)] = info
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
end_num = None
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num, count, reverse)
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
for (s_h, b_h) in self.stream_blobs.iterkeys():
|
||||
if b_h == blob_hash:
|
||||
return defer.succeed(s_h)
|
||||
return defer.succeed(None)
|
||||
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for (s_h, b_h), info in self.stream_blobs.iteritems():
|
||||
if stream_hash == s_h:
|
||||
position = info['blob_num']
|
||||
length = info['length']
|
||||
iv = info['iv']
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return defer.succeed(blob_infos)
|
||||
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
if (stream_hash, blob_hash) in self.stream_blobs:
|
||||
return defer.succeed(self.stream_blobs[(stream_hash, blob_hash)]['blob_num'])
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.sd_files[sd_blob_hash] = stream_hash
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return defer.succeed([sd_hash for sd_hash, s_h in self.sd_files.iteritems() if stream_hash == s_h])
|
||||
@@ -2,15 +2,20 @@ import binascii
|
||||
import logging
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from twisted.internet import defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from twisted.internet import defer, threads
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
|
||||
from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
|
||||
import os
|
||||
|
||||
|
||||
LBRYFileStreamType = "lbryfile"
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
EncryptedFileStreamType = "lbryfile"
|
||||
|
||||
|
||||
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
logging.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
log.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
hex_stream_name = sd_info['stream_name']
|
||||
key = sd_info['key']
|
||||
stream_hash = sd_info['stream_hash']
|
||||
@@ -26,7 +31,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
blob_num = blob['blob_num']
|
||||
iv = blob['iv']
|
||||
crypt_blobs.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
log.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
d = stream_info_manager.save_stream(stream_hash, hex_stream_name, key,
|
||||
suggested_file_name, crypt_blobs)
|
||||
|
||||
@@ -45,7 +50,7 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
|
||||
|
||||
def format_info(stream_info):
|
||||
fields = {}
|
||||
fields['stream_type'] = LBRYFileStreamType
|
||||
fields['stream_type'] = EncryptedFileStreamType
|
||||
fields['stream_name'] = stream_info[1]
|
||||
fields['key'] = stream_info[0]
|
||||
fields['suggested_file_name'] = stream_info[2]
|
||||
@@ -75,12 +80,53 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
|
||||
return d
|
||||
|
||||
|
||||
class LBRYFileStreamDescriptorValidator(object):
|
||||
def publish_sd_blob(stream_info_manager, blob_manager, stream_hash):
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
|
||||
d = get_sd_info(stream_info_manager, stream_hash, True)
|
||||
d.addCallback(descriptor_writer.create_descriptor)
|
||||
|
||||
def add_sd_blob_to_stream(sd_blob_hash):
|
||||
d = stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
d.addCallback(lambda _: sd_blob_hash)
|
||||
return d
|
||||
|
||||
d.addCallback(add_sd_blob_to_stream)
|
||||
return d
|
||||
|
||||
|
||||
def create_plain_sd(stream_info_manager, stream_hash, file_name, overwrite_existing=False):
|
||||
|
||||
def _get_file_name():
|
||||
actual_file_name = file_name
|
||||
if os.path.exists(actual_file_name):
|
||||
ext_num = 1
|
||||
while os.path.exists(actual_file_name + "_" + str(ext_num)):
|
||||
ext_num += 1
|
||||
actual_file_name = actual_file_name + "_" + str(ext_num)
|
||||
return actual_file_name
|
||||
|
||||
if overwrite_existing is False:
|
||||
d = threads.deferToThread(_get_file_name())
|
||||
else:
|
||||
d = defer.succeed(file_name)
|
||||
|
||||
def do_create(file_name):
|
||||
descriptor_writer = PlainStreamDescriptorWriter(file_name)
|
||||
d = get_sd_info(stream_info_manager, stream_hash, True)
|
||||
d.addCallback(descriptor_writer.create_descriptor)
|
||||
return d
|
||||
|
||||
d.addCallback(do_create)
|
||||
return d
|
||||
|
||||
|
||||
class EncryptedFileStreamDescriptorValidator(object):
|
||||
def __init__(self, raw_info):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
try:
|
||||
hex_stream_name = self.raw_info['stream_name']
|
||||
key = self.raw_info['key']
|
||||
@@ -88,11 +134,10 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
stream_hash = self.raw_info['stream_hash']
|
||||
blobs = self.raw_info['blobs']
|
||||
except KeyError as e:
|
||||
raise ValueError("Invalid stream descriptor. Missing '%s'" % (e.args[0]))
|
||||
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
|
||||
for c in hex_suggested_file_name:
|
||||
if c not in '0123456789abcdef':
|
||||
raise ValueError("Invalid stream descriptor: "
|
||||
"suggested file name is not a hex-encoded string")
|
||||
raise InvalidStreamDescriptorError("Suggested file name is not a hex-encoded string")
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(hex_stream_name)
|
||||
h.update(key)
|
||||
@@ -118,10 +163,11 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
for blob in blobs:
|
||||
blobs_hashsum.update(get_blob_hashsum(blob))
|
||||
if blobs[-1]['length'] != 0:
|
||||
raise ValueError("Improperly formed stream descriptor. Must end with a zero-length blob.")
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
h.update(blobs_hashsum.digest())
|
||||
if h.hexdigest() != stream_hash:
|
||||
raise ValueError("Stream hash does not match stream metadata")
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
log.debug("It is validated")
|
||||
return defer.succeed(True)
|
||||
|
||||
def info_to_show(self):
|
||||
@@ -130,9 +176,15 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
info.append(("stream_size", str(size_so_far)))
|
||||
info.append(("stream_size", str(self.get_length_of_stream())))
|
||||
suggested_file_name = self.raw_info.get("suggested_file_name", None)
|
||||
if suggested_file_name is not None:
|
||||
suggested_file_name = binascii.unhexlify(suggested_file_name)
|
||||
info.append(("suggested_file_name", suggested_file_name))
|
||||
return info
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
return size_so_far
|
||||
@@ -0,0 +1,2 @@
|
||||
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
|
||||
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
|
||||
+121
-51
@@ -3,17 +3,23 @@ import binascii
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbryfile.client.LBRYFileMetadataHandler import LBRYFileMetadataHandler
|
||||
from lbrynet.lbryfile.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
import os
|
||||
from twisted.internet import defer, threads, reactor
|
||||
from twisted.python.procutils import which
|
||||
import logging
|
||||
import traceback
|
||||
|
||||
|
||||
class LBRYFileDownloader(CryptStreamDownloader):
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
@@ -40,9 +46,34 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
def delete_data(self):
|
||||
d1 = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def get_blob_hashes(blob_infos):
|
||||
return [b[0] for b in blob_infos if b[0] is not None]
|
||||
|
||||
d1.addCallback(get_blob_hashes)
|
||||
d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash)
|
||||
|
||||
def combine_blob_hashes(results):
|
||||
blob_hashes = []
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
blob_hashes.extend(result)
|
||||
return blob_hashes
|
||||
|
||||
def delete_blobs(blob_hashes):
|
||||
self.blob_manager.delete_blobs(blob_hashes)
|
||||
return True
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
dl.addCallback(combine_blob_hashes)
|
||||
dl.addCallback(delete_blobs)
|
||||
return dl
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self))
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
@@ -60,6 +91,15 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b[3] for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
if self.download_manager is None:
|
||||
return 0
|
||||
@@ -80,10 +120,10 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
return 0
|
||||
|
||||
def _get_metadata_handler(self, download_manager):
|
||||
return LBRYFileMetadataHandler(self.stream_hash, self.stream_info_manager, download_manager)
|
||||
return EncryptedFileMetadataHandler(self.stream_hash, self.stream_info_manager, download_manager)
|
||||
|
||||
|
||||
class LBRYFileDownloaderFactory(object):
|
||||
class EncryptedFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
@@ -94,37 +134,30 @@ class LBRYFileDownloaderFactory(object):
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.wallet = wallet
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager, **kwargs):
|
||||
if options[0] is not None:
|
||||
payment_rate_manager.float(options[0])
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
|
||||
payment_rate_manager.min_blob_data_payment_rate = options[0]
|
||||
upload_allowed = options[1]
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager, sd_validator.raw_info,
|
||||
upload_allowed)
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager,
|
||||
metadata.validator.raw_info, upload_allowed)
|
||||
d = downloader.set_stream_info()
|
||||
d.addCallback(lambda _: downloader)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
|
||||
@@ -132,17 +165,24 @@ class LBRYFileDownloaderFactory(object):
|
||||
pass
|
||||
|
||||
|
||||
class LBRYFileSaver(LBRYFileDownloader):
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet, download_directory, upload_allowed, file_name=None):
|
||||
LBRYFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
EncryptedFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
|
||||
self.download_directory = download_directory
|
||||
self.file_name = file_name
|
||||
self.file_written_to = None
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
if self.file_written_to is not None:
|
||||
return str(self.file_written_to)
|
||||
else:
|
||||
return str(self.file_name)
|
||||
|
||||
def set_stream_info(self):
|
||||
d = LBRYFileDownloader.set_stream_info(self)
|
||||
d = EncryptedFileDownloader.set_stream_info(self)
|
||||
|
||||
def set_file_name():
|
||||
if self.file_name is None:
|
||||
@@ -154,14 +194,14 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
d.addCallback(lambda _: set_file_name())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
delete_blob_after_finished=not self.upload_allowed)
|
||||
|
||||
def _setup_output(self):
|
||||
def open_file():
|
||||
@@ -171,11 +211,28 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
file_name = "_"
|
||||
if os.path.exists(os.path.join(self.download_directory, file_name)):
|
||||
ext_num = 1
|
||||
|
||||
def _get_file_name(ext):
|
||||
if len(file_name.split(".")):
|
||||
fn = ''.join(file_name.split(".")[:-1])
|
||||
file_ext = ''.join(file_name.split(".")[-1])
|
||||
return fn + "-" + str(ext) + "." + file_ext
|
||||
else:
|
||||
return file_name + "_" + str(ext)
|
||||
|
||||
while os.path.exists(os.path.join(self.download_directory,
|
||||
file_name + "_" + str(ext_num))):
|
||||
_get_file_name(ext_num))):
|
||||
ext_num += 1
|
||||
file_name = file_name + "_" + str(ext_num)
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
|
||||
file_name = _get_file_name(ext_num)
|
||||
try:
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
self.file_written_to = os.path.join(self.download_directory, file_name)
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise ValueError("Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." % str(os.path.join(self.download_directory,
|
||||
file_name)))
|
||||
return threads.deferToThread(open_file)
|
||||
|
||||
def _close_output(self):
|
||||
@@ -200,38 +257,39 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
return self.stream_info_manager.delete_stream(self.stream_hash)
|
||||
|
||||
|
||||
class LBRYFileSaverFactory(LBRYFileDownloaderFactory):
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
wallet, download_directory):
|
||||
LBRYFileDownloaderFactory.__init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
EncryptedFileDownloaderFactory.__init__(self, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, wallet)
|
||||
self.download_directory = download_directory
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
|
||||
return LBRYFileSaver(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
return EncryptedFileSaver(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet,
|
||||
self.download_directory, upload_allowed)
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save"
|
||||
|
||||
|
||||
class LBRYFileOpener(LBRYFileDownloader):
|
||||
class EncryptedFileOpener(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet, upload_allowed):
|
||||
LBRYFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
EncryptedFileDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
|
||||
self.process = None
|
||||
self.process_log = None
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
delete_blob_after_finished=not self.upload_allowed)
|
||||
|
||||
def _setup_output(self):
|
||||
def start_process():
|
||||
@@ -275,10 +333,22 @@ class LBRYFileOpener(LBRYFileDownloader):
|
||||
return self.stream_info_manager.delete_stream(self.stream_hash)
|
||||
|
||||
|
||||
class LBRYFileOpenerFactory(LBRYFileDownloaderFactory):
|
||||
class EncryptedFileOpenerFactory(EncryptedFileDownloaderFactory):
|
||||
def can_download(self, sd_validator):
|
||||
if which('vlc'):
|
||||
return True
|
||||
elif os.name == "nt":
|
||||
paths = [r'C:\Program Files\VideoLAN\VLC\vlc.exe',
|
||||
r'C:\Program Files (x86)\VideoLAN\VLC\vlc.exe']
|
||||
for p in paths:
|
||||
if os.path.exists(p):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
|
||||
return LBRYFileOpener(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
return EncryptedFileOpener(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet, upload_allowed)
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Stream"
|
||||
+5
-2
@@ -4,7 +4,10 @@ from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from lbrynet.interfaces import IMetadataHandler
|
||||
|
||||
|
||||
class LBRYFileMetadataHandler(object):
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileMetadataHandler(object):
|
||||
implements(IMetadataHandler)
|
||||
|
||||
def __init__(self, stream_hash, stream_info_manager, download_manager):
|
||||
@@ -31,6 +34,6 @@ class LBRYFileMetadataHandler(object):
|
||||
if blob_hash is not None:
|
||||
infos.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
else:
|
||||
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
self._final_blob_num = blob_num - 1
|
||||
return infos
|
||||
@@ -0,0 +1,55 @@
|
||||
from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType, EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
sd_identifier.add_stream_type(EncryptedFileStreamType, EncryptedFileStreamDescriptorValidator, EncryptedFileOptions())
|
||||
|
||||
|
||||
class EncryptedFileOptions(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.base.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.base.min_blob_data_payment_rate
|
||||
|
||||
rate_choices = []
|
||||
rate_choices.append(DownloadOptionChoice(prm.base.min_blob_data_payment_rate,
|
||||
"No change - %s" % get_default_data_rate_description(),
|
||||
"No change - %s" % get_default_data_rate_description()))
|
||||
if prm.base.min_blob_data_payment_rate is not None:
|
||||
rate_choices.append(DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)))
|
||||
rate_choices.append(DownloadOptionChoice(float,
|
||||
"Enter rate in LBC/MB",
|
||||
"Enter rate in LBC/MB"))
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
rate_choices,
|
||||
"Rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.base.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(bool,
|
||||
None,
|
||||
None,
|
||||
bool_options_description=("Allow", "Disallow")),
|
||||
],
|
||||
"Allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True,
|
||||
"Allow"
|
||||
),
|
||||
]
|
||||
return options
|
||||
+43
-36
@@ -7,44 +7,41 @@ import logging
|
||||
import os
|
||||
from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter
|
||||
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from twisted.protocols.basic import FileSender
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
|
||||
|
||||
class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
"""
|
||||
A CryptStreamCreator which adds itself and its additional metadata to an LBRYFileManager
|
||||
A CryptStreamCreator which adds itself and its additional metadata to an EncryptedFileManager
|
||||
"""
|
||||
def __init__(self, blob_manager, lbry_file_manager, name=None,
|
||||
key=None, iv_generator=None, suggested_file_name=None):
|
||||
CryptStreamCreator.__init__(self, blob_manager, name, key, iv_generator)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
if suggested_file_name is None:
|
||||
self.suggested_file_name = name
|
||||
else:
|
||||
self.suggested_file_name = suggested_file_name
|
||||
self.suggested_file_name = suggested_file_name or name
|
||||
self.stream_hash = None
|
||||
self.blob_infos = []
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
logging.debug("length: %s", str(blob_info.length))
|
||||
log.debug("length: %s", str(blob_info.length))
|
||||
self.blob_infos.append(blob_info)
|
||||
|
||||
def _save_lbry_file_info(self):
|
||||
def _save_stream_info(self):
|
||||
stream_info_manager = self.lbry_file_manager.stream_info_manager
|
||||
d = stream_info_manager.save_stream(self.stream_hash, binascii.hexlify(self.name),
|
||||
binascii.hexlify(self.key),
|
||||
binascii.hexlify(self.suggested_file_name),
|
||||
d = stream_info_manager.save_stream(self.stream_hash, hexlify(self.name),
|
||||
hexlify(self.key),
|
||||
hexlify(self.suggested_file_name),
|
||||
self.blob_infos)
|
||||
return d
|
||||
|
||||
def setup(self):
|
||||
d = CryptStreamCreator.setup(self)
|
||||
d.addCallback(lambda _: self.stream_hash)
|
||||
|
||||
return d
|
||||
|
||||
def _get_blobs_hashsum(self):
|
||||
@@ -68,25 +65,21 @@ class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
|
||||
def _make_stream_hash(self):
|
||||
hashsum = get_lbry_hash_obj()
|
||||
hashsum.update(binascii.hexlify(self.name))
|
||||
hashsum.update(binascii.hexlify(self.key))
|
||||
hashsum.update(binascii.hexlify(self.suggested_file_name))
|
||||
hashsum.update(hexlify(self.name))
|
||||
hashsum.update(hexlify(self.key))
|
||||
hashsum.update(hexlify(self.suggested_file_name))
|
||||
hashsum.update(self._get_blobs_hashsum())
|
||||
self.stream_hash = hashsum.hexdigest()
|
||||
|
||||
def _finished(self):
|
||||
self._make_stream_hash()
|
||||
d = self._save_lbry_file_info()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.change_lbry_file_status(
|
||||
self.stream_hash, ManagedLBRYFileDownloader.STATUS_FINISHED
|
||||
))
|
||||
d = self._save_stream_info()
|
||||
return d
|
||||
|
||||
|
||||
def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None,
|
||||
iv_generator=None, suggested_file_name=None):
|
||||
"""
|
||||
Turn a plain file into an LBRY File.
|
||||
"""Turn a plain file into an LBRY File.
|
||||
|
||||
An LBRY File is a collection of encrypted blobs of data and the metadata that binds them
|
||||
together which, when decrypted and put back together according to the metadata, results
|
||||
@@ -96,11 +89,11 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
into chunks and encrypted, and then a stream descriptor file with the stream parameters
|
||||
and other metadata is written to disk.
|
||||
|
||||
@param session: An LBRYSession object.
|
||||
@type session: LBRYSession
|
||||
@param session: An Session object.
|
||||
@type session: Session
|
||||
|
||||
@param lbry_file_manager: The LBRYFileManager object this LBRY File will be added to.
|
||||
@type lbry_file_manager: LBRYFileManager
|
||||
@param lbry_file_manager: The EncryptedFileManager object this LBRY File will be added to.
|
||||
@type lbry_file_manager: EncryptedFileManager
|
||||
|
||||
@param file_name: The path to the plain file.
|
||||
@type file_name: string
|
||||
@@ -116,8 +109,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
|
||||
once for each blob.
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@param suggested_file_name: what the file should be called when the LBRY File is saved to disk.
|
||||
@@ -128,12 +121,14 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
"""
|
||||
|
||||
def stop_file(creator):
|
||||
logging.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
log.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
return creator.stop()
|
||||
|
||||
def make_stream_desc_file(stream_hash):
|
||||
logging.debug("creating the stream descriptor file")
|
||||
descriptor_writer = PlainStreamDescriptorWriter(file_name + conf.CRYPTSD_FILE_EXTENSION)
|
||||
log.debug("creating the stream descriptor file")
|
||||
descriptor_file_path = os.path.join(
|
||||
session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION)
|
||||
descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
|
||||
|
||||
d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)
|
||||
|
||||
@@ -143,8 +138,12 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
|
||||
base_file_name = os.path.basename(file_name)
|
||||
|
||||
lbry_file_creator = LBRYFileStreamCreator(session.blob_manager, lbry_file_manager, base_file_name,
|
||||
key, iv_generator, suggested_file_name)
|
||||
lbry_file_creator = EncryptedFileStreamCreator(
|
||||
session.blob_manager,
|
||||
lbry_file_manager,
|
||||
base_file_name, key,
|
||||
iv_generator,
|
||||
suggested_file_name)
|
||||
|
||||
def start_stream():
|
||||
file_sender = FileSender()
|
||||
@@ -156,4 +155,12 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
|
||||
d = lbry_file_creator.setup()
|
||||
d.addCallback(lambda _: start_stream())
|
||||
return d
|
||||
return d
|
||||
|
||||
|
||||
def hexlify(str_or_unicode):
|
||||
if isinstance(str_or_unicode, unicode):
|
||||
strng = str_or_unicode.encode('utf-8')
|
||||
else:
|
||||
strng = str_or_unicode
|
||||
return binascii.hexlify(strng)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
import random
|
||||
import logging
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileSaver, EncryptedFileDownloader
|
||||
from lbrynet.lbryfilemanager.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
|
||||
from lbrynet.reflector import reupload
|
||||
from lbrynet.conf import settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, upload_allowed,
|
||||
file_name=None):
|
||||
EncryptedFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, download_directory,
|
||||
upload_allowed, file_name)
|
||||
self.sd_hash = None
|
||||
self.txid = None
|
||||
self.nout = None
|
||||
self.uri = None
|
||||
self.claim_id = None
|
||||
self.rowid = rowid
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.saving_status = False
|
||||
|
||||
def restore(self):
|
||||
d = self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash)
|
||||
|
||||
def _save_stream_info(sd_hash):
|
||||
if sd_hash:
|
||||
self.sd_hash = sd_hash[0]
|
||||
d = self.wallet.get_claim_metadata_for_sd_hash(self.sd_hash)
|
||||
d.addCallback(lambda r: _save_claim(r[0], r[1], r[2]))
|
||||
return d
|
||||
else:
|
||||
return None
|
||||
|
||||
def _save_claim_id(claim_id):
|
||||
self.claim_id = claim_id
|
||||
return defer.succeed(None)
|
||||
|
||||
def _notify_bad_claim(name, txid, nout):
|
||||
err_msg = "Error loading name claim for lbry file: \
|
||||
lbry://%s, tx %s output %i does not contain a valid claim, deleting it"
|
||||
log.error(err_msg, name, txid, nout)
|
||||
return self.lbry_file_manager.delete_lbry_file(self)
|
||||
|
||||
def _save_claim(name, txid, nout):
|
||||
self.uri = name
|
||||
self.txid = txid
|
||||
self.nout = nout
|
||||
d = self.wallet.get_claimid(name, txid, nout)
|
||||
d.addCallbacks(_save_claim_id, lambda err: _notify_bad_claim(name, txid, nout))
|
||||
return d
|
||||
|
||||
reflector_server = random.choice(settings.reflector_servers)
|
||||
|
||||
d.addCallback(_save_stream_info)
|
||||
d.addCallback(lambda _: reupload.check_and_restore_availability(self, reflector_server))
|
||||
d.addCallback(lambda _: self.lbry_file_manager.get_lbry_file_status(self))
|
||||
|
||||
def restore_status(status):
|
||||
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
|
||||
return self.start()
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
|
||||
return defer.succeed(False)
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
self.completed = True
|
||||
return defer.succeed(True)
|
||||
|
||||
d.addCallback(restore_status)
|
||||
return d
|
||||
|
||||
def stop(self, err=None, change_status=True):
|
||||
|
||||
def set_saving_status_done():
|
||||
self.saving_status = False
|
||||
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
if change_status is True:
|
||||
self.saving_status = True
|
||||
d.addCallback(lambda _: self._save_status())
|
||||
d.addCallback(lambda _: set_saving_status_done())
|
||||
return d
|
||||
|
||||
def status(self):
|
||||
def find_completed_blobhashes(blobs):
|
||||
blobhashes = [b[0] for b in blobs if b[0] is not None]
|
||||
|
||||
def get_num_completed(completed_blobs):
|
||||
return len(completed_blobs), len(blobhashes)
|
||||
|
||||
inner_d = self.blob_manager.completed_blobs(blobhashes)
|
||||
inner_d.addCallback(get_num_completed)
|
||||
return inner_d
|
||||
|
||||
def make_full_status(progress):
|
||||
num_completed = progress[0]
|
||||
num_known = progress[1]
|
||||
if self.completed is True:
|
||||
s = "completed"
|
||||
elif self.stopped is True:
|
||||
s = "stopped"
|
||||
else:
|
||||
s = "running"
|
||||
status = EncryptedFileStatusReport(self.file_name, num_completed, num_known, s)
|
||||
return status
|
||||
|
||||
d = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
d.addCallback(find_completed_blobhashes)
|
||||
d.addCallback(make_full_status)
|
||||
return d
|
||||
|
||||
def _start(self):
|
||||
|
||||
d = EncryptedFileSaver._start(self)
|
||||
d.addCallback(lambda _: self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash))
|
||||
|
||||
def _save_sd_hash(sd_hash):
|
||||
if len(sd_hash):
|
||||
self.sd_hash = sd_hash[0]
|
||||
d = self.wallet.get_claim_metadata_for_sd_hash(self.sd_hash)
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
|
||||
return d
|
||||
|
||||
def _save_claim(name, txid, nout):
|
||||
self.uri = name
|
||||
self.txid = txid
|
||||
self.nout = nout
|
||||
return defer.succeed(None)
|
||||
|
||||
d.addCallback(_save_sd_hash)
|
||||
d.addCallback(lambda r: _save_claim(r[0], r[1], r[2]) if r else None)
|
||||
d.addCallback(lambda _: self._save_status())
|
||||
|
||||
return d
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
if self.completed is True:
|
||||
return "Download successful"
|
||||
else:
|
||||
return "Download stopped"
|
||||
|
||||
def _save_status(self):
|
||||
if self.completed is True:
|
||||
s = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
elif self.stopped is True:
|
||||
s = ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
else:
|
||||
s = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
return self.lbry_file_manager.change_lbry_file_status(self, s)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, lbry_file_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, file_name=None):
|
||||
data_rate = options[0]
|
||||
upload_allowed = options[1]
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.lbry_file_manager.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash,
|
||||
metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.lbry_file_manager.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(lambda stream_hash: self.lbry_file_manager.add_lbry_file(stream_hash,
|
||||
payment_rate_manager,
|
||||
data_rate,
|
||||
upload_allowed,
|
||||
download_directory=download_directory,
|
||||
file_name=file_name))
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save the file to disk"
|
||||
@@ -0,0 +1,246 @@
|
||||
"""
|
||||
Keep track of which LBRY Files are downloading and store their LBRY File specific metadata
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from twisted.enterprise import adbapi
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.core.PaymentRateManager import NegotiatedPaymentRateManager
|
||||
from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.lbryfilemanager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError, CurrentlyStoppingError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileManager(object):
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
|
||||
"""
|
||||
|
||||
def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
|
||||
self.session = session
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.lbry_files = []
|
||||
self.sql_db = None
|
||||
if download_directory:
|
||||
self.download_directory = download_directory
|
||||
else:
|
||||
self.download_directory = os.getcwd()
|
||||
log.debug("Download directory for EncryptedFileManager: %s", str(self.download_directory))
|
||||
|
||||
def setup(self):
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._add_to_sd_identifier())
|
||||
d.addCallback(lambda _: self._start_lbry_files())
|
||||
return d
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self._get_lbry_file_status(lbry_file.rowid)
|
||||
|
||||
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
|
||||
return self._set_lbry_file_payment_rate(lbry_file.rowid, new_rate)
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self._change_file_status(lbry_file.rowid, status)
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
|
||||
for lbry_file in self.lbry_files:
|
||||
ds.append(lbry_file.status())
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def filter_failures(status_reports):
|
||||
return [status_report for success, status_report in status_reports if success is True]
|
||||
|
||||
dl.addCallback(filter_failures)
|
||||
return dl
|
||||
|
||||
def _add_to_sd_identifier(self):
|
||||
downloader_factory = ManagedEncryptedFileDownloaderFactory(self)
|
||||
self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, downloader_factory)
|
||||
|
||||
def _start_lbry_files(self):
|
||||
|
||||
def set_options_and_restore(rowid, stream_hash, options):
|
||||
payment_rate_manager = NegotiatedPaymentRateManager(self.session.base_payment_rate_manager,
|
||||
self.session.blob_tracker)
|
||||
d = self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
|
||||
blob_data_rate=options)
|
||||
d.addCallback(lambda downloader: downloader.restore())
|
||||
return d
|
||||
|
||||
def log_error(err, rowid, stream_hash, options):
|
||||
log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
|
||||
log.debug(rowid)
|
||||
log.debug(stream_hash)
|
||||
log.debug(options)
|
||||
|
||||
def start_lbry_files(lbry_files_and_options):
|
||||
for rowid, stream_hash, options in lbry_files_and_options:
|
||||
d = set_options_and_restore(rowid, stream_hash, options)
|
||||
d.addErrback(lambda err: log_error(err, rowid, stream_hash, options))
|
||||
return True
|
||||
|
||||
d = self._get_all_lbry_files()
|
||||
d.addCallback(start_lbry_files)
|
||||
return d
|
||||
|
||||
def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
|
||||
download_directory=None, file_name=None):
|
||||
if not download_directory:
|
||||
download_directory = self.download_directory
|
||||
payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
|
||||
lbry_file_downloader = ManagedEncryptedFileDownloader(rowid, stream_hash,
|
||||
self.session.peer_finder,
|
||||
self.session.rate_limiter,
|
||||
self.session.blob_manager,
|
||||
self.stream_info_manager, self,
|
||||
payment_rate_manager, self.session.wallet,
|
||||
download_directory,
|
||||
upload_allowed,
|
||||
file_name=file_name)
|
||||
self.lbry_files.append(lbry_file_downloader)
|
||||
d = lbry_file_downloader.set_stream_info()
|
||||
d.addCallback(lambda _: lbry_file_downloader)
|
||||
return d
|
||||
|
||||
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
|
||||
download_directory=None, file_name=None):
|
||||
d = self._save_lbry_file(stream_hash, blob_data_rate)
|
||||
d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
|
||||
blob_data_rate, upload_allowed, download_directory, file_name))
|
||||
return d
|
||||
|
||||
def delete_lbry_file(self, lbry_file):
|
||||
for l in self.lbry_files:
|
||||
if l == lbry_file:
|
||||
lbry_file = l
|
||||
break
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
def wait_for_finished(count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
return True
|
||||
else:
|
||||
return task.deferLater(reactor, 1, wait_for_finished, count=count - 1)
|
||||
|
||||
def ignore_stopped(err):
|
||||
err.trap(AlreadyStoppedError, CurrentlyStoppingError)
|
||||
return wait_for_finished()
|
||||
|
||||
d = lbry_file.stop()
|
||||
d.addErrback(ignore_stopped)
|
||||
|
||||
def remove_from_list():
|
||||
self.lbry_files.remove(lbry_file)
|
||||
|
||||
d.addCallback(lambda _: remove_from_list())
|
||||
d.addCallback(lambda _: self._delete_lbry_file_options(lbry_file.rowid))
|
||||
return d
|
||||
|
||||
def toggle_lbry_file_running(self, lbry_file):
|
||||
"""Toggle whether a stream reader is currently running"""
|
||||
for l in self.lbry_files:
|
||||
if l == lbry_file:
|
||||
return l.toggle_running()
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
def stop(self):
|
||||
log.info('Stopping %s', self)
|
||||
ds = []
|
||||
|
||||
def wait_for_finished(lbry_file, count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
return True
|
||||
else:
|
||||
return task.deferLater(reactor, 1, wait_for_finished, lbry_file, count=count - 1)
|
||||
|
||||
def ignore_stopped(err, lbry_file):
|
||||
err.trap(AlreadyStoppedError, CurrentlyStoppingError)
|
||||
return wait_for_finished(lbry_file)
|
||||
|
||||
for lbry_file in self.lbry_files:
|
||||
d = lbry_file.stop(change_status=False)
|
||||
d.addErrback(ignore_stopped, lbry_file)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def close_db():
|
||||
self.db = None
|
||||
|
||||
dl.addCallback(lambda _: close_db())
|
||||
return dl
|
||||
|
||||
def get_count_for_stream_hash(self, stream_hash):
|
||||
return self._get_count_for_stream_hash(stream_hash)
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.sql_db = adbapi.ConnectionPool("sqlite3", os.path.join(self.session.db_dir, "lbryfile_info.db"),
|
||||
check_same_thread=False)
|
||||
return self.sql_db.runQuery("create table if not exists lbry_file_options (" +
|
||||
" blob_data_rate real, " +
|
||||
" status text," +
|
||||
" stream_hash text,"
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_lbry_file(self, stream_hash, data_payment_rate):
|
||||
def do_save(db_transaction):
|
||||
db_transaction.execute("insert into lbry_file_options values (?, ?, ?)",
|
||||
(data_payment_rate, ManagedEncryptedFileDownloader.STATUS_STOPPED,
|
||||
stream_hash))
|
||||
return db_transaction.lastrowid
|
||||
return self.sql_db.runInteraction(do_save)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_lbry_file_options(self, rowid):
|
||||
return self.sql_db.runQuery("delete from lbry_file_options where rowid = ?",
|
||||
(rowid,))
|
||||
|
||||
@rerun_if_locked
|
||||
def _set_lbry_file_payment_rate(self, rowid, new_rate):
|
||||
return self.sql_db.runQuery("update lbry_file_options set blob_data_rate = ? where rowid = ?",
|
||||
(new_rate, rowid))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_lbry_files(self):
|
||||
d = self.sql_db.runQuery("select rowid, stream_hash, blob_data_rate from lbry_file_options")
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _change_file_status(self, rowid, new_status):
|
||||
return self.sql_db.runQuery("update lbry_file_options set status = ? where rowid = ?",
|
||||
(new_status, rowid))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_lbry_file_status(self, rowid):
|
||||
d = self.sql_db.runQuery("select status from lbry_file_options where rowid = ?",
|
||||
(rowid,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else ManagedEncryptedFileDownloader.STATUS_STOPPED)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_count_for_stream_hash(self, stream_hash):
|
||||
return self.sql_db.runQuery("select count(*) from lbry_file_options where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
class LBRYFileStatusReport(object):
|
||||
class EncryptedFileStatusReport(object):
|
||||
def __init__(self, name, num_completed, num_known, running_status):
|
||||
self.name = name
|
||||
self.num_completed = num_completed
|
||||
@@ -1,149 +0,0 @@
|
||||
"""
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaver, LBRYFileDownloader
|
||||
from lbrynet.lbryfilemanager.LBRYFileStatusReport import LBRYFileStatusReport
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, upload_allowed,
|
||||
file_name=None):
|
||||
LBRYFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, download_directory,
|
||||
upload_allowed)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.file_name = file_name
|
||||
self.file_handle = None
|
||||
self.saving_status = False
|
||||
|
||||
def restore(self):
|
||||
d = self.lbry_file_manager.get_lbry_file_status(self.stream_hash)
|
||||
|
||||
def restore_status(status):
|
||||
if status == ManagedLBRYFileDownloader.STATUS_RUNNING:
|
||||
return self.start()
|
||||
elif status == ManagedLBRYFileDownloader.STATUS_STOPPED:
|
||||
return defer.succeed(False)
|
||||
elif status == ManagedLBRYFileDownloader.STATUS_FINISHED:
|
||||
self.completed = True
|
||||
return defer.succeed(True)
|
||||
|
||||
d.addCallback(restore_status)
|
||||
return d
|
||||
|
||||
def stop(self, change_status=True):
|
||||
|
||||
def set_saving_status_done():
|
||||
self.saving_status = False
|
||||
|
||||
d = LBRYFileDownloader.stop(self) # LBRYFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
if change_status is True:
|
||||
self.saving_status = True
|
||||
d.addCallback(lambda _: self._save_status())
|
||||
d.addCallback(lambda _: set_saving_status_done())
|
||||
return d
|
||||
|
||||
def status(self):
|
||||
def find_completed_blobhashes(blobs):
|
||||
blobhashes = [b[0] for b in blobs if b[0] is not None]
|
||||
|
||||
def get_num_completed(completed_blobs):
|
||||
return len(completed_blobs), len(blobhashes)
|
||||
|
||||
inner_d = self.blob_manager.completed_blobs(blobhashes)
|
||||
inner_d.addCallback(get_num_completed)
|
||||
return inner_d
|
||||
|
||||
def make_full_status(progress):
|
||||
num_completed = progress[0]
|
||||
num_known = progress[1]
|
||||
if self.completed is True:
|
||||
s = "completed"
|
||||
elif self.stopped is True:
|
||||
s = "stopped"
|
||||
else:
|
||||
s = "running"
|
||||
status = LBRYFileStatusReport(self.file_name, num_completed, num_known, s)
|
||||
return status
|
||||
|
||||
d = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
d.addCallback(find_completed_blobhashes)
|
||||
d.addCallback(make_full_status)
|
||||
return d
|
||||
|
||||
def _start(self):
|
||||
|
||||
d = LBRYFileSaver._start(self)
|
||||
|
||||
d.addCallback(lambda _: self._save_status())
|
||||
|
||||
return d
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
if self.completed is True:
|
||||
return "Download successful"
|
||||
else:
|
||||
return "Download stopped"
|
||||
|
||||
def _save_status(self):
|
||||
if self.completed is True:
|
||||
s = ManagedLBRYFileDownloader.STATUS_FINISHED
|
||||
elif self.stopped is True:
|
||||
s = ManagedLBRYFileDownloader.STATUS_STOPPED
|
||||
else:
|
||||
s = ManagedLBRYFileDownloader.STATUS_RUNNING
|
||||
return self.lbry_file_manager.change_lbry_file_status(self.stream_hash, s)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
|
||||
|
||||
|
||||
class ManagedLBRYFileDownloaderFactory(object):
|
||||
implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, lbry_file_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
data_rate = options[0]
|
||||
upload_allowed = options[1]
|
||||
|
||||
d = save_sd_info(self.lbry_file_manager.stream_info_manager, sd_validator.raw_info)
|
||||
d.addCallback(lambda stream_hash: self.lbry_file_manager.add_lbry_file(stream_hash,
|
||||
payment_rate_manager,
|
||||
data_rate,
|
||||
upload_allowed))
|
||||
return d
|
||||
|
||||
def get_description(self):
|
||||
return "Save the file to disk"
|
||||
@@ -1,255 +0,0 @@
|
||||
"""
|
||||
Keep track of which LBRY Files are downloading and store their LBRY File specific metadata
|
||||
"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
|
||||
import leveldb
|
||||
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamDescriptorValidator
|
||||
import os
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from twisted.internet import threads, defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError, CurrentlyStoppingError
|
||||
|
||||
|
||||
class LBRYFileManager(object):
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
|
||||
"""
|
||||
SETTING = "s"
|
||||
LBRYFILE_STATUS = "t"
|
||||
LBRYFILE_OPTIONS = "o"
|
||||
|
||||
def __init__(self, session, stream_info_manager, sd_identifier):
|
||||
self.session = session
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.lbry_files = []
|
||||
self.db = None
|
||||
self.download_directory = os.getcwd()
|
||||
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d.addCallback(lambda _: self._add_to_sd_identifier())
|
||||
d.addCallback(lambda _: self._start_lbry_files())
|
||||
return d
|
||||
|
||||
def get_all_lbry_file_stream_hashes_and_options(self):
|
||||
d = threads.deferToThread(self._get_all_lbry_file_stream_hashes)
|
||||
|
||||
def get_options(stream_hashes):
|
||||
ds = []
|
||||
|
||||
def get_options_for_stream_hash(stream_hash):
|
||||
d = self.get_lbry_file_options(stream_hash)
|
||||
d.addCallback(lambda options: (stream_hash, options))
|
||||
return d
|
||||
|
||||
for stream_hash in stream_hashes:
|
||||
ds.append(get_options_for_stream_hash(stream_hash))
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(lambda results: [r[1] for r in results if r[0]])
|
||||
return dl
|
||||
|
||||
d.addCallback(get_options)
|
||||
return d
|
||||
|
||||
def get_lbry_file_status(self, stream_hash):
|
||||
return threads.deferToThread(self._get_lbry_file_status, stream_hash)
|
||||
|
||||
def save_lbry_file_options(self, stream_hash, blob_data_rate):
|
||||
return threads.deferToThread(self._save_lbry_file_options, stream_hash, blob_data_rate)
|
||||
|
||||
def get_lbry_file_options(self, stream_hash):
|
||||
return threads.deferToThread(self._get_lbry_file_options, stream_hash)
|
||||
|
||||
def delete_lbry_file_options(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_lbry_file_options, stream_hash)
|
||||
|
||||
def set_lbry_file_data_payment_rate(self, stream_hash, new_rate):
|
||||
return threads.deferToThread(self._set_lbry_file_payment_rate, stream_hash, new_rate)
|
||||
|
||||
def change_lbry_file_status(self, stream_hash, status):
|
||||
logging.debug("Changing status of %s to %s", stream_hash, status)
|
||||
return threads.deferToThread(self._change_file_status, stream_hash, status)
|
||||
|
||||
def delete_lbry_file_status(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_lbry_file_status, stream_hash)
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
|
||||
for lbry_file in self.lbry_files:
|
||||
ds.append(lbry_file.status())
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def filter_failures(status_reports):
|
||||
return [status_report for success, status_report in status_reports if success is True]
|
||||
|
||||
dl.addCallback(filter_failures)
|
||||
return dl
|
||||
|
||||
def _add_to_sd_identifier(self):
|
||||
downloader_factory = ManagedLBRYFileDownloaderFactory(self)
|
||||
self.sd_identifier.add_stream_info_validator(LBRYFileStreamType, LBRYFileStreamDescriptorValidator)
|
||||
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)
|
||||
|
||||
def _start_lbry_files(self):
|
||||
|
||||
def set_options_and_restore(stream_hash, options):
|
||||
payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
d = self.add_lbry_file(stream_hash, payment_rate_manager, blob_data_rate=options[0])
|
||||
d.addCallback(lambda downloader: downloader.restore())
|
||||
return d
|
||||
|
||||
def log_error(err):
|
||||
logging.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
|
||||
|
||||
def start_lbry_files(stream_hashes_and_options):
|
||||
for stream_hash, options in stream_hashes_and_options:
|
||||
d = set_options_and_restore(stream_hash, options)
|
||||
d.addErrback(log_error)
|
||||
return True
|
||||
|
||||
d = self.get_all_lbry_file_stream_hashes_and_options()
|
||||
d.addCallback(start_lbry_files)
|
||||
return d
|
||||
|
||||
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True):
|
||||
payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
|
||||
lbry_file_downloader = ManagedLBRYFileDownloader(stream_hash, self.session.peer_finder,
|
||||
self.session.rate_limiter, self.session.blob_manager,
|
||||
self.stream_info_manager, self,
|
||||
payment_rate_manager, self.session.wallet,
|
||||
self.download_directory,
|
||||
upload_allowed)
|
||||
self.lbry_files.append(lbry_file_downloader)
|
||||
d = self.save_lbry_file_options(stream_hash, blob_data_rate)
|
||||
d.addCallback(lambda _: lbry_file_downloader.set_stream_info())
|
||||
d.addCallback(lambda _: lbry_file_downloader)
|
||||
return d
|
||||
|
||||
def delete_lbry_file(self, stream_hash):
|
||||
for l in self.lbry_files:
|
||||
if l.stream_hash == stream_hash:
|
||||
lbry_file = l
|
||||
break
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
|
||||
stream_hash)))
|
||||
|
||||
def wait_for_finished(count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
return True
|
||||
else:
|
||||
return task.deferLater(reactor, 1, wait_for_finished, count=count - 1)
|
||||
|
||||
def ignore_stopped(err):
|
||||
err.trap(AlreadyStoppedError, CurrentlyStoppingError)
|
||||
return wait_for_finished()
|
||||
|
||||
d = lbry_file.stop()
|
||||
d.addErrback(ignore_stopped)
|
||||
|
||||
def remove_from_list():
|
||||
self.lbry_files.remove(lbry_file)
|
||||
|
||||
d.addCallback(lambda _: remove_from_list())
|
||||
d.addCallback(lambda _: self.delete_lbry_file_options(stream_hash))
|
||||
d.addCallback(lambda _: self.delete_lbry_file_status(stream_hash))
|
||||
return d
|
||||
|
||||
def toggle_lbry_file_running(self, stream_hash):
|
||||
"""Toggle whether a stream reader is currently running"""
|
||||
for l in self.lbry_files:
|
||||
if l.stream_hash == stream_hash:
|
||||
return l.toggle_running()
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
|
||||
stream_hash)))
|
||||
|
||||
def get_stream_hash_from_name(self, lbry_file_name):
|
||||
for l in self.lbry_files:
|
||||
if l.file_name == lbry_file_name:
|
||||
return l.stream_hash
|
||||
return None
|
||||
|
||||
def stop(self):
|
||||
ds = []
|
||||
|
||||
def wait_for_finished(lbry_file, count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
return True
|
||||
else:
|
||||
return task.deferLater(reactor, 1, wait_for_finished, lbry_file, count=count - 1)
|
||||
|
||||
def ignore_stopped(err, lbry_file):
|
||||
err.trap(AlreadyStoppedError, CurrentlyStoppingError)
|
||||
return wait_for_finished(lbry_file)
|
||||
|
||||
for lbry_file in self.lbry_files:
|
||||
d = lbry_file.stop(change_status=False)
|
||||
d.addErrback(ignore_stopped, lbry_file)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def close_db():
|
||||
self.db = None
|
||||
|
||||
dl.addCallback(lambda _: close_db())
|
||||
return dl
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.session.db_dir, "lbryfiles.db"))
|
||||
|
||||
def _save_payment_rate(self, rate_type, rate):
|
||||
if rate is not None:
|
||||
self.db.Put(json.dumps((self.SETTING, rate_type)), json.dumps(rate), sync=True)
|
||||
else:
|
||||
self.db.Delete(json.dumps((self.SETTING, rate_type)), sync=True)
|
||||
|
||||
def _save_lbry_file_options(self, stream_hash, blob_data_rate):
|
||||
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((blob_data_rate,)),
|
||||
sync=True)
|
||||
|
||||
def _get_lbry_file_options(self, stream_hash):
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps((self.LBRYFILE_OPTIONS, stream_hash))))
|
||||
except KeyError:
|
||||
return None, None
|
||||
|
||||
def _delete_lbry_file_options(self, stream_hash):
|
||||
self.db.Delete(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), sync=True)
|
||||
|
||||
def _set_lbry_file_payment_rate(self, stream_hash, new_rate):
|
||||
|
||||
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((new_rate, )), sync=True)
|
||||
|
||||
def _get_all_lbry_file_stream_hashes(self):
|
||||
hashes = []
|
||||
for k, v in self.db.RangeIter():
|
||||
key_type, stream_hash = json.loads(k)
|
||||
if key_type == self.LBRYFILE_STATUS:
|
||||
hashes.append(stream_hash)
|
||||
return hashes
|
||||
|
||||
def _change_file_status(self, stream_hash, new_status):
|
||||
self.db.Put(json.dumps((self.LBRYFILE_STATUS, stream_hash)), new_status, sync=True)
|
||||
|
||||
def _get_lbry_file_status(self, stream_hash):
|
||||
try:
|
||||
return self.db.Get(json.dumps((self.LBRYFILE_STATUS, stream_hash)))
|
||||
except KeyError:
|
||||
return ManagedLBRYFileDownloader.STATUS_STOPPED
|
||||
|
||||
def _delete_lbry_file_status(self, stream_hash):
|
||||
self.db.Delete(json.dumps((self.LBRYFILE_STATUS, stream_hash)), sync=True)
|
||||
@@ -1,20 +1,20 @@
|
||||
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter
|
||||
from lbrynet.lbrylive.StreamDescriptor import get_sd_info, LiveStreamType, LBRYLiveStreamDescriptorValidator
|
||||
from lbrynet.lbrylive.StreamDescriptor import get_sd_info
|
||||
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
|
||||
from lbrynet.lbrylive.LiveBlob import LiveStreamBlobMaker
|
||||
from lbrynet.lbrylive.PaymentRateManager import BaseLiveStreamPaymentRateManager
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, get_pub_key, sign_with_pass_phrase
|
||||
from Crypto import Random
|
||||
import binascii
|
||||
import logging
|
||||
from lbrynet.conf import CRYPTSD_FILE_EXTENSION
|
||||
from lbrynet.conf import MIN_BLOB_INFO_PAYMENT_RATE
|
||||
from lbrynet.lbrylive.client.LiveStreamDownloader import FullLiveStreamDownloaderFactory
|
||||
from lbrynet.conf import settings
|
||||
from twisted.internet import interfaces, defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamCreator(CryptStreamCreator):
|
||||
def __init__(self, blob_manager, stream_info_manager, name=None, key=None, iv_generator=None,
|
||||
delete_after_num=None, secret_pass_phrase=None):
|
||||
@@ -23,7 +23,7 @@ class LiveStreamCreator(CryptStreamCreator):
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.delete_after_num = delete_after_num
|
||||
self.secret_pass_phrase = secret_pass_phrase
|
||||
self.file_extension = CRYPTSD_FILE_EXTENSION
|
||||
self.file_extension = settings.CRYPTSD_FILE_EXTENSION
|
||||
self.finished_blob_hashes = {}
|
||||
|
||||
def _save_stream(self):
|
||||
@@ -33,8 +33,8 @@ class LiveStreamCreator(CryptStreamCreator):
|
||||
return d
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
logging.debug("In blob_finished")
|
||||
logging.debug("length: %s", str(blob_info.length))
|
||||
log.debug("In blob_finished")
|
||||
log.debug("length: %s", str(blob_info.length))
|
||||
sig_hash = get_lbry_hash_obj()
|
||||
sig_hash.update(self.stream_hash)
|
||||
if blob_info.length != 0:
|
||||
@@ -51,11 +51,11 @@ class LiveStreamCreator(CryptStreamCreator):
|
||||
d = self.stream_info_manager.add_blobs_to_stream(self.stream_hash, [blob_info])
|
||||
|
||||
def log_add_error(err):
|
||||
logging.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
|
||||
log.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_add_error)
|
||||
logging.debug("returning from blob_finished")
|
||||
log.debug("returning from blob_finished")
|
||||
return d
|
||||
|
||||
def setup(self):
|
||||
@@ -173,17 +173,4 @@ class StdinStreamProducer(object):
|
||||
self.consumer.write(data)
|
||||
|
||||
def childConnectionLost(self, fd, reason):
|
||||
self.stopProducing()
|
||||
|
||||
|
||||
def add_live_stream_to_sd_identifier(session, stream_info_manager, sd_identifier):
|
||||
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
|
||||
session.rate_limiter,
|
||||
session.blob_manager,
|
||||
stream_info_manager,
|
||||
session.wallet,
|
||||
BaseLiveStreamPaymentRateManager(
|
||||
MIN_BLOB_INFO_PAYMENT_RATE
|
||||
))
|
||||
sd_identifier.add_stream_info_validator(LiveStreamType, LBRYLiveStreamDescriptorValidator)
|
||||
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
|
||||
self.stopProducing()
|
||||
@@ -1,11 +1,16 @@
|
||||
import time
|
||||
import logging
|
||||
import leveldb
|
||||
import json
|
||||
from twisted.enterprise import adbapi
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
import sqlite3
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
@@ -14,26 +19,22 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
def __init__(self, db_dir, hash_announcer):
|
||||
DHTHashSupplier.__init__(self, hash_announcer)
|
||||
self.db_dir = db_dir
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
|
||||
def setup(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return threads.deferToThread(self._get_all_streams)
|
||||
return self._get_all_streams()
|
||||
|
||||
def save_stream(self, stream_hash, pub_key, file_name, key, blobs):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
d = threads.deferToThread(self._store_stream, stream_hash, pub_key, file_name, key,
|
||||
next_announce_time=next_announce_time)
|
||||
d = self._store_stream(stream_hash, pub_key, file_name, key,
|
||||
next_announce_time=next_announce_time)
|
||||
|
||||
def save_blobs():
|
||||
return self.add_blobs_to_stream(stream_hash, blobs)
|
||||
@@ -48,169 +49,229 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return threads.deferToThread(self._get_stream_info, stream_hash)
|
||||
return self._get_stream_info(stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
|
||||
return self._check_if_stream_exists(stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_stream, stream_hash)
|
||||
return self._delete_stream(stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
|
||||
def add_blobs():
|
||||
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
return threads.deferToThread(add_blobs)
|
||||
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
logging.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
log.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
d1 = defer.succeed(None)
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
d2 = defer.succeed(None)
|
||||
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
|
||||
def get_positions(results):
|
||||
start_num = None
|
||||
end_num = None
|
||||
return start_num, end_num
|
||||
if results[0][0] is True:
|
||||
start_num = results[0][1]
|
||||
if results[1][0] is True:
|
||||
end_num = results[1][1]
|
||||
return start_num, end_num
|
||||
|
||||
dl.addCallback(get_positions)
|
||||
return dl
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = threads.deferToThread(get_positions_of_start_and_end)
|
||||
d = get_positions_of_start_and_end()
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
|
||||
return self._get_stream_of_blobhash(blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
|
||||
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
|
||||
return self._get_sd_blob_hashes_for_stream(stream_hash)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_streams_to_announce, next_announce_time)
|
||||
return self._get_streams_to_announce(next_announce_time)
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_info.db"))
|
||||
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_blob.db"))
|
||||
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_desc.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "live_stream.db"),
|
||||
check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists live_streams (" +
|
||||
" stream_hash text primary key, " +
|
||||
" public_key text, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" next_announce_time real" +
|
||||
")")
|
||||
transaction.execute("create table if not exists live_stream_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" revision integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" signature text, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
transaction.execute("create table if not exists live_stream_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_stream(self, stream_hash):
|
||||
desc_batch = leveldb.WriteBatch()
|
||||
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
desc_batch.Delete(sd_blob_hash)
|
||||
self.stream_desc_db.Write(desc_batch, sync=True)
|
||||
|
||||
blob_batch = leveldb.WriteBatch()
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
blob_batch.Delete(blob_hash_stream_hash)
|
||||
self.stream_blob_db.Write(blob_batch, sync=True)
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
|
||||
stream_batch = leveldb.WriteBatch()
|
||||
for s_h, stream_info in self.stream_info_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
stream_batch.Delete(s_h)
|
||||
self.stream_info_db.Write(stream_batch, sync=True)
|
||||
def do_delete(transaction, s_h):
|
||||
transaction.execute("delete from live_streams where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from live_stream_blobs where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from live_stream_descriptors where stream_hash = ?", (s_h,))
|
||||
|
||||
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _store_stream(self, stream_hash, public_key, name, key, next_announce_time=None):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
|
||||
except KeyError:
|
||||
pass
|
||||
self.stream_info_db.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into live_streams values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, public_key, key, name, next_announce_time))
|
||||
|
||||
def check_duplicate(err):
|
||||
if err.check(sqlite3.IntegrityError):
|
||||
raise DuplicateStreamHashError(stream_hash)
|
||||
return err
|
||||
|
||||
d.addErrback(check_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_streams(self):
|
||||
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams")
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_info(self, stream_hash):
|
||||
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
|
||||
d = self.db_conn.runQuery("select public_key, key, stream_name from live_streams where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda r: True if len(r) else False)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_streams_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
stream_hashes = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for stream_hash, stream_info in self.stream_info_db.RangeIter():
|
||||
public_key, key, name, announce_time = json.loads(stream_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)))
|
||||
stream_hashes.append(stream_hash)
|
||||
self.stream_info_db.Write(batch, sync=True)
|
||||
return stream_hashes
|
||||
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select stream_hash from live_streams where" +
|
||||
" (next_announce_time is null or next_announce_time < ?) " +
|
||||
" and stream_hash is not null", (timestamp, ))
|
||||
s_hs = [s_h for s_h, in r.fetchall()]
|
||||
transaction.execute("update live_streams set next_announce_time = ? where " +
|
||||
" (next_announce_time is null or next_announce_time < ?)",
|
||||
(next_announce_time, timestamp))
|
||||
return s_hs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
|
||||
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
|
||||
d = self.db_conn.runQuery("select position from live_stream_blobs where stream_hash = ? and blob_hash = ?",
|
||||
(stream_hash, blob_hash))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
position, revision, iv, length, signature = json.loads(blob_info)
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, revision, iv, length, signature))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
params = []
|
||||
q_string = "select * from ("
|
||||
q_string += " select blob_hash, position, revision, iv, length, signature from live_stream_blobs "
|
||||
q_string += " where stream_hash = ? "
|
||||
params.append(stream_hash)
|
||||
if start_num is not None:
|
||||
q_string += " and position > ? "
|
||||
params.append(start_num)
|
||||
if end_num is not None:
|
||||
q_string += " and position < ? "
|
||||
params.append(end_num)
|
||||
q_string += " order by position "
|
||||
if reverse is True:
|
||||
q_string += " DESC "
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return blob_infos
|
||||
q_string += " limit ? "
|
||||
params.append(count)
|
||||
q_string += ") order by position"
|
||||
# Order by position is done twice so that it always returns them from lowest position to
|
||||
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
|
||||
return self.db_conn.runQuery(q_string, tuple(params))
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_info in blob_infos:
|
||||
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
|
||||
try:
|
||||
self.stream_blob_db.Get(blob_hash_stream_hash)
|
||||
if ignore_duplicate_error is False:
|
||||
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
batch.Put(blob_hash_stream_hash,
|
||||
json.dumps((blob_info.blob_num,
|
||||
blob_info.revision,
|
||||
blob_info.iv,
|
||||
blob_info.length,
|
||||
blob_info.signature)))
|
||||
self.stream_blob_db.Write(batch, sync=True)
|
||||
|
||||
def add_blobs(transaction):
|
||||
for blob_info in blob_infos:
|
||||
try:
|
||||
transaction.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
|
||||
blob_info.revision, blob_info.iv, blob_info.length,
|
||||
blob_info.signature))
|
||||
except sqlite3.IntegrityError:
|
||||
if ignore_duplicate_error is False:
|
||||
raise
|
||||
|
||||
return self.db_conn.runInteraction(add_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if blob_hash == b_h:
|
||||
return s_h
|
||||
return None
|
||||
d = self.db_conn.runQuery("select stream_hash from live_stream_blobs where blob_hash = ?",
|
||||
(blob_hash,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
|
||||
return self.db_conn.runQuery("insert into live_stream_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
|
||||
d = self.db_conn.runQuery("select sd_blob_hash from live_stream_descriptors where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
|
||||
class TempLiveStreamMetadataManager(DHTHashSupplier):
|
||||
|
||||
@@ -19,11 +19,18 @@ class LiveStreamPaymentRateManager(object):
|
||||
def accept_rate_live_blob_info(self, peer, payment_rate):
|
||||
return payment_rate >= self.get_effective_min_live_blob_info_payment_rate()
|
||||
|
||||
def get_rate_blob_data(self, peer):
|
||||
return self.get_effective_min_blob_data_payment_rate()
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self._payment_rate_manager.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, payment_rate):
|
||||
return payment_rate >= self.get_effective_min_blob_data_payment_rate()
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
response = self._payment_rate_manager.strategy.respond_to_offer(offer, peer, blobs)
|
||||
return response.accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self._payment_rate_manager.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self._payment_rate_manager.strategy.offer_accepted(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_effective_min_blob_data_payment_rate(self):
|
||||
rate = self.min_blob_data_payment_rate
|
||||
@@ -42,4 +49,4 @@ class LiveStreamPaymentRateManager(object):
|
||||
return rate
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
self.points_paid += amount
|
||||
@@ -1,8 +1,11 @@
|
||||
# pylint: skip-file
|
||||
# This file is not maintained, but might be used in the future
|
||||
#
|
||||
import logging
|
||||
import sys
|
||||
from lbrynet.lbrylive.LiveStreamCreator import StdOutLiveStreamCreator
|
||||
from lbrynet.core.BlobManager import TempBlobManager
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
from lbrynet.core.Session import Session
|
||||
from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
|
||||
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
|
||||
@@ -13,9 +16,10 @@ from lbrynet.dht.node import Node
|
||||
from twisted.internet import defer, task
|
||||
|
||||
|
||||
class LBRYStdinUploader():
|
||||
class StdinUploader():
|
||||
"""This class reads from standard in, creates a stream, and makes it available on the network."""
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes):
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager, blob_manager_class=TempBlobManager):
|
||||
"""
|
||||
@param peer_port: the network port on which to listen for peers
|
||||
|
||||
@@ -25,8 +29,8 @@ class LBRYStdinUploader():
|
||||
"""
|
||||
self.peer_port = peer_port
|
||||
self.lbry_server_port = None
|
||||
self.session = LBRYSession(blob_manager_class=TempBlobManager,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager,
|
||||
self.session = Session(blob_manager_class=blob_manager_class,
|
||||
stream_info_manager_class=stream_info_manager_class,
|
||||
dht_node_class=Node, dht_node_port=dht_node_port,
|
||||
known_dht_nodes=known_dht_nodes, peer_port=self.peer_port,
|
||||
use_upnp=False)
|
||||
@@ -95,9 +99,9 @@ def launch_stdin_uploader():
|
||||
|
||||
logging.basicConfig(level=logging.WARNING, filename="ul.log")
|
||||
if len(sys.argv) == 4:
|
||||
uploader = LBRYStdinUploader(int(sys.argv[2]), int(sys.argv[3]), [])
|
||||
uploader = StdinUploader(int(sys.argv[2]), int(sys.argv[3]), [])
|
||||
elif len(sys.argv) == 6:
|
||||
uploader = LBRYStdinUploader(int(sys.argv[2]), int(sys.argv[3]), [(sys.argv[4], int(sys.argv[5]))])
|
||||
uploader = StdinUploader(int(sys.argv[2]), int(sys.argv[3]), [(sys.argv[4], int(sys.argv[5]))])
|
||||
else:
|
||||
print "Usage: lbrynet-stdin-uploader <stream_name> <peer_port> <dht_node_port>" \
|
||||
" [<dht_bootstrap_host> <dht_bootstrap port>]"
|
||||
@@ -114,4 +118,4 @@ def launch_stdin_uploader():
|
||||
d.addCallback(lambda _: start_stdin_uploader())
|
||||
d.addCallback(lambda _: shut_down())
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', uploader.shut_down)
|
||||
reactor.run()
|
||||
reactor.run()
|
||||
@@ -1,9 +1,12 @@
|
||||
# pylint: skip-file
|
||||
# This file is not maintained, but might be used in the future
|
||||
#
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from lbrynet.lbrynet_console.plugins.LBRYLive.LBRYLiveStreamDownloader import LBRYLiveStreamDownloader
|
||||
from lbrynet.lbrylive.client.LiveStreamDownloader import LiveStreamDownloader
|
||||
from lbrynet.core.BlobManager import TempBlobManager
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
from lbrynet.core.Session import Session
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorReader
|
||||
from lbrynet.lbrylive.PaymentRateManager import BaseLiveStreamPaymentRateManager
|
||||
@@ -13,17 +16,19 @@ from lbrynet.dht.node import Node
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
class LBRYStdoutDownloader():
|
||||
class StdoutDownloader():
|
||||
"""This class downloads a live stream from the network and outputs it to standard out."""
|
||||
def __init__(self, dht_node_port, known_dht_nodes):
|
||||
def __init__(self, dht_node_port, known_dht_nodes,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager, blob_manager_class=TempBlobManager):
|
||||
"""
|
||||
@param dht_node_port: the network port on which to listen for DHT node requests
|
||||
|
||||
@param known_dht_nodes: a list of (ip_address, dht_port) which will be used to join the DHT network
|
||||
|
||||
"""
|
||||
self.session = LBRYSession(blob_manager_class=TempBlobManager,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager,
|
||||
|
||||
self.session = Session(blob_manager_class=blob_manager_class,
|
||||
stream_info_manager_class=stream_info_manager_class,
|
||||
dht_node_class=Node, dht_node_port=dht_node_port, known_dht_nodes=known_dht_nodes,
|
||||
use_upnp=False)
|
||||
self.payment_rate_manager = BaseLiveStreamPaymentRateManager()
|
||||
@@ -50,7 +55,7 @@ class LBRYStdoutDownloader():
|
||||
d.addCallbacks(self.read_sd_file)
|
||||
|
||||
def start_stream(stream_hash):
|
||||
consumer = LBRYLiveStreamDownloader(stream_hash, self.session.peer_finder,
|
||||
consumer = LiveStreamDownloader(stream_hash, self.session.peer_finder,
|
||||
self.session.rate_limiter, self.session.blob_manager,
|
||||
self.stream_info_manager, self.payment_rate_manager,
|
||||
self.session.wallet)
|
||||
@@ -71,9 +76,9 @@ def launch_stdout_downloader():
|
||||
|
||||
logging.basicConfig(level=logging.WARNING, filename="dl.log")
|
||||
if len(sys.argv) == 3:
|
||||
downloader = LBRYStdoutDownloader(int(sys.argv[2]), [])
|
||||
downloader = StdoutDownloader(int(sys.argv[2]), [])
|
||||
elif len(sys.argv) == 5:
|
||||
downloader = LBRYStdoutDownloader(int(sys.argv[2]), [(sys.argv[3], int(sys.argv[4]))])
|
||||
downloader = StdoutDownloader(int(sys.argv[2]), [(sys.argv[3], int(sys.argv[4]))])
|
||||
else:
|
||||
print "Usage: lbrynet-stdout-downloader <sd_hash> <peer_port> <dht_node_port>" \
|
||||
" [<dht_bootstrap_host> <dht_bootstrap port>]"
|
||||
@@ -93,4 +98,4 @@ def launch_stdout_downloader():
|
||||
d.addErrback(print_error)
|
||||
d.addCallback(lambda _: shut_down())
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', downloader.shut_down)
|
||||
reactor.run()
|
||||
reactor.run()
|
||||
@@ -2,17 +2,20 @@ import binascii
|
||||
import logging
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
|
||||
from twisted.internet import defer, threads
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
|
||||
from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
|
||||
from lbrynet.interfaces import IStreamDescriptorValidator
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
LiveStreamType = "lbrylive"
|
||||
|
||||
|
||||
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
logging.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
log.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
hex_stream_name = sd_info['stream_name']
|
||||
public_key = sd_info['public_key']
|
||||
key = sd_info['key']
|
||||
@@ -30,7 +33,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
iv = blob['iv']
|
||||
signature = blob['signature']
|
||||
crypt_blobs.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
|
||||
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
log.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
d = stream_info_manager.save_stream(stream_hash, public_key, hex_stream_name,
|
||||
key, crypt_blobs)
|
||||
|
||||
@@ -81,14 +84,14 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
|
||||
return d
|
||||
|
||||
|
||||
class LBRYLiveStreamDescriptorValidator(object):
|
||||
class LiveStreamDescriptorValidator(object):
|
||||
implements(IStreamDescriptorValidator)
|
||||
|
||||
def __init__(self, raw_info):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
hex_stream_name = self.raw_info['stream_name']
|
||||
public_key = self.raw_info['public_key']
|
||||
key = self.raw_info['key']
|
||||
@@ -98,7 +101,7 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
h.update(public_key)
|
||||
h.update(key)
|
||||
if h.hexdigest() != stream_hash:
|
||||
raise ValueError("Stream hash does not match stream metadata")
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
blobs = self.raw_info['blobs']
|
||||
|
||||
def check_blob_signatures():
|
||||
@@ -121,11 +124,14 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
hashsum.update(iv)
|
||||
hashsum.update(str(length))
|
||||
if not verify_signature(hashsum.digest(), signature, public_key):
|
||||
raise ValueError("Invalid signature in stream descriptor")
|
||||
raise InvalidStreamDescriptorError("Invalid signature in stream descriptor")
|
||||
|
||||
return threads.deferToThread(check_blob_signatures)
|
||||
|
||||
def info_to_show(self):
|
||||
info = []
|
||||
info.append(("stream_name", binascii.unhexlify(self.raw_info.get("stream_name"))))
|
||||
return info
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
return None
|
||||
@@ -1,5 +1,5 @@
|
||||
import binascii
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from zope.interface import implements
|
||||
from lbrynet.lbrylive.client.LiveStreamMetadataHandler import LiveStreamMetadataHandler
|
||||
@@ -9,9 +9,10 @@ from lbrynet.lbrylive.StreamDescriptor import save_sd_info
|
||||
from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
|
||||
from twisted.internet import defer, threads # , process
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType
|
||||
|
||||
|
||||
class LiveStreamDownloader(CryptStreamDownloader):
|
||||
class _LiveStreamDownloader(CryptStreamDownloader):
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet, upload_allowed):
|
||||
@@ -38,10 +39,10 @@ class LiveStreamDownloader(CryptStreamDownloader):
|
||||
return defer.succeed(True)
|
||||
|
||||
|
||||
class LBRYLiveStreamDownloader(LiveStreamDownloader):
|
||||
class LiveStreamDownloader(_LiveStreamDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet, upload_allowed):
|
||||
LiveStreamDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
_LiveStreamDownloader.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
|
||||
|
||||
#self.writer = process.ProcessWriter(reactor, self, 'write', 1)
|
||||
@@ -64,17 +65,17 @@ class LBRYLiveStreamDownloader(LiveStreamDownloader):
|
||||
return write_func
|
||||
|
||||
|
||||
class FullLiveStreamDownloader(LiveStreamDownloader):
|
||||
class FullLiveStreamDownloader(_LiveStreamDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
payment_rate_manager, wallet, upload_allowed):
|
||||
LiveStreamDownloader.__init__(self, stream_hash, peer_finder, rate_limiter,
|
||||
_LiveStreamDownloader.__init__(self, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, stream_info_manager, payment_rate_manager,
|
||||
wallet, upload_allowed)
|
||||
self.file_handle = None
|
||||
self.file_name = None
|
||||
|
||||
def set_stream_info(self):
|
||||
d = LiveStreamDownloader.set_stream_info(self)
|
||||
d = _LiveStreamDownloader.set_stream_info(self)
|
||||
|
||||
def set_file_name_if_unset():
|
||||
if not self.file_name:
|
||||
@@ -85,9 +86,9 @@ class FullLiveStreamDownloader(LiveStreamDownloader):
|
||||
d.addCallback(lambda _: set_file_name_if_unset())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
def stop(self, err=None):
|
||||
d = self._close_file()
|
||||
d.addBoth(lambda _: LiveStreamDownloader.stop(self))
|
||||
d.addBoth(lambda _: _LiveStreamDownloader.stop(self, err))
|
||||
return d
|
||||
|
||||
def _start(self):
|
||||
@@ -95,7 +96,7 @@ class FullLiveStreamDownloader(LiveStreamDownloader):
|
||||
d = self._open_file()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: LiveStreamDownloader._start(self))
|
||||
d.addCallback(lambda _: _LiveStreamDownloader._start(self))
|
||||
return d
|
||||
|
||||
def _open_file(self):
|
||||
@@ -138,43 +139,44 @@ class FullLiveStreamDownloaderFactory(object):
|
||||
self.wallet = wallet
|
||||
self.default_payment_rate_manager = default_payment_rate_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for metadata (None means use application default)",
|
||||
"metadata payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
def make_downloader(self, metadata, options, payment_rate_manager):
|
||||
# TODO: check options for payment rate manager parameters
|
||||
payment_rate_manager = LiveStreamPaymentRateManager(self.default_payment_rate_manager,
|
||||
prm = LiveStreamPaymentRateManager(self.default_payment_rate_manager,
|
||||
payment_rate_manager)
|
||||
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
stream_downloader = FullLiveStreamDownloader(stream_hash, self.peer_finder, self.rate_limiter,
|
||||
self.blob_manager, self.stream_info_manager,
|
||||
payment_rate_manager, self.wallet, True)
|
||||
prm, self.wallet, True)
|
||||
# TODO: change upload_allowed=True above to something better
|
||||
d = stream_downloader.set_stream_info()
|
||||
d.addCallback(lambda _: stream_downloader)
|
||||
return d
|
||||
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
return d
|
||||
|
||||
|
||||
def add_full_live_stream_downloader_to_sd_identifier(session, stream_info_manager, sd_identifier,
|
||||
base_live_stream_payment_rate_manager):
|
||||
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
|
||||
session.rate_limiter,
|
||||
session.blob_manager,
|
||||
stream_info_manager,
|
||||
session.wallet,
|
||||
base_live_stream_payment_rate_manager)
|
||||
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
|
||||
@@ -3,13 +3,16 @@ import logging
|
||||
from zope.interface import implements
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.conf import MAX_BLOB_INFOS_TO_REQUEST
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientPaidRequest
|
||||
from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
|
||||
from lbrynet.interfaces import IRequestCreator, IMetadataHandler
|
||||
from lbrynet.core.Error import InsufficientFundsError, InvalidResponseError, RequestCanceledError
|
||||
from lbrynet.core.Error import NoResponseError
|
||||
from lbrynet.core.Error import NoResponseError, ConnectionClosedBeforeResponseError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamMetadataHandler(object):
|
||||
@@ -79,7 +82,7 @@ class LiveStreamMetadataHandler(object):
|
||||
r = None
|
||||
if self._finished_discovery() is False:
|
||||
r = self.stream_hash
|
||||
logging.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
|
||||
log.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -101,7 +104,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if blob_hash is not None:
|
||||
infos.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
|
||||
else:
|
||||
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
self._final_blob_num = blob_num - 1
|
||||
return infos
|
||||
|
||||
@@ -133,8 +136,8 @@ class LiveStreamMetadataHandler(object):
|
||||
if count is not None:
|
||||
further_blobs_request['count'] = count
|
||||
else:
|
||||
further_blobs_request['count'] = MAX_BLOB_INFOS_TO_REQUEST
|
||||
logging.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
|
||||
further_blobs_request['count'] = settings.MAX_BLOB_INFOS_TO_REQUEST
|
||||
log.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
|
||||
r_dict = {'further_blobs': further_blobs_request}
|
||||
response_identifier = 'further_blobs'
|
||||
request = ClientPaidRequest(r_dict, response_identifier, further_blobs_request['count'])
|
||||
@@ -142,7 +145,7 @@ class LiveStreamMetadataHandler(object):
|
||||
return None
|
||||
|
||||
def _get_discovery_params(self):
|
||||
logging.debug("In _get_discovery_params")
|
||||
log.debug("In _get_discovery_params")
|
||||
stream_position = self.download_manager.stream_position()
|
||||
blobs = self.download_manager.blobs
|
||||
if blobs:
|
||||
@@ -153,7 +156,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if final_blob_num is not None:
|
||||
last_blob_num = final_blob_num
|
||||
if self.download_whole is False:
|
||||
logging.debug("download_whole is False")
|
||||
log.debug("download_whole is False")
|
||||
if final_blob_num is not None:
|
||||
for i in xrange(stream_position, final_blob_num + 1):
|
||||
if not i in blobs:
|
||||
@@ -171,7 +174,7 @@ class LiveStreamMetadataHandler(object):
|
||||
return self.stream_hash, blobs[last_blob_num].blob_hash, 'end', -1 * self.max_before_skip_ahead
|
||||
else:
|
||||
return self.stream_hash, None, 'end', -1 * self.max_before_skip_ahead
|
||||
logging.debug("download_whole is True")
|
||||
log.debug("download_whole is True")
|
||||
beginning = None
|
||||
end = None
|
||||
for i in xrange(stream_position, last_blob_num + 1):
|
||||
@@ -187,17 +190,17 @@ class LiveStreamMetadataHandler(object):
|
||||
break
|
||||
if beginning is None:
|
||||
if final_blob_num is not None:
|
||||
logging.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
|
||||
str(last_blob_num + 1))
|
||||
log.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
|
||||
str(last_blob_num + 1))
|
||||
return None
|
||||
else:
|
||||
logging.debug("Discovery is not finished. final blob num is unknown.")
|
||||
log.debug("Discovery is not finished. final blob num is unknown.")
|
||||
if last_blob_num != -1:
|
||||
return self.stream_hash, blobs[last_blob_num].blob_hash, None, None
|
||||
else:
|
||||
return self.stream_hash, 'beginning', None, None
|
||||
else:
|
||||
logging.info("Discovery is not finished. Not all blobs are known.")
|
||||
log.info("Discovery is not finished. Not all blobs are known.")
|
||||
return self.stream_hash, beginning, end, None
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
@@ -244,7 +247,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if response == "RATE_ACCEPTED":
|
||||
return True
|
||||
else:
|
||||
logging.info("Rate offer has been rejected by %s", str(peer))
|
||||
log.info("Rate offer has been rejected by %s", str(peer))
|
||||
del self._protocol_prices[protocol]
|
||||
self._price_disagreements.append(peer)
|
||||
return True
|
||||
@@ -263,8 +266,8 @@ class LiveStreamMetadataHandler(object):
|
||||
if not 'blob_infos' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_infos'")
|
||||
raw_blob_infos = response['blob_infos']
|
||||
logging.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
|
||||
logging.debug("blobs: %s", str(raw_blob_infos))
|
||||
log.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
|
||||
log.debug("blobs: %s", str(raw_blob_infos))
|
||||
for raw_blob_info in raw_blob_infos:
|
||||
length = raw_blob_info['length']
|
||||
if length != 0:
|
||||
@@ -276,10 +279,10 @@ class LiveStreamMetadataHandler(object):
|
||||
iv = raw_blob_info['iv']
|
||||
signature = raw_blob_info['signature']
|
||||
blob_info = LiveBlobInfo(blob_hash, num, length, iv, revision, signature)
|
||||
logging.debug("Learned about a potential blob: %s", str(blob_hash))
|
||||
log.debug("Learned about a potential blob: %s", str(blob_hash))
|
||||
if self._verify_blob(blob_info):
|
||||
if blob_hash is None:
|
||||
logging.info("Setting _final_blob_num to %s", str(num - 1))
|
||||
log.info("Setting _final_blob_num to %s", str(num - 1))
|
||||
self._final_blob_num = num - 1
|
||||
else:
|
||||
blob_infos.append(blob_info)
|
||||
@@ -289,7 +292,7 @@ class LiveStreamMetadataHandler(object):
|
||||
|
||||
def add_blobs_to_download_manager():
|
||||
blob_nums = [b.blob_num for b in blob_infos]
|
||||
logging.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
|
||||
log.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
|
||||
self.download_manager.add_blobs_to_download(blob_infos)
|
||||
|
||||
d.addCallback(lambda _: add_blobs_to_download_manager())
|
||||
@@ -308,12 +311,12 @@ class LiveStreamMetadataHandler(object):
|
||||
return d
|
||||
|
||||
def _verify_blob(self, blob):
|
||||
logging.debug("Got an unverified blob to check:")
|
||||
logging.debug("blob_hash: %s", blob.blob_hash)
|
||||
logging.debug("blob_num: %s", str(blob.blob_num))
|
||||
logging.debug("revision: %s", str(blob.revision))
|
||||
logging.debug("iv: %s", blob.iv)
|
||||
logging.debug("length: %s", str(blob.length))
|
||||
log.debug("Got an unverified blob to check:")
|
||||
log.debug("blob_hash: %s", blob.blob_hash)
|
||||
log.debug("blob_num: %s", str(blob.blob_num))
|
||||
log.debug("revision: %s", str(blob.revision))
|
||||
log.debug("iv: %s", blob.iv)
|
||||
log.debug("length: %s", str(blob.length))
|
||||
hashsum = get_lbry_hash_obj()
|
||||
hashsum.update(self.stream_hash)
|
||||
if blob.length != 0:
|
||||
@@ -322,12 +325,12 @@ class LiveStreamMetadataHandler(object):
|
||||
hashsum.update(str(blob.revision))
|
||||
hashsum.update(blob.iv)
|
||||
hashsum.update(str(blob.length))
|
||||
logging.debug("hexdigest to be verified: %s", hashsum.hexdigest())
|
||||
log.debug("hexdigest to be verified: %s", hashsum.hexdigest())
|
||||
if verify_signature(hashsum.digest(), blob.signature, self.stream_pub_key):
|
||||
logging.debug("Blob info is valid")
|
||||
log.debug("Blob info is valid")
|
||||
return True
|
||||
else:
|
||||
logging.debug("The blob info is invalid")
|
||||
log.debug("The blob info is invalid")
|
||||
return False
|
||||
|
||||
def _request_failed(self, reason, peer):
|
||||
@@ -335,8 +338,9 @@ class LiveStreamMetadataHandler(object):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
|
||||
self._update_local_score(peer, -5.0)
|
||||
peer.update_score(-10.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
@@ -0,0 +1,73 @@
|
||||
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType, LiveStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_live_stream_to_sd_identifier(sd_identifier, base_live_stream_payment_rate_manager):
|
||||
sd_identifier.add_stream_type(LiveStreamType, LiveStreamDescriptorValidator,
|
||||
LiveStreamOptions(base_live_stream_payment_rate_manager))
|
||||
|
||||
|
||||
class LiveStreamOptions(object):
|
||||
def __init__(self, base_live_stream_payment_rate_manager):
|
||||
self.base_live_stream_prm = base_live_stream_payment_rate_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.min_blob_data_payment_rate
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(None,
|
||||
"No change",
|
||||
"No change"),
|
||||
DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)),
|
||||
DownloadOptionChoice(float,
|
||||
"Rate in LBC/MB",
|
||||
"Rate in LBC/MB")
|
||||
],
|
||||
"rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(None,
|
||||
"No change",
|
||||
"No change"),
|
||||
DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate),
|
||||
"Default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)),
|
||||
DownloadOptionChoice(float,
|
||||
"Rate in LBC/MB",
|
||||
"Rate in LBC/MB")
|
||||
],
|
||||
"rate which will be paid for metadata",
|
||||
"metadata payment rate",
|
||||
None,
|
||||
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(True,
|
||||
"Allow reuploading data downloaded for this file",
|
||||
"Allow reuploading"),
|
||||
DownloadOptionChoice(False,
|
||||
"Disallow reuploading data downloaded for this file",
|
||||
"Disallow reuploading")
|
||||
],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True,
|
||||
"Allow"
|
||||
),
|
||||
]
|
||||
return options
|
||||
@@ -3,6 +3,9 @@ from lbrynet.core.client.StreamProgressManager import StreamProgressManager
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamProgressManager(StreamProgressManager):
|
||||
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False,
|
||||
download_whole=True, max_before_skip_ahead=5):
|
||||
@@ -52,9 +55,9 @@ class LiveStreamProgressManager(StreamProgressManager):
|
||||
return
|
||||
|
||||
blobs = self.download_manager.blobs
|
||||
logging.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
|
||||
log.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
|
||||
if blobs:
|
||||
logging.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
|
||||
log.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
|
||||
if self.outputting_d is None:
|
||||
self.outputting_d = defer.Deferred()
|
||||
|
||||
@@ -71,15 +74,15 @@ class LiveStreamProgressManager(StreamProgressManager):
|
||||
reactor.callLater(0, self._output_loop)
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(current_blob_num))
|
||||
log.info("Outputting blob %s", str(current_blob_num))
|
||||
self.provided_blob_nums.append(current_blob_num)
|
||||
d = self.download_manager.handle_blob(current_blob_num)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
elif blobs and max(blobs.iterkeys()) > self.last_blob_outputted + self.max_before_skip_ahead - 1:
|
||||
self.last_blob_outputted += 1
|
||||
logging.info("Skipping blob number %s due to knowing about blob number %s",
|
||||
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
|
||||
log.info("Skipping blob number %s due to knowing about blob number %s",
|
||||
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
|
||||
self._finished_with_blob(current_blob_num)
|
||||
reactor.callLater(0, self._output_loop)
|
||||
else:
|
||||
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfoQueryHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -54,7 +57,7 @@ class CryptBlobInfoQueryHandler(object):
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
further_blobs_request = queries[self.query_identifiers[1]]
|
||||
logging.debug("Received the client's request for additional blob information")
|
||||
log.debug("Received the client's request for additional blob information")
|
||||
|
||||
if self.blob_info_payment_rate is None:
|
||||
response['further_blobs'] = {'error': 'RATE_UNSET'}
|
||||
@@ -62,7 +65,7 @@ class CryptBlobInfoQueryHandler(object):
|
||||
|
||||
def count_and_charge(blob_infos):
|
||||
if len(blob_infos) != 0:
|
||||
logging.info("Responding with %s infos", str(len(blob_infos)))
|
||||
log.info("Responding with %s infos", str(len(blob_infos)))
|
||||
expected_payment = 1.0 * len(blob_infos) * self.blob_info_payment_rate / 1000.0
|
||||
self.wallet.add_expected_payment(self.peer, expected_payment)
|
||||
self.peer.update_stats('uploaded_crypt_blob_infos', len(blob_infos))
|
||||
|
||||
@@ -0,0 +1,578 @@
|
||||
# TODO: THERE IS A LOT OF CODE IN THIS MODULE THAT SHOULD BE REMOVED
|
||||
# AS IT IS REPEATED IN THE LBRYDaemon MODULE
|
||||
import logging
|
||||
import os.path
|
||||
import argparse
|
||||
import requests
|
||||
import locale
|
||||
import sys
|
||||
import webbrowser
|
||||
|
||||
if sys.platform == "darwin":
|
||||
from appdirs import user_data_dir
|
||||
from yapsy.PluginManager import PluginManager
|
||||
from twisted.internet import defer, threads, stdio, task, error
|
||||
from lbrynet.lbrynet_daemon.auth.client import LBRYAPIClient
|
||||
|
||||
from lbrynet import analytics
|
||||
from lbrynet.core.Session import Session
|
||||
from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl
|
||||
from lbrynet.lbrynet_console.Settings import Settings
|
||||
from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.conf import settings
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
|
||||
from lbrynet.core.PTCWallet import PTCWallet
|
||||
from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileOpenerFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadataManager, TempEncryptedFileMetadataManager
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ApplicationStatusFactory, GetWalletBalancesFactory, ShutDownFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ImmediateAnnounceAllBlobsFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import EncryptedFileStatusFactory, DeleteEncryptedFileChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ToggleEncryptedFileRunningChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyApplicationDefaultsFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import CreateEncryptedFileFactory, PublishStreamDescriptorChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ShowPublishedSDHashesChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import CreatePlainStreamDescriptorChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ShowEncryptedFileStreamHashChooserFactory, AddStreamFromHashFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import AddStreamFromSDFactory, AddStreamFromLBRYcrdNameFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ClaimNameFactory, GetNewWalletAddressFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ShowServerStatusFactory, ModifyServerSettingsFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyEncryptedFileOptionsChooserFactory, StatusFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import PeerStatsAndSettingsChooserFactory, PublishFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import BlockchainStatusFactory
|
||||
from lbrynet.core.Wallet import LBRYcrdWallet, LBRYumWallet
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
alert = logging.getLogger("lbryalert." + __name__)
|
||||
|
||||
|
||||
class Console():
|
||||
"""A class which can upload and download file streams to and from the network"""
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes, fake_wallet,
|
||||
lbrycrd_conf, lbrycrd_dir, use_upnp, data_dir, created_data_dir,
|
||||
lbrycrdd_path):
|
||||
"""
|
||||
@param peer_port: the network port on which to listen for peers
|
||||
|
||||
@param dht_node_port: the network port on which to listen for dht node requests
|
||||
|
||||
@param known_dht_nodes: a list of (ip_address, dht_port) which will be used to join the DHT network
|
||||
"""
|
||||
self.peer_port = peer_port
|
||||
self.dht_node_port = dht_node_port
|
||||
self.known_dht_nodes = known_dht_nodes
|
||||
self.fake_wallet = fake_wallet
|
||||
self.lbrycrd_conf = lbrycrd_conf
|
||||
self.lbrycrd_dir = lbrycrd_dir
|
||||
if not self.lbrycrd_dir:
|
||||
if sys.platform == "darwin":
|
||||
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), "Library/Application Support/lbrycrd")
|
||||
else:
|
||||
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
|
||||
if not self.lbrycrd_conf:
|
||||
self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf")
|
||||
self.lbrycrdd_path = lbrycrdd_path
|
||||
self.use_upnp = use_upnp
|
||||
self.lbry_server_port = None
|
||||
self.session = None
|
||||
self.lbry_file_metadata_manager = None
|
||||
self.lbry_file_manager = None
|
||||
self.db_dir = data_dir
|
||||
self.current_db_revision = 1
|
||||
self.blobfile_dir = os.path.join(self.db_dir, "blobfiles")
|
||||
self.created_data_dir = created_data_dir
|
||||
self.plugin_manager = PluginManager()
|
||||
self.plugin_manager.setPluginPlaces([
|
||||
os.path.join(self.db_dir, "plugins"),
|
||||
os.path.join(os.path.dirname(__file__), "plugins"),
|
||||
])
|
||||
self.command_handlers = []
|
||||
self.query_handlers = {}
|
||||
|
||||
self.settings = Settings(self.db_dir)
|
||||
self.blob_request_payment_rate_manager = None
|
||||
self.lbryid = None
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.plugin_objects = []
|
||||
self.db_migration_revisions = None
|
||||
|
||||
def start(self):
|
||||
"""Initialize the session and restore everything to its saved state"""
|
||||
d = self._setup_controller()
|
||||
d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
|
||||
d.addCallback(lambda _: self._check_db_migration())
|
||||
d.addCallback(lambda _: self._get_settings())
|
||||
d.addCallback(lambda _: self._get_session())
|
||||
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
|
||||
d.addCallback(lambda _: self._setup_lbry_file_manager())
|
||||
d.addCallback(lambda _: self._setup_lbry_file_opener())
|
||||
d.addCallback(lambda _: self._setup_control_handlers())
|
||||
d.addCallback(lambda _: self._setup_query_handlers())
|
||||
d.addCallback(lambda _: self._load_plugins())
|
||||
d.addCallback(lambda _: self._setup_server())
|
||||
d.addCallback(lambda _: self._start_controller())
|
||||
d.addErrback(self._show_start_error)
|
||||
return d
|
||||
|
||||
def _show_start_error(self, error):
|
||||
print error.getTraceback()
|
||||
log.error("An error occurred during start up: %s", error.getTraceback())
|
||||
return error
|
||||
|
||||
def _show_shutdown_error(self, error):
|
||||
print error.getErrorMessage()
|
||||
log.error("An error occurred during shutdown: %s", error.getTraceback())
|
||||
return error
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop the session, all currently running streams, and stop the server"""
|
||||
d = self._shut_down()
|
||||
if self.session is not None:
|
||||
d.addCallback(lambda _: self.session.shut_down())
|
||||
d.addErrback(self._show_shutdown_error)
|
||||
return d
|
||||
|
||||
def add_control_handlers(self, control_handlers):
|
||||
for control_handler in control_handlers:
|
||||
self.command_handlers.append(control_handler)
|
||||
|
||||
def add_query_handlers(self, query_handlers):
|
||||
|
||||
def _set_query_handlers(statuses):
|
||||
from future_builtins import zip
|
||||
for handler, (success, status) in zip(query_handlers, statuses):
|
||||
if success is True:
|
||||
self.query_handlers[handler] = status
|
||||
|
||||
ds = []
|
||||
for handler in query_handlers:
|
||||
ds.append(self.settings.get_query_handler_status(handler.get_primary_query_identifier()))
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(_set_query_handlers)
|
||||
return dl
|
||||
|
||||
def _setup_data_directory(self):
|
||||
alert.info("Loading databases...")
|
||||
if self.created_data_dir:
|
||||
db_revision = open(os.path.join(self.db_dir, "db_revision"), mode='w')
|
||||
db_revision.write(str(self.current_db_revision))
|
||||
db_revision.close()
|
||||
log.debug("Created the db revision file: %s", str(os.path.join(self.db_dir, "db_revision")))
|
||||
if not os.path.exists(self.blobfile_dir):
|
||||
os.mkdir(self.blobfile_dir)
|
||||
log.debug("Created the blobfile directory: %s", str(self.blobfile_dir))
|
||||
|
||||
def _check_db_migration(self):
|
||||
old_revision = 1
|
||||
db_revision_file = os.path.join(self.db_dir, "db_revision")
|
||||
if os.path.exists(db_revision_file):
|
||||
old_revision = int(open(db_revision_file).read().strip())
|
||||
if old_revision < self.current_db_revision:
|
||||
from lbrynet.db_migrator import dbmigrator
|
||||
print "Upgrading your databases..."
|
||||
d = threads.deferToThread(dbmigrator.migrate_db, self.db_dir, old_revision, self.current_db_revision)
|
||||
|
||||
def print_success(old_dirs):
|
||||
success_string = "Finished upgrading the databases. It is now safe to delete the"
|
||||
success_string += " following directories, if you feel like it. It won't make any"
|
||||
success_string += " difference.\nAnyway here they are: "
|
||||
for i, old_dir in enumerate(old_dirs):
|
||||
success_string += old_dir
|
||||
if i + 1 < len(old_dir):
|
||||
success_string += ", "
|
||||
print success_string
|
||||
|
||||
d.addCallback(print_success)
|
||||
return d
|
||||
return defer.succeed(True)
|
||||
|
||||
def _get_settings(self):
|
||||
d = self.settings.start()
|
||||
d.addCallback(lambda _: self.settings.get_lbryid())
|
||||
d.addCallback(self.set_lbryid)
|
||||
return d
|
||||
|
||||
def set_lbryid(self, lbryid):
|
||||
if lbryid is None:
|
||||
return self._make_lbryid()
|
||||
else:
|
||||
self.lbryid = lbryid
|
||||
|
||||
def _make_lbryid(self):
|
||||
self.lbryid = generate_id()
|
||||
d = self.settings.save_lbryid(self.lbryid)
|
||||
return d
|
||||
|
||||
def _get_session(self):
|
||||
def get_default_data_rate():
|
||||
d = self.settings.get_default_data_payment_rate()
|
||||
d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else settings.data_rate})
|
||||
return d
|
||||
|
||||
def get_wallet():
|
||||
if self.fake_wallet:
|
||||
d = defer.succeed(PTCWallet(self.db_dir))
|
||||
elif self.lbrycrdd_path is not None:
|
||||
d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.lbrycrd_dir,
|
||||
wallet_conf=self.lbrycrd_conf,
|
||||
lbrycrdd_path=self.lbrycrdd_path))
|
||||
else:
|
||||
d = defer.succeed(LBRYumWallet(self.db_dir))
|
||||
d.addCallback(lambda wallet: {"wallet": wallet})
|
||||
return d
|
||||
|
||||
d1 = get_default_data_rate()
|
||||
d2 = get_wallet()
|
||||
|
||||
def combine_results(results):
|
||||
r = {}
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
r.update(result)
|
||||
return r
|
||||
|
||||
def create_session(results):
|
||||
|
||||
alert.info("Databases loaded.")
|
||||
|
||||
self.session = Session(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid,
|
||||
blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port,
|
||||
known_dht_nodes=self.known_dht_nodes, peer_port=self.peer_port,
|
||||
use_upnp=self.use_upnp, wallet=results['wallet'])
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
dl.addCallback(combine_results)
|
||||
|
||||
dl.addCallback(create_session)
|
||||
|
||||
dl.addCallback(lambda _: self.session.setup())
|
||||
|
||||
dl.addCallback(lambda _: self.check_first_run())
|
||||
|
||||
dl.addCallback(self._show_first_run_result)
|
||||
|
||||
return dl
|
||||
|
||||
def check_first_run(self):
|
||||
d = self.session.wallet.is_first_run()
|
||||
d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run else 0.0)
|
||||
return d
|
||||
|
||||
def _do_first_run(self):
|
||||
d = self.session.wallet.get_new_address()
|
||||
|
||||
def send_request(url, data):
|
||||
r = requests.post(url, json=data)
|
||||
if r.status_code == 200:
|
||||
return r.json()['credits_sent']
|
||||
return 0.0
|
||||
|
||||
def log_error(err):
|
||||
log.warning("unable to request free credits. %s", err.getErrorMessage())
|
||||
return 0.0
|
||||
|
||||
def request_credits(address):
|
||||
url = "http://credreq.lbry.io/requestcredits"
|
||||
data = {"address": address}
|
||||
d = threads.deferToThread(send_request, url, data)
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
d.addCallback(request_credits)
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def _show_first_run_result(credits_received):
|
||||
if credits_received != 0.0:
|
||||
points_string = locale.format_string("%.2f LBC", (round(credits_received, 2),),
|
||||
grouping=True)
|
||||
alert.info("\n\nThank you for testing the alpha version of LBRY!\n\n"
|
||||
"You have been given %s for free because we love you.\n"
|
||||
"Please give them a few minutes to show up while you\n"
|
||||
"catch up with our blockchain.\n", points_string)
|
||||
|
||||
def _setup_lbry_file_manager(self):
|
||||
self.lbry_file_metadata_manager = DBEncryptedFileMetadataManager(self.db_dir)
|
||||
d = self.lbry_file_metadata_manager.setup()
|
||||
|
||||
def set_lbry_file_manager():
|
||||
self.lbry_file_manager = EncryptedFileManager(self.session, self.lbry_file_metadata_manager, self.sd_identifier)
|
||||
return self.lbry_file_manager.setup()
|
||||
|
||||
d.addCallback(lambda _: set_lbry_file_manager())
|
||||
|
||||
return d
|
||||
|
||||
def _setup_lbry_file_opener(self):
|
||||
stream_info_manager = TempEncryptedFileMetadataManager()
|
||||
downloader_factory = EncryptedFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter,
|
||||
self.session.blob_manager, stream_info_manager,
|
||||
self.session.wallet)
|
||||
self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, downloader_factory)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _setup_control_handlers(self):
|
||||
handlers = [
|
||||
ApplicationStatusFactory(self.session.rate_limiter, self.session.dht_node),
|
||||
GetWalletBalancesFactory(self.session.wallet),
|
||||
ModifyApplicationDefaultsFactory(self),
|
||||
ShutDownFactory(self),
|
||||
PeerStatsAndSettingsChooserFactory(self.session.peer_manager),
|
||||
EncryptedFileStatusFactory(self.lbry_file_manager),
|
||||
AddStreamFromSDFactory(self.sd_identifier, self.session.base_payment_rate_manager,
|
||||
self.session.wallet),
|
||||
DeleteEncryptedFileChooserFactory(self.lbry_file_metadata_manager, self.session.blob_manager,
|
||||
self.lbry_file_manager),
|
||||
ToggleEncryptedFileRunningChooserFactory(self.lbry_file_manager),
|
||||
CreateEncryptedFileFactory(self.session, self.lbry_file_manager),
|
||||
PublishStreamDescriptorChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.session.blob_manager),
|
||||
ShowPublishedSDHashesChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.lbry_file_manager),
|
||||
CreatePlainStreamDescriptorChooserFactory(self.lbry_file_manager),
|
||||
ShowEncryptedFileStreamHashChooserFactory(self.lbry_file_manager),
|
||||
ModifyEncryptedFileOptionsChooserFactory(self.lbry_file_manager),
|
||||
AddStreamFromHashFactory(self.sd_identifier, self.session, self.session.wallet),
|
||||
StatusFactory(self, self.session.rate_limiter, self.lbry_file_manager,
|
||||
self.session.blob_manager, self.session.wallet if not self.fake_wallet else None),
|
||||
ImmediateAnnounceAllBlobsFactory(self.session.blob_manager)
|
||||
]
|
||||
self.add_control_handlers(handlers)
|
||||
if not self.fake_wallet:
|
||||
lbrycrd_handlers = [
|
||||
AddStreamFromLBRYcrdNameFactory(self.sd_identifier, self.session,
|
||||
self.session.wallet),
|
||||
ClaimNameFactory(self.session.wallet, self.lbry_file_manager,
|
||||
self.session.blob_manager),
|
||||
GetNewWalletAddressFactory(self.session.wallet),
|
||||
PublishFactory(self.session, self.lbry_file_manager, self.session.wallet),
|
||||
BlockchainStatusFactory(self.session.wallet)
|
||||
]
|
||||
self.add_control_handlers(lbrycrd_handlers)
|
||||
if self.peer_port is not None:
|
||||
server_handlers = [
|
||||
ShowServerStatusFactory(self),
|
||||
ModifyServerSettingsFactory(self),
|
||||
]
|
||||
self.add_control_handlers(server_handlers)
|
||||
|
||||
def _setup_query_handlers(self):
|
||||
handlers = [
|
||||
self.session.wallet.get_wallet_info_query_handler_factory(),
|
||||
]
|
||||
|
||||
def get_blob_request_handler_factory(rate):
|
||||
self.blob_request_payment_rate_manager = PaymentRateManager(
|
||||
self.session.base_payment_rate_manager, rate
|
||||
)
|
||||
handlers.append(BlobRequestHandlerFactory(
|
||||
self.session.blob_manager, self.session.wallet,
|
||||
self.blob_request_payment_rate_manager, analytics.Track()
|
||||
))
|
||||
|
||||
d1 = self.settings.get_server_data_payment_rate()
|
||||
d1.addCallback(get_blob_request_handler_factory)
|
||||
|
||||
dl = defer.DeferredList([d1])
|
||||
dl.addCallback(lambda _: self.add_query_handlers(handlers))
|
||||
return dl
|
||||
|
||||
def _load_plugins(self):
|
||||
d = threads.deferToThread(self.plugin_manager.collectPlugins)
|
||||
|
||||
def setup_plugins():
|
||||
ds = []
|
||||
for plugin in self.plugin_manager.getAllPlugins():
|
||||
self.plugin_objects.append(plugin.plugin_object)
|
||||
ds.append(plugin.plugin_object.setup(self))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
d.addCallback(lambda _: setup_plugins())
|
||||
return d
|
||||
|
||||
def _stop_plugins(self):
|
||||
ds = []
|
||||
for plugin_object in self.plugin_objects:
|
||||
ds.append(defer.maybeDeferred(plugin_object.stop))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _setup_server(self):
|
||||
|
||||
def restore_running_status(running):
|
||||
if running is True:
|
||||
return self.start_server()
|
||||
return defer.succeed(True)
|
||||
|
||||
dl = self.settings.get_server_running_status()
|
||||
dl.addCallback(restore_running_status)
|
||||
return dl
|
||||
|
||||
def start_server(self):
|
||||
|
||||
if self.peer_port is not None:
|
||||
|
||||
server_factory = ServerProtocolFactory(self.session.rate_limiter,
|
||||
self.query_handlers,
|
||||
self.session.peer_manager)
|
||||
from twisted.internet import reactor
|
||||
try:
|
||||
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.peer_port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer.", str(e))
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop_server(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, p = None, self.lbry_server_port
|
||||
return defer.maybeDeferred(p.stopListening)
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def _setup_controller(self):
|
||||
self.controller = ConsoleControl()
|
||||
stdio.StandardIO(self.controller)
|
||||
logger = logging.getLogger()
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
alert_handler = logging.StreamHandler(self.controller)
|
||||
alert_handler.setFormatter(formatter)
|
||||
alert_handler.addFilter(logging.Filter("lbryalert"))
|
||||
alert_handler.setLevel(logging.DEBUG)
|
||||
logger.addHandler(alert_handler)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _start_controller(self):
|
||||
return self.controller.start(self.command_handlers)
|
||||
|
||||
def _shut_down(self):
|
||||
self.plugin_manager = None
|
||||
ds = []
|
||||
if self.lbry_file_metadata_manager is not None:
|
||||
d = self.lbry_file_metadata_manager.stop()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.stop())
|
||||
ds.append(d)
|
||||
ds.append(self.stop_server())
|
||||
ds.append(self._stop_plugins())
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
|
||||
|
||||
def launch_lbry_console():
|
||||
from twisted.internet import reactor
|
||||
|
||||
parser = argparse.ArgumentParser(description="Launch a lbrynet console")
|
||||
parser.add_argument("--no_listen_peer",
|
||||
help="Don't listen for incoming data connections.",
|
||||
action="store_true")
|
||||
parser.add_argument("--peer_port",
|
||||
help="The port on which the console will listen for incoming data connections.",
|
||||
type=int, default=3333)
|
||||
parser.add_argument("--no_listen_dht",
|
||||
help="Don't listen for incoming DHT connections.",
|
||||
action="store_true")
|
||||
parser.add_argument("--dht_node_port",
|
||||
help="The port on which the console will listen for DHT connections.",
|
||||
type=int, default=4444)
|
||||
parser.add_argument("--fake_wallet",
|
||||
help="Testing purposes only. Use a non-blockchain wallet.",
|
||||
action="store_true")
|
||||
parser.add_argument("--no_dht_bootstrap",
|
||||
help="Don't try to connect to the DHT",
|
||||
action="store_true")
|
||||
parser.add_argument("--dht_bootstrap_host",
|
||||
help="The hostname of a known DHT node, to be used to bootstrap into the DHT. "
|
||||
"Must be used with --dht_bootstrap_port",
|
||||
type=str, default='104.236.42.182')
|
||||
parser.add_argument("--dht_bootstrap_port",
|
||||
help="The port of a known DHT node, to be used to bootstrap into the DHT. Must "
|
||||
"be used with --dht_bootstrap_host",
|
||||
type=int, default=4000)
|
||||
parser.add_argument("--disable_upnp",
|
||||
help="Don't try to use UPnP to enable incoming connections through the firewall",
|
||||
action="store_true")
|
||||
parser.add_argument("--data_dir",
|
||||
help=("The full path to the directory in which lbrynet data and metadata will be stored. "
|
||||
"Default: ~/.lbrynet on linux, ~/Library/Application Support/lbrynet on OS X"),
|
||||
type=str)
|
||||
parser.add_argument("--lbrycrdd_path",
|
||||
help="The path to lbrycrdd, which will be launched if it isn't running. If"
|
||||
"this option is chosen, lbrycrdd will be used as the interface to the"
|
||||
"blockchain. By default, a lightweight interface is used.")
|
||||
parser.add_argument("--lbrycrd_wallet_dir",
|
||||
help="The directory in which lbrycrd data will stored. Used if lbrycrdd is "
|
||||
"launched by this application.")
|
||||
parser.add_argument("--lbrycrd_wallet_conf",
|
||||
help="The configuration file for the LBRYcrd wallet. Default: ~/.lbrycrd/lbrycrd.conf",
|
||||
type=str)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.no_dht_bootstrap:
|
||||
bootstrap_nodes = []
|
||||
else:
|
||||
bootstrap_nodes = [(args.dht_bootstrap_host, args.dht_bootstrap_port)]
|
||||
|
||||
if args.no_listen_peer:
|
||||
peer_port = None
|
||||
else:
|
||||
peer_port = args.peer_port
|
||||
|
||||
if args.no_listen_dht:
|
||||
dht_node_port = None
|
||||
else:
|
||||
dht_node_port = args.dht_node_port
|
||||
|
||||
created_data_dir = False
|
||||
if not args.data_dir:
|
||||
if sys.platform == "darwin":
|
||||
data_dir = user_data_dir("LBRY")
|
||||
else:
|
||||
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
data_dir = args.data_dir
|
||||
if not os.path.exists(data_dir):
|
||||
os.mkdir(data_dir)
|
||||
created_data_dir = True
|
||||
|
||||
daemon = LBRYAPIClient.config()
|
||||
try:
|
||||
daemon.is_running()
|
||||
log.info("Attempt to start lbrynet-console while lbrynet-daemon is running")
|
||||
print "lbrynet-daemon is running, you must turn it off before using lbrynet-console"
|
||||
print "If you're running the app, quit before starting lbrynet-console"
|
||||
print "If you're running lbrynet-daemon in a terminal, run 'stop-lbrynet-daemon' to turn it off"
|
||||
|
||||
webbrowser.open("http://localhost:5279")
|
||||
|
||||
except:
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
formatter = logging.Formatter(log_format)
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
file_handler = logging.FileHandler(os.path.join(data_dir, "console.log"))
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.addFilter(logging.Filter("lbrynet"))
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
|
||||
console = Console(peer_port, dht_node_port, bootstrap_nodes, fake_wallet=args.fake_wallet,
|
||||
lbrycrd_conf=args.lbrycrd_wallet_conf, lbrycrd_dir=args.lbrycrd_wallet_dir,
|
||||
use_upnp=not args.disable_upnp, data_dir=data_dir,
|
||||
created_data_dir=created_data_dir, lbrycrdd_path=args.lbrycrdd_path)
|
||||
|
||||
d = task.deferLater(reactor, 0, console.start)
|
||||
d.addErrback(lambda _: reactor.stop())
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
|
||||
reactor.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
launch_lbry_console()
|
||||
@@ -1,60 +1,176 @@
|
||||
from twisted.protocols import basic
|
||||
from twisted.internet import defer
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConsoleControl(basic.LineReceiver):
|
||||
from os import linesep as delimiter
|
||||
|
||||
def __init__(self, control_handlers):
|
||||
self.control_handlers = {}
|
||||
self.categories = {}
|
||||
categories = set([category for category, handler in control_handlers])
|
||||
prompt_number = 0
|
||||
for category in categories:
|
||||
self.categories[prompt_number] = category
|
||||
for handler in [handler for cat, handler in control_handlers if cat == category]:
|
||||
self.control_handlers[prompt_number] = handler
|
||||
prompt_number += 1
|
||||
def __init__(self):
|
||||
self.connected = False
|
||||
self.buffer = []
|
||||
self.command_handlers = {}
|
||||
self.current_handler = None
|
||||
|
||||
def start(self, command_handlers):
|
||||
self.command_handlers = {h.command: h for h in command_handlers}
|
||||
self.current_handler = None
|
||||
self.send_initial_prompt()
|
||||
return defer.succeed(True)
|
||||
|
||||
def connectionMade(self):
|
||||
self.connected = True
|
||||
if self.buffer:
|
||||
self.send(self.buffer)
|
||||
self.buffer = []
|
||||
|
||||
def send_initial_prompt(self):
|
||||
self.sendLine("")
|
||||
self.sendLine("You should have received 1000 LBC the first time you ran\n"
|
||||
"this program. If you did not, let us know! But first give\n"
|
||||
"them a moment to show up. The block time is currently 30\n"
|
||||
"seconds so they should show up within a couple of minutes.\n\n"
|
||||
"Welcome to lbrynet-console!")
|
||||
self.sendLine("")
|
||||
self.sendLine("Enter a command. Try 'get wonderfullife' or 'help' to see more options.")
|
||||
self.show_prompt()
|
||||
|
||||
def send(self, s):
|
||||
self.transport.write(s)
|
||||
|
||||
def write(self, s):
|
||||
if self.connected is False:
|
||||
self.buffer.append(s)
|
||||
else:
|
||||
self.send(s)
|
||||
|
||||
def flush(self):
|
||||
if self.connected is True and self.buffer:
|
||||
self.send(self.buffer)
|
||||
self.buffer = []
|
||||
|
||||
def show_prompt(self):
|
||||
self.send("> ")
|
||||
|
||||
def show_quick_help(self):
|
||||
self.sendLine("Available commands:")
|
||||
self.sendLine("")
|
||||
showed_help_all = False
|
||||
sorted_handlers = sorted(self.command_handlers.items(), key=lambda x: x[0])
|
||||
sorted_handlers = sorted(sorted_handlers, key=lambda x: x[1].priority, reverse=True)
|
||||
for command, handler in sorted_handlers:
|
||||
if handler.priority > 0:
|
||||
if showed_help_all is False and handler.priority < 10:
|
||||
self.sendLine("help-all - Show the full list of available commands")
|
||||
showed_help_all = True
|
||||
self.sendLine(command + " - " + handler.short_help)
|
||||
self.sendLine("")
|
||||
self.sendLine("For more information about any command type 'help <command>'")
|
||||
|
||||
def show_full_help(self):
|
||||
self.sendLine("Available commands:")
|
||||
self.sendLine("")
|
||||
for command, handler in sorted(self.command_handlers.items(), key=lambda x: x[0]):
|
||||
self.sendLine(command + " - " + handler.short_help)
|
||||
self.sendLine("")
|
||||
self.sendLine("For more information about any command type 'help <command>'")
|
||||
|
||||
def handler_done(self):
|
||||
self.current_handler = None
|
||||
self.show_prompt()
|
||||
|
||||
def handler_failed(self, err):
|
||||
self.current_handler = None
|
||||
self.sendLine("An error occurred:")
|
||||
self.sendLine(err.getTraceback())
|
||||
self.show_prompt()
|
||||
|
||||
def lineReceived(self, line):
|
||||
|
||||
def show_response(response):
|
||||
if response is not None:
|
||||
self.sendLine(response)
|
||||
|
||||
def show_error(err):
|
||||
self.sendLine(err.getTraceback())
|
||||
|
||||
if not self.command_handlers:
|
||||
return
|
||||
if self.current_handler is None:
|
||||
words = line.split()
|
||||
if len(words) == 0:
|
||||
self.show_prompt()
|
||||
return
|
||||
command, args = words[0], words[1:]
|
||||
if command == "help":
|
||||
if len(args) == 0:
|
||||
self.show_quick_help()
|
||||
self.show_prompt()
|
||||
return
|
||||
if args[0] in self.command_handlers:
|
||||
self.sendLine(self.command_handlers[args[0]].full_help)
|
||||
self.show_prompt()
|
||||
return
|
||||
if args[0] == "help-all":
|
||||
self.sendLine("Show the full list of available commands!")
|
||||
self.show_prompt()
|
||||
return
|
||||
self.sendLine("Can't help you with '%s'. Sorry!" % args[0])
|
||||
self.show_prompt()
|
||||
return
|
||||
elif command == "help-all":
|
||||
self.show_full_help()
|
||||
self.show_prompt()
|
||||
return
|
||||
if command in self.command_handlers:
|
||||
command_handler = self.command_handlers[command]
|
||||
else:
|
||||
candidates = [k for k in self.command_handlers.keys() if k.startswith(command)]
|
||||
if len(candidates) == 0:
|
||||
self.sendLine("Unknown command. Type 'help' for a list of commands.")
|
||||
self.show_prompt()
|
||||
return
|
||||
if len(candidates) >= 2:
|
||||
l = "Ambiguous command. Matches: "
|
||||
for candidate in candidates:
|
||||
l += candidate
|
||||
l += ", "
|
||||
l = l[:-2]
|
||||
l += l
|
||||
self.sendLine(l)
|
||||
self.show_prompt()
|
||||
return
|
||||
else:
|
||||
command_handler = self.command_handlers[candidates[0]]
|
||||
try:
|
||||
num = int(line)
|
||||
except ValueError:
|
||||
num = None
|
||||
if num in self.control_handlers:
|
||||
self.current_handler = self.control_handlers[num].get_handler()
|
||||
line = None
|
||||
if self.current_handler is not None:
|
||||
try:
|
||||
r = self.current_handler.handle_line(line)
|
||||
done, ds = r[0], [d for d in r[1:] if d is not None]
|
||||
except Exception as e:
|
||||
done = True
|
||||
ds = [defer.fail(e)]
|
||||
if done is True:
|
||||
self.current_handler = command_handler.get_handler(self)
|
||||
except:
|
||||
self.current_handler = None
|
||||
map(lambda d: d.addCallbacks(show_response, show_error), ds)
|
||||
if self.current_handler is None:
|
||||
self.show_prompt()
|
||||
|
||||
def show_prompt(self):
|
||||
self.sendLine("Options:")
|
||||
for num, handler in self.control_handlers.iteritems():
|
||||
if num in self.categories:
|
||||
self.sendLine("")
|
||||
self.sendLine(self.categories[num])
|
||||
self.sendLine("")
|
||||
self.sendLine("[" + str(num) + "] " + handler.get_prompt_description())
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
try:
|
||||
self.current_handler.start(*args)
|
||||
except TypeError:
|
||||
self.current_handler = None
|
||||
self.sendLine("Invalid arguments. Type 'help <command>' for the argument list.")
|
||||
import traceback
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
except:
|
||||
self.current_handler = None
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
self.current_handler.finished_deferred.addCallbacks(lambda _: self.handler_done(),
|
||||
self.handler_failed)
|
||||
else:
|
||||
try:
|
||||
self.current_handler.handle_line(line)
|
||||
except Exception as e:
|
||||
self.current_handler = None
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
File diff suppressed because it is too large.
Load diff
Loaded 100 of 251 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user